${escapeHtml(section.heading)} · ${section.rows.length}
${escapeHtml(section.lead)}
| ${
escapeHtml(section.measure)
} | kind | member | test |
|---|
${body}
`;
}
/**
* What the flake share counts. The share is over runs rather than over
* failures, so a test that passes reliably reads as small however many of
* its rare failures were flakes, and a disagreement weighs less the more
* runs have followed it, so a test that has settled since reads lower
* than one that has just started. The counts beside it are what let a reader weigh it,
* since one disagreement in two runs and a thousand in two thousand are
* the same ratio and not the same claim.
*/
function flakeLead(manifest: Manifest): string {
const days = numberDial(
manifest.dials,
"FLAKE_WINDOW_DAYS",
FLAKE_WINDOW_FALLBACK_DAYS,
);
const exclusion = numberDial(
manifest.dials,
"FLAKE_EXCLUSION_RATE",
FLAKE_EXCLUSION_FALLBACK,
);
return `The share of the runs each test took part in over the last ${days} ` +
"days that it was seen disagreeing with itself over: the same commit " +
"both passing and failing, with nothing between the two runs but " +
"chance. A test that is deterministic cannot do that, so nothing is " +
"charged against the count and one disagreement among two runs reads " +
"as the half it is; a disagreement counts for less as the test goes " +
"on running without repeating it. " +
`Past ${percent(exclusion)} a test is held back from pull requests, ` +
"and the share falls again as it goes on running without disagreeing, " +
"so the exclusion reverses on its own. The counts are flat, so they " +
"are not what the share divides.";
}
/** The tests held back as flaky, worst-measured first. */
function flakyRows(manifest: Manifest): TestRow[] {
const scored = new Map(
manifest.entries.map((entry) => [testIdentityKey(entry.test), entry]),
);
return manifest.withheld
.filter((entry) => entry.reason === "flaky")
.map((entry) => ({
test: entry.test,
scored: scored.get(testIdentityKey(entry.test)),
}))
.sort((a, b) => (b.scored?.flakeRate ?? 0) - (a.scored?.flakeRate ?? 0))
.map(({ test, scored }) => ({
test,
...(scored === undefined ? {} : { measure: flakeMeasure(scored) }),
}));
}
/**
* One test's flake share with the counts it was taken from, where the
* manifest carries them. The share alone says how the exclusion was
* decided; the counts say how much is behind that decision.
*/
function flakeMeasure(entry: ManifestEntry): string {
const share = percent(entry.flakeRate);
const evidence = entry.flakeEvidence;
if (evidence === undefined) return share;
return `${share} · ${groupDigits(evidence.flakes)} in ${
groupDigits(evidence.runs)
}`;
}
/** The lanes a pull request would run, each against the budget it was packed to. */
function lanesSection(manifest: Manifest): string {
const budget = laneBudgetOf(manifest.dials);
const rows = manifest.lanes.map((lane) => {
const tests = laneTestCount(lane);
const over = lane.projectedSeconds > budget;
const filled = Math.min(100, (lane.projectedSeconds / budget) * 100);
return `Lane ${lane.lane}` +
`` +
`${seconds(lane.projectedSeconds)} of ${
seconds(budget)
}` +
`${groupDigits(tests)} tests
`;
}).join("");
return `Lanes
What each lane is projected to spend against the budget this manifest was packed to. The publisher packs these with nothing mandatory, so this is the run a pull request touching no test would get: a real one re-packs against its own diff, placing what that diff makes mandatory first and whatever the cost, and filling what is left of the budget from the same corpus.
${rows}`;
}
/** A figure, and what it was in the manifest before where that differs. */
function against(now: string, was: string | undefined): string {
return was === undefined || was === now
? escapeHtml(now)
: `${escapeHtml(now)} was ${escapeHtml(was)}`;
}
/**
* Whether the cost model this manifest was packed by still describes what
* lanes spend: what the publisher found broken, how many lanes ran past their
* bound, and each suite's charges against the manifest before and against
* what its batches spent. The suites a lane pays most to hold come first,
* since those are nearest to crowding everything else out.
*/
function healthSection(manifest: Manifest): string {
const health = manifest.health;
const heading = `Cost model
`;
if (health === undefined) {
return `${heading}
This manifest carries no figures about its cost model that this page can read.
`;
}
const budget = laneBudgetOf(manifest.dials);
const alarms = health.alarms.length === 0
? ""
: `The cost model is broken.${
health.alarms.map((alarm) => `- ${escapeHtml(alarm)}
`).join("")
}
`;
const rows = Object.entries(health.suites)
.sort(([a, x], [b, y]) => y.fixed - x.fixed || (a < b ? -1 : 1))
.map(([suite, figures]) => {
const was = health.previous?.suites[suite];
return ` budget ? ' class="over"' : ""}>` +
`${escapeHtml(suite)} | ` +
`${
against(
seconds(figures.fixed),
was === undefined ? undefined : seconds(was.fixed),
)
} | ` +
`${
against(
String(figures.tooLong),
was === undefined ? undefined : String(was.tooLong),
)
} | ` +
`${groupDigits(figures.batches)} | ` +
`${
figures.ratio === undefined ? "—" : figures.ratio.median.toFixed(2)
} | ` +
`${
figures.ratio === undefined ? "—" : figures.ratio.p90.toFixed(2)
} | `;
}).join("");
return `${heading}
${escapeHtml(healthLead(health))}
${alarms}
`;
}
/**
* What the figures in the cost model's table are, and how the lanes it was
* measured against came out.
*/
function healthLead(health: CalibrationHealth): string {
const { observed, pastBound, projectedInside, overran } = health.lanes;
const lanes = observed === 0
? "No lane in the cost window recorded its work as a whole."
: `${groupDigits(pastBound)} of the ${
groupDigits(observed)
} lanes in the cost window ran past their bound, ${
groupDigits(overran)
} of them among the ${
groupDigits(projectedInside)
} the packer projected to finish inside it.`;
const previous = health.previous === undefined
? ""
: ` A grayed figure is the one in the manifest of ${
minutePrecision(health.previous.generatedAt)
}.`;
return `${lanes} "To hold" is what a lane pays before it runs anything of ` +
"a suite; past a lane's budget, nothing can share a lane with it. " +
'"Too long" counts its tests no lane can hold. The last two columns ' +
"are what its batches spent over what the packer charged for them, at " +
"the middle batch and at the ninetieth percentile, which the model is " +
`fitted to hold near one.${previous}`;
}
/** The counts the two tiles headline, spelled out. */
function summary(manifest: Manifest): string {
const selected = selectedSeconds(manifest);
const corpus = corpusSeconds(manifest);
const facts: Array<[string, string]> = [
["selected", shareLabel(selected, corpus)],
[
"test time",
`${humanDuration(selected * 1_000)} of ${
humanDuration(corpus * 1_000)
}`,
],
[
"tests",
`${groupDigits(selectedCount(manifest))} of ${
groupDigits(manifest.entries.length)
}`,
],
["lanes", String(manifest.lanes.length)],
["flaky", groupDigits(flakyCount(manifest))],
];
return `${
facts.map(([term, value]) =>
`- ${term}
- ${escapeHtml(value)}
`
).join("")
}
`;
}
/**
* The page's frame, which every state of it wears. Everything that changes
* from one manifest to the next is inside ``, which the page brings up
* to date as the server renders it again.
*/
function frame(head: string, body: string): LivePageContent {
return {
title: "Test selection",
styles: STYLES,
head,
body: `${body}
What a pull request runs, from the newest manifest the selection publisher wrote.
`,
};
}
/** The page for one manifest, or the page saying there is not one. */
export function testSelectionPage(
manifest: Manifest | undefined,
now = Date.now(),
): LivePageContent {
if (manifest === undefined) {
return frame(
"",
`No selection manifest has been published yet.
`,
);
}
const head = `commit ${escapeHtml(manifest.commit.slice(0, 7))} · generated ${
escapeHtml(minutePrecision(manifest.generatedAt))
}, ${compactSpan(now - Date.parse(manifest.generatedAt))} ago`;
return frame(
head,
`${summary(manifest)}
${lanesSection(manifest)}
${healthSection(manifest)}
${
testSection({
heading: "Held back as flaky",
id: FLAKY_SECTION_ID,
lead: flakeLead(manifest),
measure: "flake share",
rows: flakyRows(manifest),
})
}
${
testSection({
heading: "Too long for any lane",
id: UNSCHEDULABLE_SECTION_ID,
lead:
"One execution costs more than a whole lane's budget, so no packing can place them.",
measure: "cost",
rows: [...manifest.unschedulable]
.sort((a, b) => b.cost - a.cost)
.map((entry) => ({
test: entry.test,
measure: seconds(entry.cost),
})),
})
}`,
);
}
/**
* The page for a store that could not be read, which is a different thing
* from a store holding no manifest and says so.
*/
export function testSelectionUnavailable(reason: string): LivePageContent {
return frame(
"",
`The selection manifest could not be read: ${
escapeHtml(reason)
}.
`,
);
}
/** Serves the page against the manifest the tiles are already reading. */
export async function testSelectionResponse(
read: ManifestReader = sharedTestSelection.latest,
clock?: () => number,
): Promise {
try {
return livePageResponse(testSelectionPage(await read(), clock?.()));
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error("test selection page:", message);
return livePageResponse(
testSelectionUnavailable(collectionSub(error)),
503,
);
}
}