#!/usr/bin/env -S deno run --allow-read --allow-env --allow-net --allow-run=git /** * Everything a person types about test selection goes through here, and * these modes are also how the system is checked by hand. * * deno task test-selection dials * deno task test-selection coverage * deno task test-selection explain * deno task test-selection plan --dry-run [--lane N] * deno task test-selection plan --verify * deno task test-selection health * * `explain` is the one this will be asked most often, because "why did my * test not run?" is the question a selected run provokes and the one it * would otherwise answer badly. * * Every mode that reads a manifest reads the one the lanes testing this * checkout's commit read, or the one current at the moment `--at` names. * `health` reads the newest one instead, since what it judges is the * model the lanes are packing by now. * `plan` and `explain` pack the tree through the lanes' own code, for a * change that touches nothing, so what they say a lane would do is what * such a lane does. */ import { duration } from "./test-selection/duration.ts"; import { join } from "@std/path"; import { loadAliasResolver, repositoryRoot, type TestIdentity, testIdentityKey, } from "@commonfabric/test-support/records"; import { maxOf } from "@commonfabric/utils/math"; import { DIALS, EXCLUDED_FROM_COVERAGE_GATE, LANE_BUDGET_SECONDS, } from "./test-selection/policy.ts"; import { fetchManifest } from "./test-selection/store.ts"; import { type LaneDeps, type LaneOptions, type LanePlan, lanePlan, resolveManifest, } from "./ci-lane.ts"; import { capabilitiesBySuite, loadTopology } from "./test-topology.ts"; import { type Suite, unavailableUnits } from "./test-topology/suite.ts"; import { measuredCostLines } from "./test-selection/coverage.ts"; import { coverageGateIdentity, measuredSetName, type MeasuredSetRef, measuredSets, } from "./test-topology/coverage-gates.ts"; import type { Manifest } from "./test-selection/manifest.ts"; import { calibrationHealth, healthLines } from "./test-selection/health.ts"; import { nearestBaseline, nearestOnBranch } from "./coverage-gate.ts"; import { crowdingLine, placedOnlyMandatory, unholdableSuites, } from "./test-selection/plan.ts"; import { readWorkspaceMembers } from "./workspace-tests.ts"; const USAGE = `usage: test-selection dials every dial, its value, and why you would move it coverage every workspace member and its coverage gate explain one test's score, and whether it is selected plan --dry-run [--lane N] what would run, and what it would cost plan --verify what the topology and the store disagree about health whether the cost model still describes the lanes, failing where it has broken Every mode that reads a manifest reads the one the lanes testing the checked-out commit read, except health, which reads the newest one. --at , in ISO 8601, reads the one that was current at that moment instead. An identity is its canonical key, either three parts or four when the test ran in a non-default configuration: ["unit","memory","space > writes a fact"] ["integration","patterns","counter.test.ts","server-execution"] `; /** * The lane a `--lane` argument names: a number, nothing when the flag is * absent, and "invalid" for a flag that names no lane. */ export function laneArgument( args: readonly string[], ): number | undefined | "invalid" { const at = args.indexOf("--lane"); if (at < 0) return undefined; const lane = Number(args[at + 1]); if (!Number.isInteger(lane) || lane < 1) return "invalid"; return lane; } /** * The moment an `--at` argument names, in the UTC form the store compares * against: nothing when the flag is absent, and "invalid" for a flag with * no moment after it that a `Date` can read. */ export function momentArgument( args: readonly string[], ): string | undefined | "invalid" { const at = args.indexOf("--at"); if (at < 0) return undefined; const moment = new Date(args[at + 1] ?? ""); return Number.isNaN(moment.getTime()) ? "invalid" : moment.toISOString(); } /** * What the program says before stopping, and the code it stops with. * Raised rather than exited so the whole dispatch runs in a test. */ export class Stop extends Error { readonly code: number; constructor(message: string, code: number) { super(message); this.name = "Stop"; this.code = code; } } /** Stops with the usage code, which is what a mistyped command line gets. */ function fail(message: string): never { throw new Stop(message, 2); } /** Pads a column so a printed table lines up. */ function pad(text: string, width: number): string { return text.length >= width ? text : text + " ".repeat(width - text.length); } /** Every dial, as the lines `dials` prints. */ export function dialLines(): string[] { const lines: string[] = []; const width = maxOf(DIALS.map((dial) => dial.name.length)); for (const dial of DIALS) { lines.push( `${pad(dial.name, width)} ${dial.value} ${dial.unit} ` + `(${dial.setBy})`, ); lines.push(`${" ".repeat(width)} ${dial.why}`); lines.push(""); } lines.push( "setupCost and each suite's fit come from measurement too, and are " + "published\nin each manifest rather than kept here.", ); return lines; } /** * Each measured set, its baseline and its gate's score, then every * workspace member that carries no set and the reason it does not. * * The two halves answer the two questions somebody brings here: what am I * being compared against, and why is my package not gated. The baseline * is the one the coverage gate takes, chosen by `nearest` from the * commits the manifest holds one at. */ export async function coverageLines( manifest: Manifest | undefined, sets: readonly MeasuredSetRef[], members: readonly string[], nearest: (commits: readonly string[]) => Promise, ): Promise { const names = sets.map(measuredSetName); const ungated = members.filter((member) => !sets.some((ref) => ref.set.member === member) ); const width = Math.max( 1, maxOf([...names, ...ungated].map((name) => name.length)), ); const entries = new Map( (manifest?.entries ?? []).map((entry) => [ testIdentityKey(entry.test), entry, ]), ); const lines: string[] = []; for (const [index, ref] of sets.entries()) { const baseline = await nearestBaseline( manifest?.coverageBaselines ?? [], ref.suite, ref.set.member, nearest, ); const against = baseline === undefined ? "no baseline yet" : `${baseline.uncoveredLines} uncovered lines at ${baseline.commit}`; const units = ref.set.units.length; const gate = entries.get(testIdentityKey(coverageGateIdentity(ref))); const scored = gate === undefined ? "its gate has no record yet" : `its gate scores ${gate.score.toFixed(3)}`; lines.push( `${pad(names[index]!, width)} ${units} ` + `${units === 1 ? "unit" : "units"}, against ${against}; ${scored}`, ); } for (const member of ungated) { const excluded = EXCLUDED_FROM_COVERAGE_GATE.get(member); lines.push( `${pad(member, width)} no measured set: ` + `${excluded ?? "it has no Deno-only tests"}`, ); } return lines; } /** The workspace members the coverage gate has an opinion about. */ export async function gatedMembers(): Promise { const root = repositoryRoot() ?? Deno.cwd(); return (await readWorkspaceMembers(join(root, "deno.jsonc"))) .map((member) => member.replace(/^\.\//, "")) .filter((member) => member.startsWith("packages/")) .sort(); } /** The identity a command-line argument names. */ export function parseIdentityArgument(text: string): TestIdentity | undefined { let parts: unknown; try { parts = JSON.parse(text); } catch { return undefined; } if (!Array.isArray(parts) || parts.length < 3 || parts.length > 4) { return undefined; } const [k, s, n, v] = parts; // Empty parts are not an identity. The manifest validator refuses them, // so accepting one here would report a test as unknown and mandatory // when what was really wrong was the argument. const named = (part: unknown): part is string => typeof part === "string" && part.length > 0; if (!named(k) || !named(s) || !named(n)) return undefined; if (v !== undefined && !named(v)) return undefined; const test: TestIdentity = { k, s, n }; if (typeof v === "string") test.v = v; return test; } /** What the packing decided about one identity. */ export interface PlanVerdict { selected: boolean; /** How many times it would run, which a pass may have trimmed. */ repeats?: number; /** Set when its unit runs whole and holds a withheld test. */ heldWithUnit?: boolean; /** Set when no lane can hold it, whatever the budget. */ unschedulable?: boolean; /** Set when the plan places nothing beyond its mandatory tests. */ crowded?: boolean; /** * What a lane running nothing else would pay for it: its corrected own * time plus every overhead and setup that lane would open. This is the * figure the hard bound is compared against, so it is the one to report * when the answer is that no lane can hold it. */ loneSeconds?: number; } /** What `explain` says about one identity, as lines. */ export function explainLines( manifest: Manifest, test: TestIdentity, verdict: PlanVerdict = { selected: false }, ): string[] { const key = testIdentityKey(test); const entry = manifest.entries.find( (candidate) => testIdentityKey(candidate.test) === key, ); if (entry === undefined) { return [ `${key}`, " The manifest has no entry for it, and what that means depends on", " the tree. An available unit no manifest knows is mandatory, so a", " test just added, one just renamed, and one whose records never", " say which unit it is in all run until a run on `main` records", " them. Nothing runs a test the tree no longer holds, and nothing", " runs one whose unit a configuration declares unavailable. A", " coverage gate is the exception: no lane runs one, and one no", " manifest knows is offered at the floor of the score until the", " default branch records it.", ]; } const lines = [ `${key}`, ` suite ${entry.suite}, in ${entry.unit}`, ` score ${entry.score.toFixed(3)}, costing ${duration(entry.cost)}`, ` ${entry.inputs.catches.toFixed(1)} weighted new real test failures, ` + `across ${entry.inputs.sources} sources`, entry.inputs.lastCatch === undefined ? " it has never had a new real test failure" : ` the most recent new real test failure was on ` + `${entry.inputs.lastCatch}`, ` churn ${entry.inputs.churn.toFixed(4)}, flake rate ` + `${entry.flakeRate.toFixed(4)}`, ]; // Each of the lines below is a fact of its own rather than one arm of a // choice, because the facts do not partition: a withheld identity a // change reaches runs, and every identity that runs has a number of // runs it is given. const held = manifest.withheld.find( (candidate) => testIdentityKey(candidate.test) === key, ); if (held !== undefined) { // What is said about a held-back identity stops at what holds it // back, because a change that reaches one runs it: saying it runs on // the default branch and not here would contradict the selection // line below on every identity a change edits. lines.push( " withheld: it is too flaky to judge a change by, so a pull " + "request runs it only where the change reaches it", ); } if (verdict.unschedulable) { const seconds = verdict.loneSeconds ?? entry.cost; lines.push( ` no lane can hold it: ${duration(seconds)} is past the bound a ` + "lane runs under, so it is reported rather than scheduled. Splitting " + "it is the fix.", ); } // The repeat count the packing settled on, which is what will run: a // filling pass trims repeats to fit rather than dropping the test, so // the manifest's own number is what it asked for and not what it got. const repeats = verdict.selected ? verdict.repeats ?? entry.repeats : 0; if (repeats > 1) { lines.push(` run ${repeats} times, and every one must pass`); } // The question this mode exists to answer. Everything above is a fact // about the entry; whether a lane reaches it is a fact about the // packing, and only the packing knows it. if (verdict.selected) { lines.push(" this commit's manifest selects it"); } else if (verdict.heldWithUnit) { lines.push( " held back with its unit: the unit runs whole and holds a test too " + "flaky to judge a change by, so a pull request runs it only where " + "that unit is one that must run", ); } else if (!verdict.unschedulable && held === undefined) { lines.push( verdict.crowded ? " this commit's manifest does not reach it: the tests that must " + "run leave no room in the lanes for anything else" : " this commit's manifest does not reach it: the lanes fill what " + "the tests that must run leave of their budget with other tests", ); } return lines; } /** A one-line summary of what a lane would do. */ function laneLine( lane: { lane: number; selections: unknown[]; projectedSeconds: number }, ): string { return ` lane ${lane.lane}: ${lane.selections.length} tests, ` + `${duration(lane.projectedSeconds)} of ${duration(LANE_BUDGET_SECONDS)}`; } /** What `plan --dry-run` prints, as lines. */ export function planLines( planned: LanePlan, laneNumber: number | undefined, ): string[] { const lines: string[] = []; const { laid } = planned; const corpus = planned.seen.manifest; const lanes = laneNumber === undefined ? laid.lanes : laid.lanes.filter((lane) => lane.lane === laneNumber); lines.push( `manifest of ${corpus.generatedAt}, from ${corpus.runs} runs at ` + `${corpus.commit}`, ); // The corpus the lanes below were packed from, which is what this tree // holds rather than what the manifest was published over. Counting the // manifest's own entries here would head a plan with a total the plan // does not add up to. lines.push( `${corpus.entries.length} identities in this tree, ` + `${corpus.withheld.length} withheld`, ); for (const lane of lanes) { lines.push(laneLine(lane)); if (lane.capabilities.length > 0) { lines.push(` needs ${lane.capabilities.join(", ")}`); } const byReason = new Map(); for (const selection of lane.selections) { byReason.set( selection.reason, (byReason.get(selection.reason) ?? 0) + 1, ); } for (const [reason, count] of [...byReason].sort()) { lines.push(` ${count} by ${reason}`); } } for (const gate of laid.gates) { lines.push( `gate ${testIdentityKey(gate.entry.test)} by ${gate.reason}, run in ` + `whatever lanes hold what it judges`, ); } if (laid.overBudgetSeconds > 0) { lines.push( `the mandatory set alone puts a lane ` + `${duration(laid.overBudgetSeconds)} past its budget`, ); } // A suite whose fixed charge alone is past a lane comes first, and its // identities are not listed under it. Every one of them is past the // bound by that charge and by nothing about itself, so a list of them // is one line per test saying what one line per suite already said. const unholdable = unholdableSuites(laid.crowding); for (const suite of laid.crowding) lines.push(crowdingLine(suite)); for (const entry of laid.unschedulable) { if (unholdable.has(entry.suite)) continue; lines.push( `unschedulable: ${testIdentityKey(entry.test)} costs ` + `${duration(entry.cost)}, past a lane's whole budget`, ); } lines.push( `${laid.lanes.length} lanes, ${duration(LANE_BUDGET_SECONDS)} of work each`, ); return lines; } /** * What the lanes would do with one test. Reads the lanes' own plan, so * the answer is the one the lanes would give rather than a guess from the * manifest entry alone. */ export function verdictFor( planned: LanePlan, test: TestIdentity, ): PlanVerdict & { corpus: Manifest } { const { laid } = planned; // A gate is no lane's work, so the corpus the lanes were packed from does // not hold it, and it is added back for the identity being explained. const corpus: Manifest = { ...planned.seen.manifest, entries: [ ...planned.seen.manifest.entries, ...planned.seen.gates.map((gate) => gate.entry), ], }; const key = testIdentityKey(test); const taken = [ ...laid.lanes.flatMap((lane) => lane.selections), ...laid.gates, ] .find((selection) => testIdentityKey(selection.entry.test) === key); // The corpus travels with the verdict so that whatever explains this // identity explains it against the same set the verdict came from. A // reader given the published manifest instead would be told there is // no record of a unit this tree has just added, while the verdict // beside it says a lane runs the stand-in for it. const verdict: PlanVerdict & { corpus: Manifest } = { selected: taken !== undefined, corpus, }; if (taken !== undefined) verdict.repeats = taken.repeats; if (placedOnlyMandatory(laid)) verdict.crowded = true; if (laid.heldWithUnit.some((held) => testIdentityKey(held) === key)) { verdict.heldWithUnit = true; } const refused = laid.unschedulable.find((entry) => testIdentityKey(entry.test) === key ); if (refused !== undefined) { verdict.unschedulable = true; verdict.loneSeconds = refused.cost; } return verdict; } /** What comparing the manifest against the working tree found. */ export interface Verification { lines: string[]; /** Whether anything it found should stop a build. */ fails: boolean; } /** * Whether a manifest accounts for the tree in front of it. * * Two directions, and they are not the same claim. A unit the topology * enumerates that the manifest holds nothing for is one every lane * treats as unknown and therefore mandatory, so a manifest missing many * of them is a run that selects nothing and tests everything. That is * what this fails on, and it is the check to run before anything depends * on selection working. A coverage gate the manifest holds nothing for is * no lane's work and is not mandatory: it is offered at the floor of the * score until a run records it, and this fails on that too, since a gate * the default branch never records never earns its place. * * A manifest entry naming a suite or unit the tree no longer holds is * work no lane can be asked to do, and the packer drops it silently. It * is reported rather than failed on, the way the drift guard reports its * own reverse direction: a manifest is hours old by construction, so a * unit deleted since it was published is expected to linger in it. */ export function verifyLines( manifest: Manifest, suites: readonly Suite[], ): Verification { const held = new Set( manifest.entries.map((entry) => `${entry.suite}\t${entry.unit}`), ); const enumerated = new Set(); const missing: string[] = []; for (const suite of suites) { // A unit this configuration declares unavailable does not run, so a // manifest holding nothing for it is the manifest being right. const unavailable = unavailableUnits(suite); for (const unit of suite.units) { if (unavailable.has(unit)) continue; enumerated.add(`${suite.id}\t${unit}`); if (held.has(`${suite.id}\t${unit}`)) continue; // A gate is no lane's work, so a gate nothing has recorded is not run // as unknown; it waits at the floor of the score for its first record. missing.push( suite.judges === undefined ? `no identity for ${suite.id}: ${unit}, so every lane runs it as ` + `unknown` : `no record of the gate ${suite.id}: ${unit}, so a change is ` + `offered it at the floor of the score`, ); } } const stale = [...held].filter((key) => !enumerated.has(key)).sort(); const lines = [ `manifest of ${manifest.generatedAt}, from ${manifest.runs} runs at ` + `${manifest.commit}`, `${enumerated.size} units enumerated, ${manifest.entries.length} ` + `identities in the manifest`, ]; for (const line of missing.sort()) lines.push(` ${line}`); for (const key of stale) { const [suite, unit] = key.split("\t") as [string, string]; lines.push( ` reported: ${suite} no longer enumerates ${unit}, so nothing runs it`, ); } if (missing.length === 0) { lines.push(" the manifest accounts for every unit the topology holds"); } return { lines, fails: missing.length > 0 }; } /** What the dispatch reads that is not in its arguments. */ export interface Sources { /** Where manifests come from, read at a moment, as a lane reads them. */ manifest: LaneDeps["manifest"]; /** The workspace members the coverage gate has an opinion about. */ members(): Promise; /** * The alias file, which joins a renamed test to its own history. Named * by what the dispatch asks of it rather than by the class the live one * happens to be. */ aliases(): Promise<{ resolve(test: TestIdentity, day: string): TestIdentity; }>; /** The suites, read from the working tree at `root`. */ topology(root: string): Promise; /** Which of a set of commits the checkout at `root` holds most recently. */ nearest( root: string, ): (commits: readonly string[]) => Promise; } const LIVE: Sources = { manifest: fetchManifest, members: gatedMembers, aliases: loadAliasResolver, topology: loadTopology, nearest: nearestOnBranch, }; /** * The manifest `root`'s commit resolves, or the one current `at` where a * moment is named. Returns `undefined`, having said why, where there is * none. */ async function commitManifest( root: string, at: string | undefined, sources: Sources, ): Promise { const found = await resolveManifest( at === undefined ? { root } : { root, at }, sources, ); if (found.manifest === undefined) { console.error(`no manifest: ${found.absent}`); } return found.manifest; } /** * The plan every lane testing `root`'s commit computes for a change that * touches nothing, over the manifest current `at` where a moment is * named. Returns `undefined`, having said why, where there is no manifest * to compute it from. */ async function commitPlan( root: string, at: string | undefined, sources: Sources, ): Promise { const options: LaneOptions = { lane: 1, full: false, dryRun: true, laneCount: false, root, ...(at === undefined ? {} : { at }), }; const planned = await lanePlan( options, await sources.topology(root), sources, ); if (planned.fetched.absent !== undefined) { console.error(`no manifest: ${planned.fetched.absent}`); return undefined; } return planned; } /** * Runs one command line against the checkout at `root` and returns the * code to stop with. Every line it means a person to read goes to the * console; every reason to stop is a `Stop`, so nothing here ends the * process. */ export async function dispatch( args: readonly string[], sources: Sources = LIVE, root: string = repositoryRoot() ?? Deno.cwd(), ): Promise { const mode = args[0]; if (mode === undefined || mode === "--help" || mode === "-h") { console.log(USAGE); return 0; } // A moment this cannot read is a mistake to report rather than a reason // to read the commit's manifest in its place. const at = momentArgument(args); if (at === "invalid") fail(`--at takes a moment in ISO 8601\n\n${USAGE}`); switch (mode) { case "dials": for (const line of dialLines()) console.log(line); return 0; case "coverage": { // The one mode that reads a manifest and carries on without one: // which sets exist is a fact about the tree, and only the baseline // each is measured against and what the lanes have fitted running // one costs come from a manifest. const manifest = await commitManifest(root, at, sources); const topology = await sources.topology(root); const sets = measuredSets(topology); for ( const line of await coverageLines( manifest, sets, await sources.members(), sources.nearest(root), ) ) { console.log(line); } // What a set costs is read from the manifest's fitted costs, so a // tree read without one has nothing to say about it. if (manifest !== undefined) { for (const line of measuredCostLines(manifest, topology)) { console.log(line); } } return 0; } case "explain": { const argument = args[1]; if (argument === undefined) fail(USAGE); const test = parseIdentityArgument(argument); if (test === undefined) { fail(`not an identity key: ${argument}\n\n${USAGE}`); } const planned = await commitPlan(root, at, sources); if (planned === undefined) return 1; // Resolving through the alias file is what joins the two halves of // a renamed test's history under today's name. const resolver = await sources.aliases(); const resolved = resolver.resolve( test, planned.seen.manifest.generatedAt.slice(0, 10), ); const verdict = verdictFor(planned, resolved); for (const line of explainLines(verdict.corpus, resolved, verdict)) { console.log(line); } return 0; } case "plan": { // Before the manifest, because a lane number this cannot read is a // mistake to report rather than a reason to reach for the store. const laneNumber = laneArgument(args); // Silently filtering every lane would exit zero having printed // nothing, which reads as "this lane runs no tests". if (laneNumber === "invalid") { fail("--lane takes a whole number from 1"); } if (args.includes("--verify")) { const manifest = await commitManifest(root, at, sources); if (manifest === undefined) return 1; const verification = verifyLines( manifest, await sources.topology(root), ); for (const line of verification.lines) console.log(line); return verification.fails ? 1 : 0; } const planned = await commitPlan(root, at, sources); if (planned === undefined) return 1; const lanes = planned.laid.lanes.length; if (laneNumber !== undefined && laneNumber > lanes) { fail(`--lane names lane ${laneNumber}, and this plan has ${lanes}`); } for (const line of planLines(planned, laneNumber)) console.log(line); return 0; } case "health": { // The newest manifest rather than the commit's, because what this // judges is the model the lanes are packing by now. const found = await sources.manifest({ at: at ?? new Date().toISOString(), }); if (found.manifest === undefined) { console.error(`no manifest: ${found.absent}`); return 1; } const manifest = found.manifest; console.log(`the manifest of ${manifest.generatedAt}`); // A manifest published before the publisher measured its model // carries no health. What can be read from it and the one before it // is read, and what only the lanes' measurements say is absent. let health = manifest.health; if (health === undefined) { console.log( "it carries no health, so it is judged on its own charges and the " + "manifest before it, and no lane figures are read", ); const before = await sources.manifest({ at: manifest.generatedAt }); const topology = await sources.topology(root); if (before.unreachable) { console.error(`no previous manifest: ${before.absent}`); return 1; } health = calibrationHealth({ manifest, previous: before.manifest, capabilities: capabilitiesBySuite(topology), observations: { charges: [], lanes: [] }, }); } for (const line of healthLines(health)) console.log(line); return health.alarms.length === 0 ? 0 : 1; } default: fail(`unknown mode ${mode}\n\n${USAGE}`); } } if (import.meta.main) { try { Deno.exit(await dispatch(Deno.args)); } catch (error) { if (!(error instanceof Stop)) throw error; console.error(error.message); Deno.exit(error.code); } }