/** * Tile tests: each tile is a pure collect(ctx) -> TileView, exercised with a * hand-made Ctx. No server, no network, no subprocess — the CI tiles get canned * runs and the token-gated tiles get an empty env (their gray-out contract). */ import { assert, assertEquals, assertRejects, assertStringIncludes, } from "@std/assert"; import { TILE_LAYOUT_FIXTURES } from "./tile-layout-fixtures.ts"; import { type Ctx, type Run, runSource, type RunSource } from "./types.ts"; import { CI_WORKFLOW, LOOM_CI_WORKFLOW, LOOM_REPO, REPO, WEAVER_CI_WORKFLOW, WEAVER_REPO, } from "./config.ts"; import { labsCiTrust, loomCiTrust, weaverCiTrust } from "./tiles/ci-trust.ts"; import { labsCiDuration, loomCiDuration, weaverCiDuration, } from "./tiles/ci-duration.ts"; import { recentCommits } from "./tiles/recent-commits.ts"; import { dau, foldSeries, parseExcludes } from "./tiles/dau.ts"; import { githubMembers } from "./tiles/github-members.ts"; import { buildSnapshot, discordOnline } from "./tiles/discord-online.ts"; import { gcpSpend } from "./tiles/gcp-spend.ts"; import { prodErrors } from "./tiles/prod-errors.ts"; import { GITHUB_LAG_DAYS, githubCiSpend, } from "./tiles/github-ci-spend.ts"; import { projectMonthly, settled } from "./spend.ts"; import { modelSpend } from "./tiles/model-spend.ts"; import { benchmark, trendPct, trendStatus } from "./tiles/benchmark.ts"; import { TILES } from "./registry.ts"; import { REPOS_PATH } from "./repo-page-href.ts"; import { byUrl, collectFromWorkingRuns, type GithubAnswer, withGithubAttempt, } from "./test/github-attempts.ts"; function ctx( runs: Run[], env: Record = {}, runsByRepo?: (repo: string) => Run[], ): Ctx { return { runs: () => Promise.resolve(runs), runsFor: ({ repo }) => Promise.resolve(runsByRepo ? runsByRepo(repo) : runs), env: (k) => env[k], }; } let nextRunId = 1; function run(over: Partial): Run { const startedAt = new Date(Date.now() - 3_600_000).toISOString(); return { id: nextRunId++, status: "completed", conclusion: "success", run_attempt: 1, event: "push", head_sha: "sha", display_title: "t", created_at: startedAt, run_started_at: startedAt, updated_at: new Date().toISOString(), html_url: "", head_commit: { message: "t (#1)" }, ...over, }; } // An earlier attempt of `of`, as GitHub's attempt endpoint returns it. function attemptOf(of: Run, attempt: number, conclusion: string): Run { return { ...of, run_attempt: attempt, conclusion }; } function attemptUrl(attempt: Run, repo = REPO): string { return `https://api.github.com/repos/${repo}/actions/runs/${attempt.id}/attempts/${attempt.run_attempt}`; } // Where the job listing of `attempt` starts. function jobsUrl(attempt: Run, repo = REPO): string { return jobPageUrl(attempt, 1, repo); } // A cancelled attempt's job listing when a newer run replaced it before it // listed any job. const NO_JOBS = { total_count: 0, jobs: [] }; // A job a newer run replaced while it was queued. const QUEUED = { id: 1, conclusion: "cancelled", runner_name: "" }; // A job a runner was running when its attempt was cancelled, here by its own // `timeout-minutes`, as its annotations say. const STOPPED = { id: 2, conclusion: "cancelled", runner_name: "GitHub Actions 1", }; // A cancelled attempt's job listing when it stopped a job a runner was running. const STOPPED_JOB = { total_count: 1, jobs: [STOPPED] }; // A job a runner was running when a concurrency group cancelled it in favor // of a newer run, as its annotations say. const REPLACED = { id: 3, conclusion: "cancelled", runner_name: "GitHub Actions 1", }; // What GitHub annotates a job with when it cancels it for each reason, beside // an annotation every job on the runner gets. const RUNNER_NOTICE = { annotation_level: "notice", message: "The ubuntu-latest label will migrate to Ubuntu 26", }; const REASONS = { replaced: [{ annotation_level: "failure", message: "Canceling since a higher priority waiting request for ci-refs/heads/main exists", }, RUNNER_NOTICE], timedOut: [RUNNER_NOTICE, { annotation_level: "failure", message: "The job has exceeded the maximum execution time of 1h10m0s", }], stopped: [{ annotation_level: "failure", message: "The run was canceled by @someone.", }], }; // Where page `page` of the annotations of `job`'s check run is requested. function annotationsUrl( job: { readonly id: number }, repo = REPO, page = 1, ): string { return `https://api.github.com/repos/${repo}/check-runs/${job.id}/annotations?per_page=100&page=${page}`; } // The answers for the annotations of STOPPED and REPLACED. function annotationAnswers(repo = REPO): [string, GithubAnswer][] { return [ [annotationsUrl(STOPPED, repo), REASONS.timedOut], [annotationsUrl(REPLACED, repo), REASONS.replaced], ]; } // The trust grid's cell colors, in the newest-first order the runs arrive in. function cellColors(extra: string | undefined): string[] { return [...(extra ?? "").matchAll(/background:(var\(--[^)]+\))/g)] .map((match) => match[1]) .reverse(); } Deno.test("labs ci trust: only first-attempt success counts as green", async () => { // Two of four completed runs passed first try. A success on retry remains in // the denominator but does not count as first-try green. const retried = run({ conclusion: "success", run_attempt: 2 }); const runs = [ run({ conclusion: "success", run_attempt: 1 }), run({ conclusion: "success", run_attempt: 1 }), retried, run({ conclusion: "failure" }), ]; await withGithubAttempt( attemptOf(retried, 1, "failure"), async () => { const v = await labsCiTrust.collect(ctx(runs)); assertEquals(v.value, "50.0%"); assertEquals(v.status, "bad"); assertEquals(v.sub, "first-try green · last 4 runs"); }, ); }); Deno.test("ci trust: each tile links to its repository's page, and each cell to its run", async () => { const cases = [ { tile: labsCiTrust, page: "/repos?name=labs" }, { tile: loomCiTrust, page: "/repos?name=loom" }, { tile: weaverCiTrust, page: "/repos?name=commonfabric-weaver" }, ]; for (const { tile, page } of cases) { const passed = run({ html_url: `https://github.com/${tile.repo}/actions/runs/1` }); const v = await tile.collect(ctx([passed])); assertEquals(v.href, page); assertEquals(v.hint, "repository ↗"); assertStringIncludes(v.extra ?? "", `href="${passed.html_url}"`); } }); Deno.test("labs and loom ci trust: a cancelled run that never started is left out of the share", async () => { const cases = [ { tile: labsCiTrust, repo: REPO }, { tile: loomCiTrust, repo: LOOM_REPO }, ]; for (const { tile, repo } of cases) { const cancelled = [ run({ conclusion: "cancelled" }), run({ conclusion: "cancelled" }), ]; const runs = [ run({ conclusion: "success" }), cancelled[0], cancelled[1], run({ conclusion: "failure" }), ]; const jobs = cancelled.map((attempt) => jobsUrl(attempt, repo)); await withGithubAttempt( byUrl(jobs.map((url) => [url, NO_JOBS])), async (urls) => { const v = await tile.collect(ctx(runs)); assertEquals(v.value, "50.0%"); assertEquals(v.sub, "first-try green · 2 of last 4 runs"); assertEquals(cellColors(v.extra), [ "var(--status-good)", "var(--status-unknown)", "var(--status-unknown)", "var(--status-bad)", ]); assertEquals(urls.toSorted(), jobs.toSorted()); }, ); } }); Deno.test("labs and loom ci trust: a cancelled run that stopped a job on a runner counts as a failure", async () => { const cases = [ { tile: labsCiTrust, repo: REPO }, { tile: loomCiTrust, repo: LOOM_REPO }, ]; for (const { tile, repo } of cases) { const timedOut = run({ conclusion: "cancelled" }); const runs = [ run({ conclusion: "success" }), timedOut, run({ conclusion: "success" }), run({ conclusion: "failure" }), ]; await withGithubAttempt( byUrl([[jobsUrl(timedOut, repo), STOPPED_JOB], ...annotationAnswers(repo)]), async (urls) => { const v = await tile.collect(ctx(runs)); assertEquals(v.value, "50.0%"); assertEquals(v.sub, "first-try green · last 4 runs"); assertEquals(cellColors(v.extra), [ "var(--status-good)", "var(--status-bad)", "var(--status-good)", "var(--status-bad)", ]); assertEquals(urls, [ jobsUrl(timedOut, repo), annotationsUrl(STOPPED, repo), ]); }, ); } }); Deno.test("labs and loom ci trust: a cancelled run is left out of the share only when a concurrency group stopped the jobs it ran", async () => { // A concurrency group that cancels in progress stops a run's jobs on their // runners when a newer run starts. A timeout or a person stopping them // instead says something about the commit. const cases = [ { tile: labsCiTrust, repo: REPO }, { tile: loomCiTrust, repo: LOOM_REPO }, ]; const reasons = [ { annotations: REASONS.replaced, color: "var(--status-unknown)" }, { annotations: REASONS.timedOut, color: "var(--status-bad)" }, { annotations: REASONS.stopped, color: "var(--status-bad)" }, { annotations: [RUNNER_NOTICE], color: "var(--status-bad)" }, { annotations: [], color: "var(--status-bad)" }, // GitHub's annotations may carry no message. { annotations: [{ message: null }], color: "var(--status-bad)" }, { annotations: [{ message: null }, ...REASONS.replaced], color: "var(--status-unknown)", }, ]; for (const { tile, repo } of cases) { for (const { annotations, color } of reasons) { const cancelled = run({ conclusion: "cancelled" }); const passed = { id: 4, conclusion: "success", runner_name: "Runner 2" }; await withGithubAttempt( byUrl([ [jobsUrl(cancelled, repo), { total_count: 3, jobs: [passed, REPLACED, QUEUED], }], [annotationsUrl(REPLACED, repo), annotations], ]), async (urls) => { const v = await tile.collect(ctx([ run({ conclusion: "success" }), cancelled, ])); assertEquals(cellColors(v.extra), ["var(--status-good)", color]); assertEquals(urls, [ jobsUrl(cancelled, repo), annotationsUrl(REPLACED, repo), ]); }, ); } } }); Deno.test("labs ci trust: a cancelled attempt is a failure when any job stopped on a runner was not stopped by a concurrency group", async () => { // A job timed out, and then a newer run replaced the run while its other // jobs ran. const cancelled = run({ conclusion: "cancelled" }); await withGithubAttempt( byUrl([ [jobsUrl(cancelled), { total_count: 3, jobs: [REPLACED, STOPPED, REPLACED], }], ...annotationAnswers(), ]), async (urls) => { const v = await labsCiTrust.collect(ctx([cancelled])); assertEquals(v.value, "0.0%"); assertEquals(urls, [ jobsUrl(cancelled), annotationsUrl(REPLACED), annotationsUrl(STOPPED), ]); }, ); }); Deno.test("labs ci trust: a cancelled job's annotations are read page by page until one gives the reason", async () => { const cancelled = run({ conclusion: "cancelled" }); const warnings = Array.from({ length: 100 }, () => RUNNER_NOTICE); await withGithubAttempt( byUrl([ [jobsUrl(cancelled), { total_count: 1, jobs: [REPLACED] }], [annotationsUrl(REPLACED, REPO, 1), warnings], [annotationsUrl(REPLACED, REPO, 2), REASONS.replaced], ]), async (urls) => { const v = await labsCiTrust.collect(ctx([ run({ conclusion: "success" }), cancelled, ])); assertEquals(v.value, "100.0%"); assertEquals(urls, [ jobsUrl(cancelled), annotationsUrl(REPLACED, REPO, 1), annotationsUrl(REPLACED, REPO, 2), ]); }, ); }); Deno.test("labs ci trust: annotations GitHub does not return fail the collection", async (t) => { const cases: { name: string; answer: GithubAnswer; message: string }[] = [ { name: "failed request", answer: new Error("annotations unavailable"), message: "annotations unavailable", }, { name: "answer that is not an array", answer: NO_JOBS, message: "is not an array of objects each with a string or null", }, { name: "annotation without a message", answer: [{}], message: "is not an array of objects each with a string or null", }, { name: "annotation whose message is not a string", answer: [{ message: 7 }], message: "is not an array of objects each with a string or null", }, ]; for (const { name, answer, message } of cases) { await t.step(name, async () => { const cancelled = run({ conclusion: "cancelled" }); await withGithubAttempt( byUrl([ [jobsUrl(cancelled), { total_count: 1, jobs: [REPLACED] }], [annotationsUrl(REPLACED), answer], ]), async (urls) => { await assertRejects( () => labsCiTrust.collect(ctx([cancelled])), Error, message, ); assertEquals(urls, [jobsUrl(cancelled), annotationsUrl(REPLACED)]); }, ); }); } }); Deno.test("labs and loom ci trust: a cancelled run whose cancelled jobs never reached a runner is left out of the share", async () => { // A job-level concurrency group replaces the waiting job rather than the // run, so the run lists the job, cancelled with an empty runner name, beside // the jobs that ran. const cases = [ { tile: labsCiTrust, repo: REPO }, { tile: loomCiTrust, repo: LOOM_REPO }, ]; for (const { tile, repo } of cases) { const replaced = run({ conclusion: "cancelled" }); const runs = [ run({ conclusion: "success" }), replaced, run({ conclusion: "failure" }), ]; await withGithubAttempt( byUrl([[jobsUrl(replaced, repo), { total_count: 3, jobs: [ QUEUED, { id: 4, conclusion: "success", runner_name: "GitHub Actions 2" }, { id: 5, conclusion: "skipped", runner_name: null }, ], }]]), async (urls) => { const v = await tile.collect(ctx(runs)); assertEquals(v.value, "50.0%"); assertEquals(v.sub, "first-try green · 2 of last 3 runs"); assertEquals(urls, [jobsUrl(replaced, repo)]); assertEquals(cellColors(v.extra), [ "var(--status-good)", "var(--status-unknown)", "var(--status-bad)", ]); }, ); } }); Deno.test("labs ci trust: a cancelled run with a job that failed on a runner counts as a failure", async () => { // Someone cancels a run after one job failed and while another was queued. const cancelled = run({ conclusion: "cancelled" }); await withGithubAttempt( byUrl([[jobsUrl(cancelled), { total_count: 2, jobs: [ { id: 4, conclusion: "failure", runner_name: "GitHub Actions 1" }, QUEUED, ], }]]), async (urls) => { const v = await labsCiTrust.collect(ctx([ run({ conclusion: "success" }), cancelled, ])); assertEquals(v.value, "50.0%"); // A job that failed needs no annotation to say why. assertEquals(urls, [jobsUrl(cancelled)]); assertEquals(cellColors(v.extra), [ "var(--status-good)", "var(--status-bad)", ]); }, ); }); Deno.test("labs ci trust: the job listing of a cancelled attempt is read only as far as its first job that did not succeed on a runner", async () => { const timedOut = run({ conclusion: "cancelled" }); await withGithubAttempt( byUrl([[ jobPageUrl(timedOut, 1), { total_count: 101, jobs: [STOPPED, ...Array.from({ length: 99 }, () => QUEUED)], }, ], ...annotationAnswers()]), async (urls) => { const v = await labsCiTrust.collect(ctx([timedOut])); assertEquals(v.value, "0.0%"); assertEquals(urls, [jobPageUrl(timedOut, 1), annotationsUrl(STOPPED)]); }, ); }); Deno.test("labs ci trust: a job cancelled on a runner counts as a failure on any page of the job listing", async () => { const timedOut = run({ conclusion: "cancelled" }); await withGithubAttempt( byUrl([ [ jobPageUrl(timedOut, 1), { total_count: 101, jobs: Array.from({ length: 100 }, () => QUEUED) }, ], [jobPageUrl(timedOut, 2), { total_count: 101, jobs: [STOPPED] }], ...annotationAnswers(), ]), async (urls) => { const v = await labsCiTrust.collect(ctx([ run({ conclusion: "success" }), timedOut, ])); assertEquals(v.value, "50.0%"); assertEquals(cellColors(v.extra), [ "var(--status-good)", "var(--status-bad)", ]); assertEquals(urls, [ jobPageUrl(timedOut, 1), jobPageUrl(timedOut, 2), annotationsUrl(STOPPED), ]); }, ); }); Deno.test("labs ci trust: a run is green only when the one attempt left once cancellations without a verdict are passed over succeeded", async () => { const afterUnstartedGreen = run({ conclusion: "success", run_attempt: 2 }); const afterUnstartedRed = run({ conclusion: "failure", run_attempt: 2 }); const redAfterUnstarted = run({ conclusion: "success", run_attempt: 3 }); const rerunOfGreen = run({ conclusion: "success", run_attempt: 2 }); const neverStarted = run({ conclusion: "cancelled", run_attempt: 2 }); const cancelledRetry = run({ conclusion: "cancelled", run_attempt: 2 }); const successAfterTimeout = run({ conclusion: "success", run_attempt: 2 }); const failedRerunOfGreen = run({ conclusion: "failure", run_attempt: 2 }); const timedOutRerunOfGreen = run({ conclusion: "cancelled", run_attempt: 2 }); const unstartedRerunOfGreen = run({ conclusion: "cancelled", run_attempt: 2, }); const earlier = [ attemptOf(afterUnstartedGreen, 1, "cancelled"), attemptOf(afterUnstartedRed, 1, "cancelled"), attemptOf(redAfterUnstarted, 1, "cancelled"), attemptOf(redAfterUnstarted, 2, "failure"), attemptOf(rerunOfGreen, 1, "success"), attemptOf(neverStarted, 1, "cancelled"), attemptOf(cancelledRetry, 1, "failure"), attemptOf(successAfterTimeout, 1, "cancelled"), attemptOf(failedRerunOfGreen, 1, "success"), attemptOf(timedOutRerunOfGreen, 1, "success"), attemptOf(unstartedRerunOfGreen, 1, "success"), ]; const unstarted = [ attemptOf(afterUnstartedGreen, 1, "cancelled"), attemptOf(afterUnstartedRed, 1, "cancelled"), attemptOf(redAfterUnstarted, 1, "cancelled"), attemptOf(neverStarted, 1, "cancelled"), neverStarted, unstartedRerunOfGreen, ]; const ranJobs = [ attemptOf(successAfterTimeout, 1, "cancelled"), timedOutRerunOfGreen, ]; await withGithubAttempt( byUrl([ ...earlier.map((attempt) => [attemptUrl(attempt), attempt] as const), ...unstarted.map((attempt) => [jobsUrl(attempt), NO_JOBS] as const ), ...ranJobs.map((attempt) => [jobsUrl(attempt), STOPPED_JOB] as const ), ...annotationAnswers(), ]), async (urls) => { const v = await labsCiTrust.collect(ctx([ afterUnstartedGreen, afterUnstartedRed, redAfterUnstarted, rerunOfGreen, neverStarted, cancelledRetry, successAfterTimeout, failedRerunOfGreen, timedOutRerunOfGreen, unstartedRerunOfGreen, ])); // Green: the first and last, each left with one successful attempt once // the attempt cancelled before starting is passed over. The fifth was // cancelled before starting on every attempt and is left out. Every other // run is red: it failed, or ran jobs before it was cancelled, or needed a // rerun of an attempt that was left. assertEquals(cellColors(v.extra), [ "var(--status-good)", "var(--status-bad)", "var(--status-bad)", "var(--status-bad)", "var(--status-unknown)", "var(--status-bad)", "var(--status-bad)", "var(--status-bad)", "var(--status-bad)", "var(--status-good)", ]); assertEquals(v.value, "22.2%"); assertEquals(v.sub, "first-try green · 9 of last 10 runs"); assertEquals( urls.toSorted(), [ ...earlier.map((attempt) => attemptUrl(attempt)), ...[...unstarted, ...ranJobs].map((attempt) => jobsUrl(attempt)), ...ranJobs.map(() => annotationsUrl(STOPPED)), ].toSorted(), ); }, ); }); Deno.test("labs ci trust: an earlier attempt is requested once", async () => { const retried = run({ conclusion: "success", run_attempt: 2 }); const earlier = attemptOf(retried, 1, "cancelled"); await withGithubAttempt( byUrl([ [attemptUrl(earlier), earlier], [jobsUrl(earlier), NO_JOBS], ]), async (urls) => { assertEquals((await labsCiTrust.collect(ctx([retried]))).value, "100.0%"); assertEquals((await labsCiTrust.collect(ctx([retried]))).value, "100.0%"); assertEquals(urls, [attemptUrl(earlier), jobsUrl(earlier)]); }, ); }); Deno.test("labs ci trust: the job listing of a cancelled attempt is requested once, and no other attempt's is", async () => { const neverStarted = run({ conclusion: "cancelled" }); const timedOut = run({ conclusion: "cancelled" }); const runs = [ run({ conclusion: "success" }), neverStarted, run({ conclusion: "failure" }), timedOut, run({ conclusion: "timed_out" }), ]; const jobs = [jobsUrl(neverStarted), jobsUrl(timedOut)]; await withGithubAttempt( byUrl([[jobs[0], NO_JOBS], [jobs[1], STOPPED_JOB], ...annotationAnswers()]), async (urls) => { for (let collection = 0; collection < 2; collection++) { const v = await labsCiTrust.collect(ctx(runs)); assertEquals(v.value, "25.0%"); assertEquals(v.sub, "first-try green · 4 of last 5 runs"); } assertEquals( urls.toSorted(), [...jobs, annotationsUrl(STOPPED)].toSorted(), ); }, ); }); Deno.test("labs ci trust: a job listing is forgotten once its run leaves the window", async () => { const cancelled = run({ conclusion: "cancelled" }); const other = run({ conclusion: "success" }); await withGithubAttempt(NO_JOBS, async (urls) => { await labsCiTrust.collect(ctx([cancelled])); await labsCiTrust.collect(ctx([other])); await labsCiTrust.collect(ctx([cancelled])); assertEquals(urls, [jobsUrl(cancelled), jobsUrl(cancelled)]); }); }); Deno.test("labs ci trust: an earlier attempt GitHub does not return fails the collection", async () => { const retried = run({ conclusion: "success", run_attempt: 2 }); await withGithubAttempt( new Error("attempt endpoint unavailable"), async () => { await assertRejects( () => labsCiTrust.collect(ctx([retried])), Error, "attempt endpoint unavailable", ); }, ); }); Deno.test("labs ci trust: an earlier attempt that is not the one asked for fails the collection", async () => { const retried = run({ conclusion: "success", run_attempt: 2 }); const answers: Run[] = [ // Still going, so it carries no verdict to read. { ...attemptOf(retried, 1, "failure"), status: "in_progress", conclusion: null }, // Another run's attempt. { ...attemptOf(retried, 1, "failure"), id: retried.id + 1 }, // Another attempt of the same run. attemptOf(retried, 2, "failure"), ]; for (const answer of answers) { await withGithubAttempt(answer, async () => { await assertRejects( () => labsCiTrust.collect(ctx([retried])), Error, "did not include a completed conclusion", ); }); } }); Deno.test("labs ci trust: a job listing GitHub does not return fails the collection", async (t) => { const cases: { name: string; answer: GithubAnswer; message: string }[] = [ { name: "failed request", answer: new Error("job listing unavailable"), message: "job listing unavailable", }, { name: "count that is not a number", answer: { total_count: "38" }, message: "did not include a numeric `total_count`", }, { name: "missing count", answer: { total_count: undefined }, message: "did not include a numeric `total_count`", }, { name: "missing jobs", answer: { total_count: 1 }, message: "did not include a numeric `total_count` and a `jobs` array", }, { name: "listing that ends short of its count", answer: { total_count: 2, jobs: [] }, message: "job listing ended after 0 of its 2 jobs", }, { name: "job without a runner name", answer: { total_count: 1, jobs: [{ id: 1, conclusion: "failure" }] }, message: "holds a job without a numeric `id` and a string or null", }, { name: "job without an id", answer: { total_count: 1, jobs: [{ conclusion: "failure", runner_name: "GitHub Actions 1" }], }, message: "holds a job without a numeric `id` and a string or null", }, ]; for (const { name, answer, message } of cases) { await t.step(name, async () => { const cancelled = run({ conclusion: "cancelled" }); await withGithubAttempt(answer, async (urls) => { await assertRejects( () => labsCiTrust.collect(ctx([ run({ conclusion: "success" }), cancelled, ])), Error, message, ); assertEquals(urls, [jobsUrl(cancelled)]); }); }); } }); Deno.test( "labs ci trust grid: every run in the latest 160-run window has a cell", async () => { const createdAt = (index: number) => new Date(index * 1_000).toISOString(); const runs = [ run({ status: "in_progress", conclusion: null, created_at: createdAt(160), }), ...Array.from({ length: 159 }, (_, index) => run({ conclusion: "success", created_at: createdAt(159 - index), })), run({ conclusion: "failure", created_at: createdAt(0) }), ]; const view = await labsCiTrust.collect(ctx(runs)); const cells = (view.extra ?? "").match(/class="cell"/g)?.length ?? 0; assertEquals(cells, 160); assertStringIncludes( view.extra ?? "", 'class="cells labeled"', ); assertEquals( view.sub, "first-try green · 159 of last 160 runs", ); assert( !(view.extra ?? "").includes("var(--status-bad)"), "the 161st run must be outside the grid and percentage window", ); }, ); Deno.test("labs ci trust grid: cell colors match trust scoring", async () => { const retried = run({ conclusion: "success", run_attempt: 2 }); const neverStarted = run({ conclusion: "cancelled" }); const timedOut = run({ conclusion: "cancelled" }); const runs = [ run({ conclusion: "success", run_attempt: 1 }), retried, run({ conclusion: "failure" }), neverStarted, timedOut, run({ status: "in_progress", conclusion: null }), run({ status: "queued", conclusion: null }), run({ status: "completed", conclusion: null }), ]; const earlier = attemptOf(retried, 1, "failure"); await withGithubAttempt( byUrl([ [attemptUrl(earlier), earlier], [jobsUrl(neverStarted), NO_JOBS], [jobsUrl(timedOut), STOPPED_JOB], ...annotationAnswers(), ]), async () => { const view = await labsCiTrust.collect(ctx(runs)); const colors = cellColors(view.extra); assertEquals(view.value, "25.0%"); assertEquals(colors.filter((color) => color === "var(--status-good)").length, 1); assertEquals(colors.filter((color) => color === "var(--status-bad)").length, 3); assertEquals(colors.filter((color) => color === "var(--running)").length, 1); assertEquals( colors.filter((color) => color === "var(--status-unknown)").length, 3, ); }, ); }); Deno.test("labs ci trust grid: duration spans the displayed runs", async () => { const now = Date.now(); const runs = [ run({ created_at: new Date(now).toISOString() }), run({ created_at: new Date(now - 2 * 86_400_000).toISOString() }), run({ created_at: new Date(now - 5 * 86_400_000).toISOString() }), ]; const view = await labsCiTrust.collect(ctx(runs)); assertEquals(view.duration, 5 * 86_400_000); assertStringIncludes(view.extra ?? "", 'class="cells labeled"'); const malformed = await labsCiTrust.collect(ctx([ runs[0], run({ created_at: "not a timestamp" }), runs[2], ])); assertEquals(malformed.duration, 0); assert(!(malformed.extra ?? "").includes("cells labeled")); const window = Array.from({ length: 161 }, (_, i) => run({ created_at: new Date(now - i * 86_400_000).toISOString() }) ); const limited = await labsCiTrust.collect(ctx(window)); assertEquals(limited.duration, 159 * 86_400_000); }); Deno.test("ci-duration window: the 6h window when it has >= 20 runs, else the most recent 20", async () => { const now = Date.now(); const at = (minsAgo: number) => run({ created_at: new Date(now - minsAgo * 60_000).toISOString(), run_started_at: new Date(now - minsAgo * 60_000).toISOString(), updated_at: new Date(now - minsAgo * 60_000 + 5 * 60_000).toISOString(), }); // 25 runs within the last ~25 min -> the 6h window wins (25 >= 20). const busy = Array.from({ length: 25 }, (_, i) => at(i)); assertStringIncludes( (await collectFromWorkingRuns(labsCiDuration, ctx(busy))).sub ?? "", "median of 25 PR runs in 6h", ); // 5 recent + 30 from two days ago -> only 5 in 6h (< 20) -> fall back to the last 20. const quiet = [ ...Array.from({ length: 5 }, (_, i) => at(i)), ...Array.from({ length: 30 }, () => at(60 * 48)), ]; assertStringIncludes( (await collectFromWorkingRuns(labsCiDuration, ctx(quiet))).sub ?? "", "median of last 20 PR runs", ); }); Deno.test("ci-duration: only runs that passed end to end count", async () => { const now = Date.now(); const at = (i: number, over: Partial) => run({ created_at: new Date(now - i * 60_000).toISOString(), run_started_at: new Date(now - i * 60_000).toISOString(), updated_at: new Date(now - i * 60_000 + 5 * 60_000).toISOString(), ...over, }); const runs = [ ...Array.from({ length: 20 }, (_, i) => at(i, { conclusion: "success" })), at(0, { conclusion: "failure" }), at(1, { conclusion: "cancelled" }), at(2, { conclusion: "timed_out" }), at(3, { status: "in_progress", conclusion: null }), // Passed on a rerun, so its span includes the wait before the rerun. at(4, { run_attempt: 2, conclusion: "success" }), ]; // Only the 20 runs that passed on their first attempt are counted. assertStringIncludes( (await collectFromWorkingRuns(labsCiDuration, ctx(runs))).sub ?? "", "median of 20 PR runs in 6h", ); }); Deno.test("ci-duration: a run without a usable landing span is dropped", async () => { const now = Date.now(); const usable = run({ created_at: new Date(now - 10 * 60_000).toISOString(), run_started_at: new Date(now - 10 * 60_000).toISOString(), updated_at: new Date(now).toISOString(), }); // A final update at or before the landing trigger, and a landing trigger that // does not parse. Both would otherwise reach the median as a duration of zero // or NaN minutes. const endsBeforeItLands = run({ created_at: new Date(now).toISOString(), run_started_at: new Date(now).toISOString(), updated_at: new Date(now - 60_000).toISOString(), }); const unparseableLanding = run({ created_at: "the fourteenth of never", run_started_at: new Date(now - 20 * 60_000).toISOString(), updated_at: new Date(now).toISOString(), }); const mixed = await collectFromWorkingRuns(labsCiDuration, ctx([usable, endsBeforeItLands, unparseableLanding]), ); assertEquals(mixed.value, "10m"); assertStringIncludes(mixed.sub ?? "", "median of last 1 PR run"); const none = await collectFromWorkingRuns(labsCiDuration, ctx([endsBeforeItLands, unparseableLanding]), ); assertEquals(none.status, "unknown"); assertEquals(none.value, "—"); assertEquals(none.sub, "no passing PR runs"); }); // Where page `page` of the job listing of `attempt` is requested. function jobPageUrl(attempt: Run, page: number, repo = REPO): string { return `${attemptUrl(attempt, repo)}/jobs?per_page=100&page=${page}`; } // A job listing page holding one job for each conclusion in `conclusions`, // each run on a runner unless it was skipped. function jobPage(total: number, conclusions: readonly string[]): GithubAnswer { return { total_count: total, jobs: conclusions.map((conclusion, id) => ({ id, conclusion, runner_name: conclusion === "skipped" ? null : "GitHub Actions 1", })), }; } Deno.test("weaver ci duration: a run that ran no job, or ran one job and skipped the rest, did no work and is left out", async () => { const now = Date.now(); const minutes = (m: number) => run({ created_at: new Date(now - m * 60_000).toISOString(), run_started_at: new Date(now - m * 60_000).toISOString(), updated_at: new Date(now).toISOString(), }); const statusOnly = minutes(1); const jobless = minutes(2); const oneJobWorkflow = minutes(8); const worked = minutes(12); const view = await withGithubAttempt( byUrl([ [ jobPageUrl(statusOnly, 1, WEAVER_REPO), jobPage(3, ["skipped", "success", "skipped"]), ], [jobPageUrl(jobless, 1, WEAVER_REPO), jobPage(0, [])], [jobPageUrl(oneJobWorkflow, 1, WEAVER_REPO), jobPage(1, ["success"])], [ jobPageUrl(worked, 1, WEAVER_REPO), jobPage(3, ["skipped", "success", "success"]), ], ]), () => weaverCiDuration.collect( ctx([statusOnly, jobless, oneJobWorkflow, worked]), ), ); // The median of 8 and 12 minutes; the one- and two-minute runs are not // counted. assertEquals(view.value, "10m"); assertStringIncludes(view.sub ?? "", "median of last 2 PR runs"); }); Deno.test("ci-duration: a job listing is read to its last page, once", async () => { const now = Date.now(); const long = run({ created_at: new Date(now - 20 * 60_000).toISOString(), run_started_at: new Date(now - 20 * 60_000).toISOString(), updated_at: new Date(now).toISOString(), }); const skippedPage = Array.from({ length: 100 }, () => "skipped"); const pages = [jobPageUrl(long, 1), jobPageUrl(long, 2)]; await withGithubAttempt( byUrl([ [pages[0], jobPage(102, skippedPage)], [pages[1], jobPage(102, ["success", "success"])], ]), async (urls) => { // Only the second page shows that more than one job ran. assertEquals((await labsCiDuration.collect(ctx([long]))).value, "20m"); assertEquals(urls, pages); await labsCiDuration.collect(ctx([long])); assertEquals(urls, pages, "a completed run's job counts are held"); }, ); }); Deno.test("ci-duration: a job listing that cannot be read fails the collection", async () => { const unreadable = run({}); await withGithubAttempt( byUrl([[jobPageUrl(unreadable, 1), { total_count: 1 }]]), () => assertRejects( () => labsCiDuration.collect(ctx([unreadable])), Error, "job listing page 1", ), ); }); Deno.test("ci-duration: a run's job counts are forgotten once it leaves the snapshot", async () => { const kept = run({}); const passing = run({}); const listing = jobPage(2, ["success", "success"]); await withGithubAttempt( byUrl([[jobPageUrl(kept, 1), listing], [jobPageUrl(passing, 1), listing]]), async (urls) => { await labsCiDuration.collect(ctx([kept, passing])); await labsCiDuration.collect(ctx([kept])); await labsCiDuration.collect(ctx([kept, passing])); assertEquals(urls, [ jobPageUrl(kept, 1), jobPageUrl(passing, 1), jobPageUrl(passing, 1), ]); }, ); }); Deno.test("ci trust: every repository keeps its strip at the tile bottom", async () => { const one = ctx([run({ conclusion: "success" })]); for (const tile of [labsCiTrust, loomCiTrust, weaverCiTrust]) { assertEquals((await tile.collect(one)).alignChartBottom, true); } }); Deno.test("runSource creates workflow snapshot metadata", () => { assertEquals(runSource(REPO, CI_WORKFLOW, "main"), { repo: REPO, workflow: CI_WORKFLOW, scope: "main", }); assertEquals(runSource(REPO, CI_WORKFLOW, "pull requests"), { repo: REPO, workflow: CI_WORKFLOW, scope: "pull requests", }); }); Deno.test("CI tiles declare the workflow snapshots that drive them", () => { const main = (repo: string, workflow: string): RunSource[] => [ { repo, workflow, scope: "main" }, ]; const pulls = (repo: string, workflow: string): RunSource[] => [ { repo, workflow, scope: "pull requests" }, ]; assertEquals(labsCiTrust.runSources, main(REPO, CI_WORKFLOW)); assertEquals(loomCiTrust.runSources, main(LOOM_REPO, LOOM_CI_WORKFLOW)); assertEquals( weaverCiTrust.runSources, main(WEAVER_REPO, WEAVER_CI_WORKFLOW), ); assertEquals(labsCiDuration.runSources, pulls(REPO, CI_WORKFLOW)); assertEquals(loomCiDuration.runSources, pulls(LOOM_REPO, LOOM_CI_WORKFLOW)); assertEquals( weaverCiDuration.runSources, pulls(WEAVER_REPO, WEAVER_CI_WORKFLOW), ); assertEquals(recentCommits.runSources, [ ...main(REPO, CI_WORKFLOW), ...main(LOOM_REPO, LOOM_CI_WORKFLOW), ...main(WEAVER_REPO, WEAVER_CI_WORKFLOW), ]); }); Deno.test("dau: distinct identities per UTC day, excluding the DIDs we name", () => { const D = 86_400_000; const ser = (did: string, pts: [number, number][]) => ({ labels: [{ key: { name: "user.did" }, value: did }], values: pts.map(([timestamp, value]) => ({ timestamp, value })), }); const day1 = 1_783_641_600_000, day2 = day1 + D; const series = [ ser("did:key:alice", [[day1, 998], [day2, 3184]]), ser("did:key:bob", [[day2, 12]]), ser("did:key:server", [[day1, 500], [day2, 500]]), // a service principal ]; const all = foldSeries(series, new Set()); assertEquals(all.get(day1)?.size, 2); assertEquals(all.get(day2)?.size, 3); // Naming the service principal takes it out of every day it appears in. const human = foldSeries(series, new Set(["did:key:server"])); assertEquals(human.get(day1)?.size, 1); assertEquals(human.get(day2)?.size, 2); // A bucket carrying no spans for an identity is not that identity being active. assertEquals( foldSeries([ser("did:key:alice", [[day1, 0]])], new Set()).size, 0, ); }); Deno.test("dau: the exclusion list is read defensively", () => { assertEquals([...parseExcludes("did:key:a, did:key:b ,, ")], [ "did:key:a", "did:key:b", ]); assertEquals(parseExcludes(undefined).size, 0); assertEquals(parseExcludes("").size, 0); }); Deno.test("gated tiles gray out cleanly without their env", async () => { const cases = [ [discordOnline, "DISCORD_BOT_TOKEN"], [gcpSpend, "GCP_BILLING_TABLE"], [prodErrors, "SIGNOZ_URL"], [githubCiSpend, "GH_TOKEN"], [modelSpend, "OPENAI_ADMIN_KEY"], [benchmark, "GH_TOKEN"], [dau, "SIGNOZ_URL"], [githubMembers, "GH_TOKEN"], ] as const; for (const [tile, needle] of cases) { const v = await tile.collect(ctx([])); assertEquals(v.status, "unknown"); assertEquals(v.value, "—"); assertStringIncludes(v.sub ?? "", needle); } }); Deno.test("discord snapshot: splits online members into Team and Visitors", () => { const guild = { id: "g", roles: [ { id: "team", name: "Team", color: 0x2ecc71, position: 5 }, { id: "g", name: "@everyone", color: 0, position: 0 }, ], members: [ { user: { id: "a" }, roles: ["team"] }, { user: { id: "b" }, roles: [] }, { user: { id: "c" }, roles: ["team"] }, // offline, excluded { user: { id: "d" }, roles: ["team"] }, ], presences: [ { user: { id: "a" }, status: "online" }, { user: { id: "b" }, status: "idle" }, { user: { id: "c" }, status: "offline" }, { user: { id: "d" }, status: "dnd" }, ], }; const snap = buildSnapshot(guild); assert(snap); assertEquals(snap.online, 3); // a, b, d (c is offline) assertEquals(snap.team, 2); // a, d carry the role and are online assertEquals(snap.visitors, 1); // b assertEquals(snap.teamColor, "#2ecc71"); // the role's own color }); Deno.test("projectMonthly: >=2-week (or month-to-date) window, spilling into last month", () => { const lastMonth = Array.from({ length: 30 }, () => 100); // $100/day last month // >= 2 weeks of data this month -> pure month-to-date rate, last month ignored. // $2000 over 20 days, 30-day month -> $3000. assertEquals(projectMonthly(2000, 20, 30, lastMonth), 3000); // < 2 weeks -> 14-day window = 6 days this month + last 8 of last month. // window = 1200 + 8*100 = 2000 over 14 days -> (2000/14)*30. assert( Math.abs(projectMonthly(1200, 6, 30, lastMonth) - (2000 / 14) * 30) < 1e-6, ); // No data this month yet -> window is the last 14 days of last month. assertEquals(projectMonthly(0, 0, 31, lastMonth), 100 * 31); // No data anywhere -> falls back to mtd. assertEquals(projectMonthly(0, 0, 31, []), 0); }); Deno.test("settled: a day is known once billing has had time to finish it", () => { const L = GITHUB_LAG_DAYS; // 2 // Day 15 of the month. Spend every day through the 15th: the last two days have // figures, but billing is still adding to them, so they are not known yet. assertEquals( settled(Array.from({ length: 31 }, (_, i) => (i < 15 ? 10 : 0)), 15, L) .length, 13, ); // Nothing since the 5th. The days that carried spend stop at 5, but by the 15th // billing has settled everything up to the 13th, so those quiet days are known. assertEquals( settled(Array.from({ length: 31 }, (_, i) => (i < 5 ? 10 : 0)), 15, L) .length, 13, ); // The window keeps growing as the month goes on rather than stalling at 5. assertEquals( settled(Array.from({ length: 31 }, (_, i) => (i < 5 ? 10 : 0)), 31, L) .length, 29, ); // Nothing at all this month: still only what billing has settled. assertEquals(settled(new Array(31).fill(0), 15, L).length, 13); // Day 1: nothing is settled yet and nothing has a figure. assertEquals(settled(new Array(31).fill(0), 1, L).length, 0); // A completed prior month, seen from part-way through the next one, is all known. assertEquals(settled(new Array(30).fill(0), 30 + 15, L).length, 30); // ...except at the very start of the next month, where its tail has not settled. assertEquals(settled(new Array(30).fill(0), 30 + 1, L).length, 29); assertEquals(settled([], 15, L).length, 0); }); Deno.test("projectMonthly: the window keeps up with the calendar once spend stops", () => { // The real shape that started this: $1.01 across a fortnight, a penny on the 1st // and a dollar on the 14th, nothing else, no prior month. const month = new Array(31).fill(0); month[0] = 0.01; month[13] = 1.0; const rate = (day: number) => { const covered = settled(month, day, 1).length; return projectMonthly(1.01, covered, 31, []); }; // On the 15th the fortnight is rated over 14 days, not over the 2 that spent. assert(Math.abs(rate(15) - (1.01 / 14) * 31) < 1e-6); // Counting only the days that spent would rate it over 2 and project ~7x higher. assert(projectMonthly(1.01, 2, 31, []) / rate(15) > 6); // By the end of the month the projection has converged on what was actually spent, // instead of stalling at the 14th and claiming double. assert( rate(31) < 1.15, `end-of-month projection should approach $1.01, got ${rate(31)}`, ); assert(rate(31) < rate(15), "the rate falls as quiet days accumulate"); // The same money landing early is rated over the same elapsed fortnight, not over // the two days at the start of it. const early = new Array(31).fill(0); early[0] = 0.01; early[1] = 1.0; assertEquals(settled(early, 15, 1).length, 14); }); Deno.test("projectMonthly: a borrowed tail is settled before it is borrowed", () => { // Early in a month the borrowed days are last month's tail, the part most likely // not to have settled. Taking those zeros as real quiet days rates the window near // zero and projects far under the truth. const steady = Array.from({ length: 30 }, () => 10); // $10/day last month const truth = 310; // day 3 of a 31-day month, $30 spent at the same $10/day assertEquals(projectMonthly(30, 3, 31, steady), truth); // On the 1st, with a 2-day lag, last month's final day is not known yet. const unsettledTail = [...steady.slice(0, 29), 0]; assertEquals(settled(unsettledTail, 30 + 1, GITHUB_LAG_DAYS).length, 29); assertEquals( projectMonthly(30, 3, 31, settled(unsettledTail, 30 + 1, GITHUB_LAG_DAYS)), truth, ); // Borrowed raw, that one zero drags the rate down. assert( projectMonthly(30, 3, 31, unsettledTail) < truth, "an unsettled tail under-projects", ); // A prior month with no data at all borrows nothing rather than a month of zeros. assertEquals( settled(new Array(30).fill(0), 30 + 1, GITHUB_LAG_DAYS).length, 29, ); // But by mid-month a genuinely quiet prior month is known, and counts as quiet. assertEquals( settled(new Array(30).fill(0), 30 + 15, GITHUB_LAG_DAYS).length, 30, ); }); Deno.test("benchmark: trend classification — flat/down good, up warn, steep up bad", () => { const t = (v: number[]) => v.map((_, i) => i * 86_400_000); // one sample per day const st = (v: number[]) => trendStatus(trendPct(t(v), v)); assertEquals(st([100, 101, 99, 100, 100, 101, 99, 100]), "good"); // flat assertEquals(st([130, 125, 120, 115, 110, 105, 100]), "good"); // falling assertEquals(st([100, 102, 104, 106, 108, 110, 112]), "warn"); // ~12% rise assertEquals(st([100, 120, 140, 160, 180, 200, 240]), "bad"); // steep rise }); Deno.test("benchmark: the trend ignores a lone spike", () => { const flat = [100, 100, 100, 100, 100, 100, 100, 100]; const spiked = [...flat]; spiked[3] = 400; // a 4x outlier — a median level does not move to meet it const times = flat.map((_, i) => i * 86_400_000); assertEquals(trendStatus(trendPct(times, spiked)), "good"); }); Deno.test("registry: unique labels and positive intervals", () => { const labels = TILES.map((t) => t.label); assertEquals(new Set(labels).size, labels.length, "tile labels must be unique"); for (const t of TILES) { assert(t.intervalMs > 0, `${t.label} needs a positive intervalMs`); } }); Deno.test("every tile's drill-down link reaches a route the dashboard serves", () => { // An unrecognized path falls through to the dashboard itself, so a tile // linking at a page nobody serves lands the viewer back where they started. // The server serves the repository pages beside the tiles' own routes. const served = new Set([ ...TILES.flatMap((tile) => tile.routes ?? []).map((route) => route.path), REPOS_PATH, ]); for (const { label, view } of TILE_LAYOUT_FIXTURES) { if (view.href === undefined || /^https?:/.test(view.href)) continue; const path = new URL(view.href, "http://dashboard").pathname; assert( served.has(path), `${label} links to ${path}, which no registered tile serves`, ); } }); Deno.test("layout fixtures cover every registered tile in registry order", () => { assertEquals( TILE_LAYOUT_FIXTURES.map(({ label }) => label), TILES.map(({ label }) => label), ); assertEquals( TILE_LAYOUT_FIXTURES.filter(({ wide }) => wide).map(({ label }) => label), TILES.filter(({ wide }) => wide).map(({ label }) => label), ); });