/** * Reports the median wall-clock time of recent successful pull request CI * runs, with a trend sparkline. One factory builds the labs, loom, and weaver * instances against their own repository and workflow. The labs and loom * instances drill down to that repository's main-run history on /bench. The * labs instance owns the Gantt image route, whose view exposes the * scripts/ci-gantt.ts controls for both repositories. */ import { fromFileUrl } from "@std/path"; import { maxOf, minOf } from "@commonfabric/utils/math"; import { type GreenMark, type Route, type Run, runSource, type RunSource, type Status, type Tile, type TileView, } from "../types.ts"; import { CompletedAttempts } from "../completed-attempts.ts"; import { greenBranchOf } from "../green-branch.ts"; import { GREEN_STAR_RULES, greenStar } from "../green-star.ts"; import { ciDurationSub, escapeHtml, friendlyError, median, sparkline, } from "../lib.ts"; import { CI_RUNS_MAX_AGE_DAYS, CI_WORKFLOW, DUR_GOOD, DUR_MAX_AGE_HOURS, DUR_MIN_RUNS, DUR_WARN, LOOM_CI_WORKFLOW, LOOM_REPO, REPO, WEAVER_CI_WORKFLOW, WEAVER_REPO, } from "../config.ts"; import { ciCommitGanttProgressResponse, ciFetchProgressPanel, type CiGanttOptions, ciGanttOptions, ciGanttProgressResponse, ciHistoryDays, type CiHistorySource, ciHistorySource, collectCiGanttInput, collectCommitCiGanttInput, GANTT_MAX_RUNS, } from "../ci-job-history.ts"; import { PERFORMANCE_VIEW_STYLES, performanceViewNav, } from "../performance-views.ts"; import { CHART_HIGHLIGHT, CHART_LINE, DASHBOARD_THEME_CLIENT, DASHBOARD_THEME_HEAD, dashboardThemeToggle, } from "../theme.ts"; const CIGANTT = fromFileUrl( new URL("../../../scripts/ci-gantt.ts", import.meta.url), ); const GANTT_REFRESH_MS = 30 * 60_000; // How many job listings a duration tile requests at once. Its first // collection reads one for each counted run in its snapshot. const JOB_LISTING_CONCURRENCY = 8; // Writes the renderer's input to `destination` and reports how many runs it // holds. The chart is written a run at a time rather than returned, so a // hundred and fifty runs of job and step timings never sit in this process. export type CiGanttDataProvider = ( source: CiHistorySource, options: CiGanttOptions, destination: string, ) => Promise; export async function renderGantt( p: URLSearchParams, dataProvider: CiGanttDataProvider = collectCiGanttInput, signal?: AbortSignal, ): Promise { const source = ciHistorySource(p.get("repo")); const options = ciGanttOptions(p); const { limit, mainOnly, allConclusions } = options; let out: string | undefined; let input: string | undefined; try { input = await Deno.makeTempFile({ prefix: "ci-gantt-input-", suffix: ".json", }); const runCount = await dataProvider(source, options, input); if (signal?.aborted) { return new Response(null, { status: 204, headers: { "cache-control": "no-store" }, }); } out = await Deno.makeTempFile({ prefix: "ci-gantt-", suffix: ".svg" }); const args = [ "run", `--allow-read=${input}`, `--allow-write=${out}`, CIGANTT, "--input", input, "--repo", source.repo, "--workflow", source.workflow, "--limit", String(limit), "--theme", p.get("theme") === "light" ? "default" : "dark", "--out", out, ]; if (mainOnly) args.push("--main-only"); if (allConclusions) args.push("--all-conclusions"); for (const { runId } of options.selectedRuns ?? []) { args.push("--run-id", String(runId)); } const defaultMinimum = options.selectedRuns?.length ? 1 : mainOnly ? 2 : Math.max(5, Math.round(0.1 * runCount)); args.push( "--min-runs", String(Math.min(defaultMinimum, Math.max(1, runCount))), ); const { success, stderr } = await new Deno.Command(Deno.execPath(), { args, stdout: "piped", stderr: "piped", signal, }).output(); if (signal?.aborted) { return new Response(null, { status: 204, headers: { "cache-control": "no-store" }, }); } if (!success) { console.error("ci-gantt failed:", new TextDecoder().decode(stderr)); return new Response("ci-gantt failed (see server log)", { status: 500 }); } return new Response(await Deno.readFile(out), { headers: { "content-type": "image/svg+xml", "cache-control": "no-store", }, }); } catch (e) { if (signal?.aborted) { return new Response(null, { status: 204, headers: { "cache-control": "no-store" }, }); } const message = e instanceof Error ? e.message : String(e); console.error("ci-gantt render error:", message); const safeMessage = friendlyError(message); if (safeMessage === "rate limit hit") { return new Response("rate limit hit", { status: 429, headers: { "cache-control": "no-store" }, }); } return new Response(safeMessage, { status: 500, headers: { "cache-control": "no-store" }, }); } finally { if (out) await Deno.remove(out).catch(() => {}); if (input) await Deno.remove(input).catch(() => {}); } } export function renderGanttRoute( request: Request, url: URL, dataProvider: CiGanttDataProvider = collectCiGanttInput, ): Promise { return renderGantt(url.searchParams, dataProvider, request.signal); } export function ciGanttPage(url: URL): string { const source = ciHistorySource(url.searchParams.get("repo")); const days = ciHistoryDays(url.searchParams.get("days")); const sort = url.searchParams.get("sort") ?? "job"; const stat = url.searchParams.get("stat") ?? "p99"; const viewNav = performanceViewNav("gantt", { repo: source.key, days, sort, stat, }); return `CI run Gantt ${DASHBOARD_THEME_HEAD}
← dashboardPerformance history${ escapeHtml(source.repo) } · ${ escapeHtml(source.workflow) } · scripts/ci-gantt.ts
${viewNav}
${ciFetchProgressPanel(undefined, { ariaLabel: "CI Gantt fetch progress" })}
CI Gantt chart

Run, job, and step timings share the persistent server cache used by CI history. Regeneration reads cached past runs and fetches only missing runs or newer attempts.

${dashboardThemeToggle()} ${DASHBOARD_THEME_CLIENT} `; } function commitGanttParameters(url: URL): URLSearchParams | null { const sha = url.searchParams.get("sha") ?? ""; if (!/^[0-9a-f]{40}$/i.test(sha)) return null; const selected = new Map(); for (const run of ciGanttOptions(url.searchParams).selectedRuns ?? []) { const current = selected.get(run.runId); if (!current || current.runAttempt < run.runAttempt) { selected.set(run.runId, run); } } if (!selected.size || selected.size > GANTT_MAX_RUNS) return null; const parameters = new URLSearchParams({ repo: ciHistorySource(url.searchParams.get("repo")).key, sha: sha.toLowerCase(), limit: String(selected.size), mainOnly: "1", }); for (const { runId, runAttempt } of selected.values()) { parameters.append("run", `${runId}:${runAttempt}`); } return parameters; } /** * The page charting the runs `url` selects for one commit, which names the * commit and wears the green branch's star when `markOf` gives the commit a * mark. `markOf` defaults to what the repository's green branch says. */ export function ciCommitGanttPage( url: URL, markOf: (repo: string, sha: string) => GreenMark | undefined = (repo, sha) => greenBranchOf(repo)?.mark(sha), ): string { const source = ciHistorySource(url.searchParams.get("repo")); const parameters = commitGanttParameters(url); const sha = parameters?.get("sha") ?? url.searchParams.get("sha") ?? ""; const shortSha = sha.slice(0, 7) || "unknown commit"; const runCount = parameters?.getAll("run").length ?? 0; const commitUrl = `https://github.com/${source.repo}/commit/${sha}`; const imageUrl = parameters ? `/ci-gantt.svg?${parameters}` : ""; const progressUrl = parameters ? `/ci-gantt-progress?${parameters}` : ""; const chart = parameters ? `
CI Gantt for ${
      escapeHtml(shortSha)
    }
` : `
No successful main CI runs were supplied for this commit.
`; const script = parameters ? `` : ""; return `CI Gantt · ${ escapeHtml(shortSha) } ${DASHBOARD_THEME_HEAD}
← dashboardCI Gantt${ escapeHtml(source.repo) } · ${ greenStar(markOf(source.repo, sha.toLowerCase())) }${ escapeHtml(shortSha) } ↗
${ ciFetchProgressPanel(undefined, { ariaLabel: "Commit CI Gantt fetch progress", }) } ${chart} ${dashboardThemeToggle()} ${DASHBOARD_THEME_CLIENT} ${script} `; } export function commitGanttUrl(url: URL): URL | null { const parameters = commitGanttParameters(url); if (!parameters) return null; const theme = url.searchParams.get("theme"); if (theme === "dark" || theme === "light") parameters.set("theme", theme); const normalized = new URL(url); normalized.search = parameters.toString(); return normalized; } function redirectGanttImage(url: URL, pathname: string): Response { return new Response(null, { status: 308, headers: { location: `${pathname}${url.search}` }, }); } const ganttRoutes: Route[] = [ { path: "/ci-gantt", async handler(_request, url) { // The page does not change once served, so the green branch is read // before it is drawn rather than after. await greenBranchOf(ciHistorySource(url.searchParams.get("repo")).repo) ?.refresh(Date.now() - CI_RUNS_MAX_AGE_DAYS * 86_400_000); return new Response(ciCommitGanttPage(url), { headers: { "content-type": "text/html; charset=utf-8" }, }); }, }, { path: "/ci-gantt.svg", handler(request, url) { const normalized = commitGanttUrl(url); return normalized ? renderGanttRoute(request, normalized, collectCommitCiGanttInput) : Promise.resolve( new Response("invalid commit Gantt selection", { status: 400, headers: { "cache-control": "no-store" }, }), ); }, }, { path: "/ci-gantt.png", handler(_request, url) { return redirectGanttImage(url, "/ci-gantt.svg"); }, }, { path: "/ci-gantt-progress", handler(request, url) { const normalized = commitGanttUrl(url); return normalized ? ciCommitGanttProgressResponse(request, normalized) : new Response("invalid commit Gantt selection", { status: 400 }); }, }, { path: "/bench/gantt.svg", handler: renderGanttRoute, }, { path: "/bench/gantt.png", handler(_request, url) { return redirectGanttImage(url, "/bench/gantt.svg"); }, }, { path: "/bench/gantt-progress", handler: ciGanttProgressResponse, }, ]; interface PassedRun { run: Run; createdAt: number; durationMins: number; } /** * The runs among `runs` that succeeded on their first attempt and did work, * each with how long it took from creation to finish, in the order of `runs`. * A run that passed only on a rerun is left out, because its span includes * the wait before someone asked for the rerun. A run that ran no job did * nothing, and one that ran one job and skipped the rest did nothing beyond * reporting its status, as weaver's run for a draft pull request does, so * both are left out too. Rejects when a run's job listing cannot be read. */ async function passedRuns( runs: readonly Run[], attempts: CompletedAttempts, ): Promise { const spans = runs.flatMap((run) => { if ( run.status !== "completed" || run.conclusion !== "success" || run.run_attempt !== 1 ) { return []; } const createdAt = Date.parse(run.created_at); const finishedAt = Date.parse(run.updated_at); if ( !Number.isFinite(createdAt) || !Number.isFinite(finishedAt) || finishedAt <= createdAt ) { return []; } return [{ run, createdAt, durationMins: (finishedAt - createdAt) / 60_000 }]; }); attempts.observe(spans.map(({ run }) => run)); const worked = new Set(); for (let at = 0; at < spans.length; at += JOB_LISTING_CONCURRENCY) { await Promise.all( spans.slice(at, at + JOB_LISTING_CONCURRENCY).map(async ({ run }) => { const { ran, skipped } = await attempts.jobCounts(run); if (ran > 1 || (ran === 1 && skipped === 0)) worked.add(run.id); }), ); } return spans.filter(({ run }) => worked.has(run.id)); } function makeCiDuration( opts: { label: string; source: RunSource; link?: { href: string; hint: string }; routes?: Route[]; }, ): Tile { const attempts = new CompletedAttempts(opts.source.repo); return { label: opts.label, repo: opts.source.repo, intervalMs: 5 * 60_000, runSources: [opts.source], routes: opts.routes, async collect(ctx): Promise { let runs: Run[]; try { runs = await ctx.runsFor(opts.source); } catch (error) { const message = error instanceof Error ? error.message : String(error); return { status: "unknown", value: "—", sub: friendlyError(message), ...opts.link, }; } const passed = await passedRuns(runs, attempts); // Median window = the successful runs in the last DUR_MAX_AGE_HOURS, or the // most recent DUR_MIN_RUNS — whichever has more runs. const cutoff = Date.now() - DUR_MAX_AGE_HOURS * 3_600_000; const inTimeCount = passed.filter((run) => run.createdAt >= cutoff).length; const usingTime = inTimeCount >= DUR_MIN_RUNS; // time window wins when it has enough runs // A count-based prefix (not the filter set itself) so the median runs are // always the newest slice of passed — which is what the sparkline // highlights. passed is created-at ordered, so a re-run can otherwise // make the filter set non-contiguous with the front. const window = passed.slice(0, usingTime ? inTimeCount : DUR_MIN_RUNS); const durs = window.map((run) => run.durationMins).sort((a, b) => a - b); const medianMins = Math.round(median(durs)); // The sparkline spans every successful run (oldest -> newest). window is // always the newest slice of passed, so the runs feeding the median are the // trailing window.length points — drawn brighter over the dimmer long-run // trend. const series = [...passed].reverse().map((run) => run.durationMins); // How long the sparkline spans (oldest to newest run), for the corner label. const times = passed.map((run) => run.createdAt); const spanMs = times.length >= 2 ? maxOf(times) - minOf(times) : 0; const s: Status = window.length === 0 ? "unknown" : medianMins <= DUR_GOOD ? "good" : medianMins <= DUR_WARN ? "warn" : "bad"; return { status: s, value: window.length === 0 ? "—" : `${medianMins}m`, sub: ciDurationSub( window.length, usingTime ? DUR_MAX_AGE_HOURS : undefined, ), extra: sparkline(series, CHART_LINE, { count: window.length, color: CHART_HIGHLIGHT, }, true), duration: spanMs, ...opts.link, }; }, }; } export const labsCiDuration = makeCiDuration({ label: "labs ci duration", source: runSource(REPO, CI_WORKFLOW, "pull requests"), link: { href: "/bench?view=ci&repo=labs", hint: "main run history ↗" }, routes: ganttRoutes, }); export const loomCiDuration = makeCiDuration({ label: "loom ci duration", source: runSource(LOOM_REPO, LOOM_CI_WORKFLOW, "pull requests"), link: { href: "/bench?view=ci&repo=loom", hint: "main run history ↗" }, }); export const weaverCiDuration = makeCiDuration({ label: "weaver ci duration", source: runSource(WEAVER_REPO, WEAVER_CI_WORKFLOW, "pull requests"), });