/** * Runs a Deno workload against CI's matching baked toolshed in a fresh store. * Saves exact commands, posture, load samples, raw output, and server statistics. * Usage: `deno run -A tools/server-execution-topics/run-arm.ts default DIR * correctness test -A packages/patterns/integration/EXAMPLE.test.ts`. * `latency` mode refuses a loaded machine and invalidates a run that gets busy. */ import { dirname, join, resolve } from "@std/path"; import { type CapabilityId, type Exec, openCapabilities, } from "../../tasks/ci-capabilities.ts"; import { assertServerExecutionCiPosture, serverExecutionCiLane, } from "../../tasks/server-execution-ci.ts"; const [role, directory, mode, ...args] = Deno.args; if ( (role !== "default" && role !== "opposite") || !directory || (mode !== "correctness" && mode !== "latency" && mode !== "profile") || args.length === 0 ) { throw new Error( "Expected role, new artifact directory, mode, and Deno args.", ); } const root = resolve(import.meta.dirname!, "../.."); const runDir = resolve(directory); await Deno.mkdir(runDir); const store = join(runDir, "store"); await Deno.mkdir(store); const lane = serverExecutionCiLane(role); const capability: CapabilityId = role === "default" ? "toolshed-baked" : "toolshed-baked-opposite"; const commands: { command: string; args: readonly string[]; cwd: string }[] = []; /** Executes and records a setup command, placing this run's server in its store. */ const exec: Exec = async (command, commandArgs, options = {}) => { const cwd = command.endsWith(`toolshed-baked-${role}`) ? store : options.cwd ?? root; const index = commands.length; commands.push({ command, args: commandArgs, cwd }); const result = await new Deno.Command(command, { args: [...commandArgs], cwd, env: options.env, stdout: "piped", stderr: "piped", }).output(); await Deno.writeFile(join(runDir, `setup-${index}.stdout`), result.stdout); await Deno.writeFile(join(runDir, `setup-${index}.stderr`), result.stderr); if (!result.success) { throw new Error(`Setup command ${index} failed with exit ${result.code}.`); } return new TextDecoder().decode(result.stdout); }; const samples: { at: string; load: number[] }[] = []; const sampleLoad = () => samples.push({ at: new Date().toISOString(), load: Deno.loadavg(), }); sampleLoad(); const head = (await exec("git", ["rev-parse", "HEAD"])).trim(); const patch = await exec("git", ["diff", "HEAD", "--binary"]); await Deno.writeTextFile(join(runDir, "worktree.patch"), patch); const sourcePaths = [ "tools/server-execution-topics/run-arm.ts", "tools/server-execution-topics/seed-check.ts", "packages/patterns/integration/topic-board-fixture.ts", ]; for (const path of sourcePaths) { const destination = join(runDir, "sources", path); await Deno.mkdir(dirname(destination), { recursive: true }); await Deno.copyFile(join(root, path), destination); } const flags = Object.fromEntries( Object.entries(Deno.env.toObject()).filter(([key]) => key.startsWith("EXPERIMENTAL_") || key.startsWith("CF_TOPIC") || ["CF_TIMING_MEASURES", "CF_MEMORY_FRAME_LOG", "CF_PROF_CPU"].includes(key) ), ); flags.EXPERIMENTAL_SERVER_EXECUTION = String(lane.enabled); const cpuModel = Deno.build.os === "darwin" ? (await exec("/usr/sbin/sysctl", ["-n", "machdep.cpu.brand_string"])).trim() : Deno.build.os === "linux" ? (await Deno.readTextFile("/proc/cpuinfo")) .match(/^model name\s*:\s*(.+)$/m)?.[1] : undefined; const manifest: Record = { version: 1, mode, role, expectedPosture: lane, workloadHead: head, sourcePaths, root, store, cache: "fresh store and server; existing Deno dependency cache", machine: { cpu: cpuModel, arch: Deno.build.arch, os: Deno.build.os, logicalCpus: navigator.hardwareConcurrency, totalmem: Deno.systemMemoryInfo().total, }, runtime: Deno.version, flags, commands, samples, status: "preparing", }; const save = () => Deno.writeTextFile( join(runDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n", ); await save(); if ( mode === "latency" && ["CF_TIMING_MEASURES", "CF_MEMORY_FRAME_LOG", "CF_PROF_CPU"].some((key) => flags[key] !== undefined && !["", "0", "false"].includes(flags[key]) ) ) { manifest.status = "blocked-instrumentation"; await save(); throw new Error( "Latency runs must be separate from timing/frame/CPU profiling.", ); } if (mode === "latency" && samples[0].load[0] > 5) { manifest.status = "blocked-load"; await save(); throw new Error("Latency run requires one-minute load at or below 5."); } let opened: Awaited> | undefined; let sampling: ReturnType | undefined; try { opened = await openCapabilities([capability], { root, dryRun: false, workDir: runDir, exec, }); const endpoint = opened.envFor([capability]); const meta = await (await fetch(`${endpoint.API_URL}/api/meta`)).json(); const before = await (await fetch(`${endpoint.API_URL}/api/health/stats`)) .json(); assertServerExecutionCiPosture(role, meta, before); if (meta.gitSha !== head) { throw new Error( `Server build ${meta.gitSha} does not match workload head ${head}. ` + "Rebuild both binaries with COMMIT_SHA set to the workload head.", ); } manifest.meta = meta; manifest.serverBuildHead = meta.gitSha; await Deno.writeTextFile( join(runDir, "stats-before.json"), JSON.stringify(before), ); const binary = join(root, ".ci-cache/binaries", `toolshed-baked-${role}`); manifest.binarySha256 = (await exec("shasum", ["-a", "256", binary])) .split(" ")[0]; const env = { ...endpoint, ...flags, FRONTEND_URL: endpoint.API_URL, SPACE_NAME: `topics-campaign-${crypto.randomUUID()}`, HEADLESS: "1", CF_LOG_LEVEL: "silent", CF_CAMPAIGN_RUN_DIR: runDir, }; manifest.workloadEnvironment = env; commands.push({ command: Deno.execPath(), args, cwd: root }); sampleLoad(); // Periodic samples observe contention during the workload; they do not // delay it or determine when its completion conditions have been reached. sampling = setInterval(sampleLoad, 1000); manifest.status = "running"; await save(); const result = await new Deno.Command(Deno.execPath(), { args, cwd: root, env, stdout: "piped", stderr: "piped", }).output(); clearInterval(sampling); sampling = undefined; sampleLoad(); await Deno.writeFile(join(runDir, "workload.stdout"), result.stdout); await Deno.writeFile(join(runDir, "workload.stderr"), result.stderr); const after = await (await fetch(`${endpoint.API_URL}/api/health/stats`)) .json(); await Deno.writeTextFile( join(runDir, "stats-after.json"), JSON.stringify(after), ); manifest.exitCode = result.code; manifest.latencyEligible = mode === "latency" && samples.every(({ load }) => load[0] <= 5); manifest.status = !result.success ? "failed-workload" : mode === "latency" && manifest.latencyEligible !== true ? "invalid-load" : "passed"; await save(); if (!result.success) { throw new Error(`Workload failed with exit ${result.code}.`); } if (manifest.status === "invalid-load") { throw new Error( "Machine load exceeded the latency threshold during this run.", ); } } catch (error) { manifest.error = error instanceof Error ? error.message : String(error); if (manifest.status === "preparing" || manifest.status === "running") { manifest.status = "failed"; } await save(); throw error; } finally { if (sampling !== undefined) clearInterval(sampling); await opened?.close(); }