/// // The fetch builtins coordinate across replicas through durable state: a claim // in `internal` for fetch.ts, a `fetching` cache entry for fetch-program.ts. A // staleness bound decides when a claim left by another replica is treated as // abandoned, because nothing reports whether that replica is still there. // // Two of these cases pin what keeps an early takeover survivable. Both are // about the request that is *not* the stale one: it must still reach its // result. // // - a resolution running in this replica is never judged by the clock, so a // slow one is neither restarted underneath itself nor left with nowhere to // write when it finishes; // - a failing request never erases a result another request already recorded // for the same inputs. // // The rest cover the claim's other transitions: handing it back when the // pattern stops, the staleness branch that decides a takeover, a resolution // that fails, and an empty URL. // // docs/features/fetch-request-deadlines.md carries the reasoning. import { afterEach, beforeEach, describe, it } from "@std/testing/bdd"; import { expect } from "@std/expect"; import { Identity } from "@commonfabric/identity"; import { StorageManager } from "@commonfabric/runner/storage/cache.deno"; import { Runtime } from "../src/runtime.ts"; import { createBuilder } from "../src/builder/factory.ts"; import { createTrustedBuilder } from "./support/trusted-builder.ts"; import { type IExtendedStorageTransaction } from "../src/storage/interface.ts"; import { setPatternEnvironment } from "../src/env.ts"; import { resolveLink } from "../src/link-resolution.ts"; import { computeInputHashFromValue, internalSchema, MUTEX_STALE_AFTER, tryClaimMutex, } from "../src/builtins/fetch-utils.ts"; import type { Schema } from "../src/builder/types.ts"; const signer = await Identity.fromPassphrase("test fetch claim takeover"); const space = signer.did(); function moduleResponse(): Response { return new Response("export const value = 1;\n", { status: 200, headers: { "Content-Type": "text/plain" }, }); } interface Deferred { promise: Promise; resolve: (value: T) => void; reject: (reason?: unknown) => void; } function deferred(): Deferred { let resolve!: (value: T) => void; let reject!: (reason?: unknown) => void; const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); return { promise, resolve, reject }; } describe("fetch builtins: taking over a claim", () => { let storageManager: ReturnType; let runtime: Runtime; let tx: IExtendedStorageTransaction; let pattern: ReturnType["commonfabric"]["pattern"]; let byRef: ReturnType["commonfabric"]["byRef"]; let originalFetch: typeof globalThis.fetch; beforeEach(() => { storageManager = StorageManager.emulate({ as: signer }); runtime = new Runtime({ apiUrl: new URL(import.meta.url), storageManager, }); tx = runtime.edit(); const { commonfabric } = createTrustedBuilder(runtime); pattern = commonfabric.pattern; byRef = commonfabric.byRef; setPatternEnvironment({ apiUrl: new URL("http://mock-test-server.local"), }); originalFetch = globalThis.fetch; }); afterEach(async () => { globalThis.fetch = originalFetch; await tx.commit().settled; await runtime?.dispose(); await storageManager?.close(); }); it("releases the entry it claimed when the pattern is stopped", async () => { const slowUrl = "http://mock-test-server.local/stopped-program.ts"; const heldRequests: Deferred[] = []; const firstRequest = deferred(); globalThis.fetch = () => { const held = deferred(); heldRequests.push(held); firstRequest.resolve(); return held.promise; }; const fetchProgram = byRef("fetchProgram"); const testPattern = pattern<{ url: string }>( ({ url }) => fetchProgram({ url }), ); const resultCell = runtime.getCell( space, "program-stop-result", undefined, tx, ); const result = runtime.run(tx, testPattern, { url: slowUrl }, resultCell); tx.commit(); await result.pull(); await firstRequest.promise; await clock.settle(); expect(heldRequests.length).toBe(1); // Stopping the pattern hands the claim back, so the entry is available // again rather than left standing by a replica that has gone. runtime.runner.stop(resultCell); await clock.settle(); // Running again is how that shows: a released entry is claimable, so a // second resolution starts. A claim still standing would be left alone. const restartTx = runtime.edit(); const restarted = runtime.run( restartTx, testPattern, { url: slowUrl }, resultCell, ); restartTx.commit(); await restarted.pull(); await clock.settle(); expect(heldRequests.length).toBe(2); for (const held of heldRequests) held.resolve(moduleResponse()); await runtime.settled(); }); it("surfaces a program that fails to resolve as an error", async () => { const missingUrl = "http://mock-test-server.local/missing-program.ts"; globalThis.fetch = () => Promise.resolve( new Response("not found", { status: 404, statusText: "Not Found" }), ); const fetchProgram = byRef("fetchProgram"); const testPattern = pattern<{ url: string }>( ({ url }) => fetchProgram({ url }), ); const resultCell = runtime.getCell( space, "program-error-result", undefined, tx, ); const result = runtime.run( tx, testPattern, { url: missingUrl }, resultCell, ); tx.commit(); await result.pull(); await runtime.settled(); await result.pull(); expect(result.key("error").get()).toContain("404"); expect(result.key("result").get()).toBeUndefined(); expect(result.key("pending").get()).toBe(false); }); it("clears its outputs when the program URL is empty", async () => { let requests = 0; globalThis.fetch = () => { requests++; return Promise.resolve(moduleResponse()); }; const fetchProgram = byRef("fetchProgram"); const testPattern = pattern<{ url: string }>( ({ url }) => fetchProgram({ url }), ); const resultCell = runtime.getCell( space, "program-empty-url-result", undefined, tx, ); const result = runtime.run(tx, testPattern, { url: "" }, resultCell); tx.commit(); await result.pull(); await runtime.settled(); expect(requests).toBe(0); expect(result.key("pending").get()).toBe(false); expect(result.key("result").get()).toBeUndefined(); expect(result.key("error").get()).toBeUndefined(); }); it("publishes a slow program resolution that outlives the staleness bound", async () => { const slowUrl = "http://mock-test-server.local/slow-program.ts"; const otherUrl = "http://mock-test-server.local/other-program.ts"; // The slow program's module request is held open, so the resolution // started for it stays in flight until this case releases it. const heldRequests: Deferred[] = []; const firstSlowRequest = deferred(); globalThis.fetch = (input: string | URL | Request) => { const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; if (url.includes("slow-program")) { const held = deferred(); heldRequests.push(held); firstSlowRequest.resolve(); return held.promise; } return Promise.resolve(moduleResponse()); }; const fetchProgram = byRef("fetchProgram"); const testPattern = pattern<{ url: string }>( ({ url }) => fetchProgram({ url }), ); const inputs = runtime.getCell<{ url: string }>( space, "program-takeover-inputs", undefined, tx, ); inputs.set({ url: slowUrl }); const resultCell = runtime.getCell( space, "program-takeover-result", undefined, tx, ); const result = runtime.run(tx, testPattern, inputs, resultCell); tx.commit(); await result.pull(); await firstSlowRequest.promise; await clock.settle(); expect(heldRequests.length).toBe(1); // Point the pattern at a different program and back again. That round trip // is the wake-up: the staleness check is a predicate the action evaluates // when it next runs, not a timer, so it is only ever reached because // something else re-ran the action. The first resolution is in flight // throughout — an input change does not abort it. await runtime.editWithRetry((edit) => { inputs.withTx(edit).key("url").set(otherUrl); }); await result.pull(); await clock.settle(); // Well past any staleness bound this builtin would plausibly use. await clock.tick(120_000); await runtime.editWithRetry((edit) => { inputs.withTx(edit).key("url").set(slowUrl); }); await result.pull(); await clock.settle(); // The resolution was never restarted, and its cache entry is still the one // it will write into. expect(heldRequests.length).toBe(1); heldRequests[0].resolve(moduleResponse()); await runtime.settled(); await result.pull(); const program = result.key("result").get() as | { main: string; files: { name: string }[] } | undefined; expect(program).toBeDefined(); expect(program!.main).toBe("/slow-program.ts"); expect(program!.files.map((file) => file.name)).toEqual([ "/slow-program.ts", ]); expect(result.key("error").get()).toBeUndefined(); expect(result.key("pending").get()).toBe(false); }); it("keeps a result recorded for the same inputs when a request fails", async () => { const url = "http://mock-test-server.local/api/takeover"; const heldRequest = deferred(); const requestIssued = deferred(); globalThis.fetch = () => { requestIssued.resolve(); return heldRequest.promise; }; const fetchJson = byRef("fetchJson"); // This case is about the error path, not about the bound, so it puts the // bound out of reach: the auto-advance clock jumps logical time forward // whenever the loop idles, and a takeover here would clear `result` on its // way in and mask what is being pinned. const testPattern = pattern<{ url: string }>( ({ url }) => fetchJson({ url, options: { mutexTimeoutMs: 600_000 } }), ); const resultCell = runtime.getCell( space, "fetch-takeover-result", undefined, tx, ); const result = runtime.run(tx, testPattern, { url }, resultCell); tx.commit(); await result.pull(); await requestIssued.promise; await clock.settle(); // Stand in for the replica that took the claim over and got there first: a // result for these very inputs is now recorded. The pattern's result cell // holds a link to the builtin's own result cell, and a plain link is not // followed on write, so resolve it and write the cell the builtin writes. const builtinResult = runtime.getCellFromLink( resolveLink( runtime, runtime.readTx(), result.key("result").getAsNormalizedFullLink(), ), ); await runtime.editWithRetry((edit) => { builtinResult.withTx(edit).set({ from: "the other replica" }); }); await clock.settle(); heldRequest.reject(new Error("simulated network failure")); await runtime.settled(); expect(result.key("result").get()).toEqual({ from: "the other replica" }); expect(result.key("error").get()).toBeUndefined(); expect(result.key("pending").get()).toBe(false); }); describe("the staleness branch of the mutex", () => { // The takeover itself, driven through `tryClaimMutex` directly: a claim // another replica made is believed until it goes stale, and then taken // over. This is the one branch where the clock decides anything. const inputs = { url: "http://mock-test-server.local/api/claimed" }; async function claimAgainst( claimAgeMs: number, ): Promise<{ claimed: boolean; requestId: string; pending: boolean }> { const inputsCell = runtime.getCell( space, `mutex-staleness-inputs-${claimAgeMs}`, undefined, tx, ); const pending = runtime.getCell( space, `mutex-staleness-pending-${claimAgeMs}`, undefined, tx, ); const result = runtime.getCell( space, `mutex-staleness-result-${claimAgeMs}`, undefined, tx, ); const error = runtime.getCell( space, `mutex-staleness-error-${claimAgeMs}`, undefined, tx, ); const internal = runtime.getCell>( space, `mutex-staleness-internal-${claimAgeMs}`, undefined, tx, ); // A claim standing in the durable state, made by a replica that is not // this one, `claimAgeMs` ago. inputsCell.set(inputs); pending.set(true); internal.set({ requestId: "another-replica:whatever", lastActivity: Date.now() - claimAgeMs, inputHash: computeInputHashFromValue(inputs), }); await tx.commit().settled; tx = runtime.edit(); const claim = await tryClaimMutex( runtime, inputsCell, pending, result, error, internal, "this-replica:whatever", (cell) => cell.get() ?? ({} as typeof inputs), ); return { claimed: claim.claimed, requestId: internal.get().requestId, pending: pending.get(), }; } it("leaves a claim younger than the bound alone", async () => { const outcome = await claimAgainst(MUTEX_STALE_AFTER / 2); expect(outcome.claimed).toBe(false); expect(outcome.requestId).toBe("another-replica:whatever"); expect(outcome.pending).toBe(true); }); it("takes over a claim older than the bound", async () => { const outcome = await claimAgainst(MUTEX_STALE_AFTER * 2); expect(outcome.claimed).toBe(true); expect(outcome.requestId).toBe("this-replica:whatever"); expect(outcome.pending).toBe(true); }); }); describe("the completed-request guard of the mutex", () => { // The claim-time memo guard (server-execution v2 stage G, // serving-loop.md §4): a COMPLETED request — stored hash matching the // expected inputs AND a result (or error-shaped result) present — is // never re-claimed; the stored value IS the value. The serving // posture's deferred flush can re-admit a key whose first effect // already completed, and without this guard the late claim would // clear the result and issue a second egress for inputs that already // have their answer. const inputs = { url: "http://mock-test-server.local/api/completed" }; async function claimWithStored( stored: "result" | "error" | "none", ): Promise<{ claimed: boolean; result: unknown; pending: boolean }> { const inputsCell = runtime.getCell( space, `mutex-completed-inputs-${stored}`, undefined, tx, ); const pending = runtime.getCell( space, `mutex-completed-pending-${stored}`, undefined, tx, ); const result = runtime.getCell( space, `mutex-completed-result-${stored}`, undefined, tx, ); const error = runtime.getCell( space, `mutex-completed-error-${stored}`, undefined, tx, ); const internal = runtime.getCell>( space, `mutex-completed-internal-${stored}`, undefined, tx, ); // The completed request's durable shape: hash recorded for these // very inputs, pending long false, and — in the guarded arms — // the settled value standing. inputsCell.set(inputs); pending.set(false); if (stored === "result") { result.set({ from: "the completed request" }); } else if (stored === "error") { error.set({ message: "the stored error-shaped result" }); } internal.set({ requestId: "", lastActivity: 0, inputHash: computeInputHashFromValue(inputs), }); await tx.commit().settled; tx = runtime.edit(); const claim = await tryClaimMutex( runtime, inputsCell, pending, result, error, internal, "this-replica:whatever", (cell) => cell.get() ?? ({} as typeof inputs), computeInputHashFromValue(inputs), ); return { claimed: claim.claimed, result: result.get(), pending: pending.get(), }; } it("never claims over a stored result whose hash matches", async () => { const outcome = await claimWithStored("result"); expect(outcome.claimed).toBe(false); // The stored value stands untouched — no clear on the way out. expect(outcome.result).toEqual({ from: "the completed request" }); expect(outcome.pending).toBe(false); }); it("never claims over a stored error-shaped result whose hash matches", async () => { const outcome = await claimWithStored("error"); expect(outcome.claimed).toBe(false); expect(outcome.pending).toBe(false); }); it("still claims when the hash matches but nothing has landed", async () => { const outcome = await claimWithStored("none"); expect(outcome.claimed).toBe(true); expect(outcome.pending).toBe(true); }); }); });