/// /** * What the fetch builtins report when a completion writeback cannot be * written, which is two different things. * * The writeback is the last step of a fetch: the response has arrived, and * `tryWriteResult()` puts it in the result cell. Its two "did not write" * outcomes are _distinct_ for the caller. Inputs that moved mean the request * was superseded, so the new inputs' own request owns the cells and there is * nothing left to do. A commit the storage layer refused means the opposite — * the claim is still durably pending and this response was the only completion * it will ever get, so `fetch.ts` must not retire the effect as fulfilled. It * converts that refusal into an error-shaped result instead, and propagates * when even that cannot commit. * * Those failure reports run nowhere except where a writeback's commit is * refused, which a suite reaches only by tearing a runtime down around one, so * a suite that does not construct the refusal covers them or not according to * how the run was scheduled. Each case here constructs the refusal it wants, * which is what makes the reports run on every suite run and under every shard * layout. `docs/development/COVERAGE.md` ("Failure reports reached only when * the operation fails") carries the reasoning. */ import { afterEach, beforeEach, describe, it } from "@std/testing/bdd"; import { expect } from "@std/expect"; import { Identity } from "@commonfabric/identity"; import { StorageManager } from "@commonfabric/runner/storage/cache.deno"; import { Runtime } from "../src/runtime.ts"; import { createBuilder } from "../src/builder/factory.ts"; import { createTrustedBuilder } from "./support/trusted-builder.ts"; import { type IExtendedStorageTransaction } from "../src/storage/interface.ts"; import { getTransactionWriteAttempts } from "../src/storage/transaction-inspection.ts"; import { getPatternEnvironment, setPatternEnvironment } from "../src/env.ts"; import { resolveLink } from "../src/link-resolution.ts"; import { computeInputHashFromValue, internalSchema, tryWriteResult, } from "../src/builtins/fetch-utils.ts"; import type { Schema } from "../src/builder/types.ts"; import { createTransactionCommitReceipt } from "../src/storage/commit-receipt.ts"; const signer = await Identity.fromPassphrase("test fetch writeback"); const space = signer.did(); /** The inputs every case in this file writes back against. */ const INPUTS = { url: "http://mock-test-server.local/api/writeback" }; const INPUT_HASH = computeInputHashFromValue(INPUTS); /** * The refusal every case injects. `AuthorizationError` without the server's * `retriable` marker is outside the retryable vocabulary, so `editWithRetry` * reports it after a single commit rather than retrying against it. */ const REFUSAL = { name: "AuthorizationError", message: "the space refused the write", }; interface Deferred { promise: Promise; resolve: (value: T) => void; reject: (reason?: unknown) => void; } function deferred(): Deferred { let resolve!: (value: T) => void; let reject!: (reason?: unknown) => void; const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); return { promise, resolve, reject }; } describe("fetch builtins: a completion writeback the storage layer refuses", () => { let storageManager: ReturnType; let runtime: Runtime; let tx: IExtendedStorageTransaction; let pattern: ReturnType["commonfabric"]["pattern"]; let byRef: ReturnType["commonfabric"]["byRef"]; let originalFetch: typeof globalThis.fetch; let originalPatternEnvironment: ReturnType; let pristineEdit: Runtime["edit"]; beforeEach(() => { storageManager = StorageManager.emulate({ as: signer }); runtime = new Runtime({ apiUrl: new URL(import.meta.url), storageManager, }); tx = runtime.edit(); const { commonfabric } = createTrustedBuilder(runtime); pattern = commonfabric.pattern; byRef = commonfabric.byRef; originalPatternEnvironment = getPatternEnvironment(); setPatternEnvironment({ apiUrl: new URL("http://mock-test-server.local"), }); originalFetch = globalThis.fetch; pristineEdit = runtime.edit.bind(runtime); }); afterEach(async () => { // Hand the runtime its real commits back before teardown: a disposal whose // own writes are refused reports failures that belong to no case. (runtime as any).edit = pristineEdit; globalThis.fetch = originalFetch; setPatternEnvironment(originalPatternEnvironment); tx.abort("case over"); // `closeStorage: false` drains the runtime's outstanding work first, // including the background loads a cell read starts; closing under them // reports each one as a sync failure. await runtime?.dispose({ closeStorage: false }); await storageManager?.close(); }); /** * Refuse the commit of every transaction this runtime opens from now on * whose write set `wanted` accepts, up to `limit` of them, leaving the * transaction's reads and writes real and only its outcome injected. * * Reports how many transactions `wanted` accepted, counted past `limit` * rather than capped at it: a case states the number it meant, so a run that * opened a further matching transaction fails on the count instead of * quietly leaving that one to commit. */ function refuseCommits( wanted: (writtenDocuments: readonly string[]) => boolean, limit: number, ): () => number { let matched = 0; (runtime as any).edit = (...args: Parameters) => { const opened: IExtendedStorageTransaction = pristineEdit(...args); const commit = opened.commit.bind(opened); (opened as any).commit = (...commitArgs: unknown[]) => { const written = (getTransactionWriteAttempts(opened) ?? []).map( (attempt) => attempt.id as string, ); if (wanted(written)) { matched++; if (matched <= limit) { opened.abort(REFUSAL); return createTransactionCommitReceipt( Promise.resolve({ error: REFUSAL }), ); } } return (commit as (...args: unknown[]) => unknown)(...commitArgs); }; return opened; }; return () => matched; } describe("tryWriteResult()", () => { /** The three cells a writeback is handed, seeded and committed. */ async function newCells(cause: string) { const inputsCell = runtime.getCell( space, `${cause}-inputs`, undefined, tx, ); const internal = runtime.getCell>( space, `${cause}-internal`, internalSchema, tx, ); const result = runtime.getCell( space, `${cause}-result`, undefined, tx, ); inputsCell.set(INPUTS); // A claim standing for these inputs, with no result recorded yet: the // state a response arrives into. internal.set({ requestId: "this-replica:claim", lastActivity: 1, inputHash: "", }); await tx.commit().settled; tx = runtime.edit(); return { inputsCell, internal, result }; } it("writes the result and records the hash when the inputs have not moved", async () => { const { inputsCell, internal, result } = await newCells("landed"); const written = await tryWriteResult( runtime, internal, inputsCell, INPUT_HASH, (writeTx) => result.withTx(writeTx).set({ from: "the response" }), ); expect(written).toEqual({ written: true }); expect(result.get()).toEqual({ from: "the response" }); expect(internal.get().inputHash).toBe(INPUT_HASH); }); it("reports a superseded writeback without a commit error, and writes nothing", async () => { const { inputsCell, internal, result } = await newCells("superseded"); const movedHash = computeInputHashFromValue({ url: "http://mock-test-server.local/api/the-inputs-that-moved", }); expect(movedHash).not.toBe(INPUT_HASH); let actionRan = false; const written = await tryWriteResult( runtime, internal, inputsCell, movedHash, (writeTx) => { actionRan = true; result.withTx(writeTx).set({ from: "the superseded response" }); }, ); // `written: false` and no `commitError`: the inputs moved, the action // never ran, and the request that owns the cells now is someone else's. expect(written).toEqual({ written: false }); expect(written.commitError).toBeUndefined(); expect(actionRan).toBe(false); expect(result.get()).toBeUndefined(); expect(internal.get().inputHash).toBe(""); }); it("carries a refused commit back as a commit error, and writes nothing", async () => { const { inputsCell, internal, result } = await newCells("refused"); const refused = refuseCommits(() => true, 1); let actionRan = false; const written = await tryWriteResult( runtime, internal, inputsCell, INPUT_HASH, (writeTx) => { actionRan = true; result.withTx(writeTx).set({ from: "the response" }); }, ); // The other "did not write" shape. Here the hash matched and the action // ran, so a caller reading only `written` could not tell this from the // superseded case above — and retiring the effect on it would leave the // claim pending with nothing left to complete it. expect(refused()).toBe(1); expect(actionRan).toBe(true); expect(written.written).toBe(false); expect(written.commitError).toBe(REFUSAL); expect(result.get()).toBeUndefined(); expect(internal.get().inputHash).toBe(""); }); }); describe("what fetchJson does with the refusal", () => { /** * Run `fetchJson` against a held response, and return the handles a case * needs to refuse its writeback: the pattern's result cell, the document * the builtin's own result cell lives in, the release for the response, * and every promise the builtin registered as tracked async work (the * channel the serving outbox counts `outbox.failed` through). */ async function fetchJsonAwaitingItsResponse(cause: string) { const release = deferred(); const issued = deferred(); globalThis.fetch = () => { issued.resolve(); return release.promise; }; const work: Promise[] = []; runtime.asyncWorkObserver = (registered) => void work.push(registered); const fetchJson = byRef("fetchJson"); // The staleness bound is put out of reach: the auto-advancing clock // would otherwise let a takeover clear `result` on its way in while the // response is held, which is not what these cases are about. const testPattern = pattern<{ url: string }>( ({ url }) => fetchJson({ url, options: { mutexTimeoutMs: 600_000 } }), ); const resultCell = runtime.getCell(space, cause, undefined, tx); const result = runtime.run(tx, testPattern, INPUTS, resultCell); tx.commit(); await result.pull(); await issued.promise; await clock.settle(); // The pattern's result cell holds links to the builtin's own cells, and // a plain link is not followed on write, so resolve them to name the // documents a writeback writes. const documentOf = (key: string) => runtime.getCellFromLink( resolveLink( runtime, runtime.readTx(), result.key(key).getAsNormalizedFullLink(), ), ).getAsNormalizedFullLink().id; return { result, resultDocument: documentOf("result"), pendingDocument: documentOf("pending"), work, respond: () => release.resolve( new Response(JSON.stringify({ from: "the response" }), { status: 200, headers: { "Content-Type": "application/json" }, }), ), }; } it("turns a refused completion writeback into an error-shaped result", async () => { const { result, resultDocument, work, respond } = await fetchJsonAwaitingItsResponse("fetch-writeback-refused"); // The writeback is the transaction that carries the response into the // builtin's result document. The error writeback that follows does not // write it — it clears a result that is already absent — so naming the // document refuses the completion write and nothing else. One // transaction matches, which the count states rather than assumes. const refused = refuseCommits( (written) => written.includes(resultDocument), 1, ); respond(); await runtime.settled(); expect(refused()).toBe(1); // Not silence, and not a success: the response's failure to land is // reported as the node's error, which is retryable and input-driven. const error = result.key("error").get() as { message?: string }; expect(error?.message).toBe( `fetchJson completion write failed: ${REFUSAL.message}`, ); expect(result.key("result").get()).toBeUndefined(); // The claim is released, so the node is not left showing a spinner. expect(result.key("pending").get()).toBe(false); // Nothing rejected: an error-shaped result is a completion. The count // is stated too, so an empty `work` cannot pass this vacuously. expect(work.length).toBeGreaterThan(0); const outcomes = await Promise.allSettled(work); expect(outcomes.filter((o) => o.status === "rejected")).toEqual([]); }); it("rejects the tracked work when even the error-shaped result cannot commit", async () => { const { result, pendingDocument, work, respond } = await fetchJsonAwaitingItsResponse("fetch-writeback-refused-twice"); // Both writebacks refused: the completion first, then the error-shaped // result that stands in for it. Both release the claim, so the pending // document names the two of them and nothing else — refusing every // commit instead would make the count depend on whatever the scheduler // happened to open in the same window. const refused = refuseCommits( (written) => written.includes(pendingDocument), Infinity, ); respond(); await runtime.settled(); expect(refused()).toBe(2); // Loud rather than silent: a `return` here would retire the effect as // completed, and recovery depends on the failure being counted. The // completion failure is carried as the cause, so the report names what // could not be written as well as what could not be reported. const outcomes = await Promise.allSettled(work); const rejections = outcomes.flatMap((outcome) => outcome.status === "rejected" ? [outcome.reason as Error] : [] ); expect(rejections.length).toBe(1); expect(rejections[0].message).toBe( `fetchJson error writeback failed to commit: ${REFUSAL.message}`, ); expect((rejections[0].cause as Error).message).toBe( `fetchJson completion write failed: ${REFUSAL.message}`, ); // Neither writeback landed, so the claim stays as the request left it: // pending, with no result and no error recorded. expect(result.key("pending").get()).toBe(true); expect(result.key("result").get()).toBeUndefined(); expect(result.key("error").get()).toBeUndefined(); }); }); });