///
/**
* Async builtin work must reach `runtime.trackAsyncWork()`, and it must reach
* it under the run that started it.
*
* `settled()` and `settledFor()` are the runtime's "everything has finished"
* barriers, and a barrier is only as truthful as its registry. Work a builtin
* never registers is work the barrier cannot see, so the barrier reports
* quiescence while the builtin is still running. That failure is silent, and it
* is worse than a deadline: a deadline firing early reports a failure, whereas
* a barrier returning early reports success with the data missing. An LLM tool
* call waits on `settledFor()`, so an unregistered builtin hands the model an
* empty tool result and the conversation continues as if that were the answer.
*
* `generateObject` was exactly that hole. Both of its paths built a promise
* spanning the model call and the writeback, then attached only a `.catch()` to
* it, so the whole tool-calling subagent shape ran outside the registry.
*
* Two halves guard it:
*
* - Each builtin below runs with its response held open, and both barriers must
* stay open for as long as it is held, while a barrier scoped to an unrelated
* cell must return. The pair pins both properties at once: the work reached
* the registry, and it reached it under the run that started it rather than
* under nobody, which would make every run wait for every other.
* - The source scan fails when a builtin file grows a post-commit side effect
* without registering async work. A builtin added later cannot reopen the
* hole without either registering or saying in
* `POST_COMMIT_WITHOUT_TRACKED_WORK` why it does not have to.
*/
import { afterEach, beforeEach, describe, it } from "@std/testing/bdd";
import { expect } from "@std/expect";
import { Identity } from "@commonfabric/identity";
import { StorageManager } from "@commonfabric/runner/storage/cache.deno";
import {
addMockObjectResponse,
addMockResponse,
clearMockResponses,
enableMockMode,
setMockResponseGate,
} from "@commonfabric/llm/client";
import type { BuiltInLLMMessage, BuiltInLLMTool } from "@commonfabric/api";
import { defer } from "@commonfabric/utils/defer";
import type { Cell, JSONSchema } from "../src/builder/types.ts";
import { createBuilder } from "../src/builder/factory.ts";
import { createTrustedBuilder } from "./support/trusted-builder.ts";
import { Runtime } from "../src/runtime.ts";
import type { IExtendedStorageTransaction } from "../src/storage/interface.ts";
import { LLMMessageSchema } from "../src/builtins/llm-schemas.ts";
const signer = await Identity.fromPassphrase("test operator");
const space = signer.did();
enableMockMode();
/**
* Builtin source files that enqueue a post-commit side effect and deliberately
* do not hand a promise to `trackAsyncWork`, with the reason. Anything absent
* from this list has to register, and the source scan names it when it does not.
*
* Empty: every builtin that enqueues one registers work of its own. Two of them
* register the settle for an abandoned request rather than the flush, and say
* at the flush itself why that one is not registered.
*/
const POST_COMMIT_WITHOUT_TRACKED_WORK: Record = {};
const OK_SCHEMA = {
type: "object",
properties: { ok: { type: "boolean" } },
required: ["ok"],
} as const satisfies JSONSchema;
describe("async builtin work registration", () => {
let storageManager: ReturnType;
let runtime: Runtime;
let tx: IExtendedStorageTransaction;
let builder: ReturnType["commonfabric"];
let held: ReturnType>;
beforeEach(() => {
clearMockResponses();
storageManager = StorageManager.emulate({ as: signer });
runtime = new Runtime({
apiUrl: new URL(import.meta.url),
storageManager,
});
tx = runtime.edit();
builder = createTrustedBuilder(runtime).commonfabric;
// Hold the model's answer so there is a window in which the request is
// genuinely outstanding. A mock that answers within a microtask leaves no
// such window, and an unregistered builtin looks correct.
held = defer();
setMockResponseGate(() => held.promise);
});
afterEach(async () => {
setMockResponseGate(undefined);
await tx.commit().settled;
await runtime.idle();
await runtime?.dispose();
await storageManager?.close();
});
/**
* Neither the whole-runtime barrier nor the one scoped to this run may return
* while the response is held, and both must return once it is released with
* the result already in the cell. A barrier scoped to a cell that owns nothing
* must return while the response is still held: the work belongs to the run
* that started it, not to the runtime at large.
*/
const expectBarriersSpanTheCall = async (
run: Cell,
readResult: () => unknown,
expected: unknown,
) => {
// The built-in is a computation: a reader has to demand it before any
// request goes out, and the barriers are measured against that demand.
const cancelDemand = run.sink(() => {});
let settledReturned = false;
const settledPromise = runtime.settled().then(() => {
settledReturned = true;
});
let settledForReturned = false;
const settledForPromise = runtime.settledFor(run).then(() => {
settledForReturned = true;
});
const unrelated = runtime.getCell(space, "async-work-unrelated-run");
let unrelatedReturned = false;
const unrelatedPromise = runtime.settledFor(unrelated).then(() => {
unrelatedReturned = true;
});
// Drain every microtask and zero-delay timer to a fixpoint without moving
// the clock. Anything the runtime was going to do while the response is held
// has now happened, so a barrier that was going to return early has.
await clock.settle();
expect(settledReturned).toBe(false);
expect(settledForReturned).toBe(false);
expect(unrelatedReturned).toBe(true);
held.resolve();
await settledPromise;
await settledForPromise;
await unrelatedPromise;
// Read straight out of the cell, with no wait between this and the barriers
// above. Anything that has to be waited for here is work they failed to
// cover.
expect(readResult()).toEqual(expected);
cancelDemand();
};
it("registers the generateObject direct path against its run", async () => {
addMockObjectResponse(
(req) =>
req.messages.some((message: BuiltInLLMMessage) =>
typeof message.content === "string" &&
message.content.includes("direct-path-prompt")
),
{ object: { ok: true }, id: "mock-direct" },
);
const testPattern = builder.pattern>(() =>
builder.generateObject({
prompt: "direct-path-prompt",
schema: OK_SCHEMA,
} as any)
);
const resultCell = runtime.getCell(
space,
"async-work-generate-object-direct",
testPattern.resultSchema,
tx,
);
const run = runtime.run(tx, testPattern, {}, resultCell);
tx.commit();
await expectBarriersSpanTheCall(
run,
() => run.withTx().key("result").get(),
{ ok: true },
);
});
it("registers the generateObject tool-calling path against its run", async () => {
addMockResponse(
(req) => req.tools?.["presentResult"] !== undefined,
{
role: "assistant",
content: [{
type: "tool-call",
toolCallId: "call_present_result",
toolName: "presentResult",
input: { ok: true },
}],
id: "mock-tools",
},
);
const helper = builder.pattern, { ok: boolean }>(
() => ({ ok: true }),
{ type: "object", additionalProperties: false },
OK_SCHEMA,
);
const testPattern = builder.pattern>(() =>
builder.generateObject({
prompt: "tools-path-prompt",
schema: OK_SCHEMA,
tools: {
helperTool: builder.patternTool(helper) as unknown as BuiltInLLMTool,
},
} as any)
);
const resultCell = runtime.getCell(
space,
"async-work-generate-object-tools",
testPattern.resultSchema,
tx,
);
const run = runtime.run(tx, testPattern, {}, resultCell);
tx.commit();
await expectBarriersSpanTheCall(
run,
() => run.withTx().key("result").get(),
{ ok: true },
);
});
it("registers a generateText call against its run", async () => {
addMockResponse(
(req) =>
req.messages.some((message: BuiltInLLMMessage) =>
typeof message.content === "string" &&
message.content.includes("generate-text-prompt")
),
{ role: "assistant", content: "generated", id: "mock-text" },
);
const testPattern = builder.pattern>(() =>
builder.generateText({ prompt: "generate-text-prompt" } as any)
);
const resultCell = runtime.getCell(
space,
"async-work-generate-text",
testPattern.resultSchema,
tx,
);
const run = runtime.run(tx, testPattern, {}, resultCell);
tx.commit();
await expectBarriersSpanTheCall(
run,
() => run.withTx().key("result").get(),
"generated",
);
});
it("registers an llmDialog turn against its run", async () => {
addMockResponse(
(req) =>
req.messages.some((message: BuiltInLLMMessage) =>
typeof message.content === "string" &&
message.content.includes("dialog-prompt")
),
{ role: "assistant", content: "Hi there!", id: "mock-dialog" },
);
const resultSchema = {
type: "object",
properties: {
addMessage: { ...LLMMessageSchema, asCell: ["stream"] },
pending: { type: "boolean" },
messages: {
type: "array",
items: { type: "object", additionalProperties: true },
},
},
required: ["addMessage"],
} as const satisfies JSONSchema;
const testPattern = builder.pattern(
() => {
const messages = builder.Cell.of([]);
const dialog = builder.llmDialog({ messages });
return {
addMessage: dialog.addMessage,
pending: dialog.pending,
messages,
};
},
false,
resultSchema,
);
const resultCell = runtime.getCell(
space,
"async-work-llm-dialog",
resultSchema,
tx,
);
const run = runtime.run(tx, testPattern, {}, resultCell);
tx.commit();
const addMessage = await run.key("addMessage").pull();
addMessage.send({ role: "user", content: "dialog-prompt" });
await expectBarriersSpanTheCall(
run,
() => run.withTx().key("messages").get()?.at(-1)?.content,
"Hi there!",
);
});
it("has every builtin that enqueues a post-commit effect register its work", async () => {
const builtinsDir = new URL("../src/builtins/", import.meta.url);
const unregistered: string[] = [];
// An exemption that no longer describes its file is a claim nobody is
// checking, so it is reported rather than left to authorize a future gap.
const staleExemptions: string[] = [];
const seen = new Set();
for await (const entry of Deno.readDir(builtinsDir)) {
if (!entry.isFile || !entry.name.endsWith(".ts")) continue;
const source = await Deno.readTextFile(new URL(entry.name, builtinsDir));
const exempt = entry.name in POST_COMMIT_WITHOUT_TRACKED_WORK;
if (exempt) seen.add(entry.name);
const enqueues = source.includes("enqueuePostCommitEffect") ||
source.includes("enqueueSinkRequestPostCommitEffect");
const registers = source.includes("trackAsyncWork");
if (!enqueues || registers) {
if (exempt) staleExemptions.push(entry.name);
continue;
}
if (!exempt) unregistered.push(entry.name);
}
for (const name of Object.keys(POST_COMMIT_WITHOUT_TRACKED_WORK)) {
if (!seen.has(name)) staleExemptions.push(name);
}
expect(unregistered).toEqual([]);
expect(staleExemptions).toEqual([]);
});
});