import { env } from "@commontools/integration"; import { sleep } from "@commontools/utils/sleep"; import { CharmsController } from "@commontools/charm/ops"; import { ShellIntegration } from "@commontools/integration/shell-utils"; import { afterAll, beforeAll, describe, it } from "@std/testing/bdd"; import { join } from "@std/path"; import { assert, assertEquals } from "@std/assert"; import { Identity } from "@commontools/identity"; import { TEST_LLM } from "./flags.ts"; const { API_URL, FRONTEND_URL, SPACE_NAME } = env; const ignore = !TEST_LLM; // LLM tests are skipped in CI until we handle llm() calls properly in CI environments. // This requires either: // 1. Adding a flag to enable LLM tests in CI with proper API keys // 2. Resurrecting the LLM cache functionality from toolshed describe("LLM pattern test", () => { const shell = new ShellIntegration(); shell.bindLifecycle(); let charmId: string; let identity: Identity; let cc: CharmsController; if (!ignore) { beforeAll(async () => { identity = await Identity.generate({ implementation: "noble" }); cc = await CharmsController.initialize({ spaceName: SPACE_NAME, apiUrl: new URL(API_URL), identity: identity, }); const charm = await cc.create( await Deno.readTextFile( join( import.meta.dirname!, "..", "examples", "llm.tsx", ), ), { start: false }, ); charmId = charm.id; }); afterAll(async () => { if (cc) await cc.dispose(); }); } it({ name: "should load the LLM test charm and display initial UI", ignore, fn: async () => { const page = shell.page(); await shell.goto({ frontendUrl: FRONTEND_URL, view: { spaceName: SPACE_NAME, charmId, }, identity, }); // Wait for the component to render by waiting for title const title = await page.waitForSelector("h2", { strategy: "pierce", }); assert(title, "Should find title element"); const titleText = await title.evaluate((el: HTMLElement) => el.textContent ); assertEquals(titleText?.trim(), "LLM Test"); // Check for the message input const messageInput = await page.waitForSelector("ct-message-input", { strategy: "pierce", }); assert(messageInput, "Should find message input element"); }, }); it({ name: "should handle question input and display question", ignore, fn: async () => { const page = shell.page(); // Find the message input component const messageInput = await page.waitForSelector("ct-message-input", { strategy: "pierce", }); assert(messageInput, "Should find message input"); // Look for the actual input element inside the message input component const inputElement = await page.waitForSelector("input", { strategy: "pierce", }); assert(inputElement, "Should find input element"); // Type a question const testQuestion = "What is 2 + 2?"; await inputElement.type(testQuestion); // Find and click the send button const sendButton = await page.waitForSelector("[data-ct-button]", { strategy: "pierce", }); assert(sendButton, "Should find send button"); await sendButton.click(); // Wait for the question to appear by waiting for blockquote const questionSection = await page.waitForSelector("blockquote", { strategy: "pierce", }); assert(questionSection, "Should find question blockquote"); const questionText = await questionSection.evaluate((el: HTMLElement) => el.textContent ); assertEquals(questionText?.trim(), testQuestion); }, }); it({ name: "should display LLM response after asking a question", ignore, fn: async () => { const page = shell.page(); // Wait for LLM response to appear (this may take some time) // The response appears in a
element
const responseElement = await page.waitForSelector("pre", {
strategy: "pierce",
timeout: 30000, // 30 second timeout for LLM response
});
assert(responseElement, "Should find LLM response element");
const responseText = await responseElement.evaluate((el: HTMLElement) =>
el.textContent
);
assert(responseText, "Should have response text");
assert(responseText.trim().length > 0, "Response should not be empty");
// The response should contain some form of answer to "What is 2 + 2?"
// We'll just check that we got some text back rather than checking exact content
// since LLM responses can vary
console.log("LLM Response:", responseText);
},
});
it({
name: "should handle multiple questions in sequence",
ignore,
fn: async () => {
const page = shell.page();
// Reduced wait for system to settle (was 2000ms)
await sleep(200);
// Ask a second question - need to refocus the input first
const inputElement = await page.waitForSelector("input", {
strategy: "pierce",
});
assert(inputElement, "Should find input element");
// Focus and clear previous input, then type new question
await inputElement.click(); // Focus the input
await inputElement.evaluate((el: HTMLInputElement) => {
el.value = "";
});
const secondQuestion = "What is the capital of France?";
await inputElement.type(secondQuestion);
const sendButton = await page.waitForSelector("[data-ct-button]", {
strategy: "pierce",
});
await sendButton.click();
// Wait for UI to update
await sleep(200);
// Check that the new question is displayed
const questionSection = await page.waitForSelector("blockquote", {
strategy: "pierce",
});
const questionText = await questionSection.evaluate((el: HTMLElement) =>
el.textContent
);
assertEquals(questionText?.trim(), secondQuestion);
// Wait for new response - increased timeout for LLM
const responseElement = await page.waitForSelector("pre", {
strategy: "pierce",
timeout: 60000, // 60 second timeout for LLM response
});
const responseText = await responseElement.evaluate((el: HTMLElement) =>
el.textContent
);
assert(responseText, "Should have response text for second question");
console.log("Second LLM Response:", responseText);
},
});
});