/// /** * USPS Informed Delivery Mail Analyzer * * Processes USPS Informed Delivery emails to extract information about * incoming mail using LLM vision analysis. * * Features: * - Embeds gmail-importer directly (no separate piece needed) * - Pre-configured with USPS filter query and settings * - Auto-analyzes mail piece images with LLM vision * - Learns household members from recipient names over time * - Classifies mail type and spam likelihood * * Usage: * 1. Deploy a google-auth piece and complete OAuth * 2. Deploy this pattern * 3. Link: ct piece link google-auth/auth usps/overrideAuth */ import { computed, Default, generateObject, handler, JSONSchema, NAME, pattern, UI, Writable, } from "commontools"; import type { Schema } from "commontools/schema"; import GmailExtractor, { type Auth } from "../core/gmail-extractor.tsx"; import ProcessingStatus from "../core/processing-status.tsx"; // Debug flag for development - disable in production const DEBUG_USPS = false; // Email type - matches GmailImporter's Email type interface Email { id: string; from: string; to: string; subject: string; date: string; snippet: string; threadId: string; labelIds: string[]; htmlContent: string; plainText: string; markdownContent: string; } // ============================================================================= // TYPES // ============================================================================= type MailType = | "personal" | "bill" | "financial" | "advertisement" | "package" | "government" | "medical" | "subscription" | "charity" | "other"; /** A learned household member */ interface HouseholdMember { name: string; aliases: string[]; mailCount: number; firstSeen: number; isConfirmed: boolean; } // ============================================================================= // CONSTANTS // ============================================================================= const USPS_SENDER = "informeddelivery.usps.com"; // Schema for LLM mail piece analysis const MAIL_ANALYSIS_SCHEMA = { type: "object", properties: { recipient: { type: "string", description: "Full name of the recipient shown on the mail piece", }, sender: { type: "string", description: "Name of the sender or company (from return address)", }, mailType: { type: "string", enum: [ "personal", "bill", "financial", "advertisement", "package", "government", "medical", "subscription", "charity", "other", ], description: "Type/category of this mail piece", }, isLikelySpam: { type: "boolean", description: "Whether this appears to be junk mail or spam", }, spamConfidence: { type: "number", minimum: 0, maximum: 100, description: "Confidence score for spam classification (0-100)", }, isUrgent: { type: "boolean", description: "Whether this mail appears urgent or time-sensitive (bills with due dates, government notices, legal documents, medical correspondence, tax forms, etc.)", }, urgentReason: { type: "string", description: "Brief explanation of why this mail is considered urgent, or empty string if not urgent", }, summary: { type: "string", description: "Brief one-sentence description of this mail piece", }, }, required: [ "recipient", "sender", "mailType", "isLikelySpam", "spamConfidence", "isUrgent", "urgentReason", "summary", ], } as const satisfies JSONSchema; type MailAnalysis = Schema; // ============================================================================= // HELPERS // ============================================================================= /** * Extract image URLs/CIDs from USPS Informed Delivery email HTML content. * * USPS Informed Delivery emails embed mail piece images as inline MIME attachments * referenced by Content-ID (cid:). The images are NOT external URLs. * * Format in HTML: Mailpiece Image * * To use these images: * 1. gmail-importer needs to fetch attachments and resolve CID references * 2. Replace cid: URLs with base64 data URLs * 3. Then the LLM can analyze them * * For now, we extract cid: references to show what images exist, and also * accept base64 data URLs (if gmail-importer resolves them). */ function extractMailPieceImages(htmlContent: string): string[] { const images: string[] = []; // Look for img tags const imgRegex = /]+src=["']([^"']+)["'][^>]*>/gi; let match; while ((match = imgRegex.exec(htmlContent)) !== null) { const src = match[1]; // USPS mail piece images are inline attachments referenced by cid: // Example: cid:1019388469-033.jpg // Filter for mailpiece images (numeric ID pattern, not logos like "mailer-" or "content-") if (src.startsWith("cid:") && /^cid:\d+/.test(src)) { images.push(src); } // Accept base64 encoded images (resolved from cid: by gmail-importer) else if (src.startsWith("data:image")) { images.push(src); } } return images; } /** * Normalize a recipient name for comparison. */ function normalizeName(name: string | null | undefined): string { if (!name) return ""; return name .trim() .toLowerCase() .replace(/[^a-z\s]/g, "") .replace(/\s+/g, " "); } /** * Check if two names are likely the same person (fuzzy match). * Prefixed with _ as not currently used - preserved for future use. */ function _namesMatch(name1: string, name2: string): boolean { const n1 = normalizeName(name1); const n2 = normalizeName(name2); if (n1 === n2) return true; // Check if one is a substring of the other (handles initials) const parts1 = n1.split(" "); const parts2 = n2.split(" "); // Same last name? if (parts1.length > 0 && parts2.length > 0) { const last1 = parts1[parts1.length - 1]; const last2 = parts2[parts2.length - 1]; if (last1 === last2) return true; } return false; } // ============================================================================= // HANDLERS // ============================================================================= // Handler to confirm a household member // Uses cell reference with .equals() - idiomatic approach const confirmMember = handler< unknown, { member: Writable } >((_event, { member }) => { const current = member.get(); member.set({ ...current, isConfirmed: true }); }); // Handler to delete a household member // Uses cell reference - pass householdMembers array and the member cell const deleteMember = handler< unknown, { householdMembers: Writable; member: Writable; } >((_event, { householdMembers, member }) => { householdMembers.remove(member); }); // NOTE: No triggerAnalysis handler needed! // generateObject must be called at pattern level (reactive), not inside handlers. // The pattern uses .map() to process emails reactively - see mailPieceAnalyses in pattern body. // ============================================================================= // PATTERN // ============================================================================= interface PatternInput { householdMembers?: Default; // Optional: Link auth directly from a Google Auth piece // Use: ct piece link googleAuthPiece/auth uspsPiece/overrideAuth overrideAuth?: Auth; } /** USPS Informed Delivery mail analyzer. #uspsInformedDelivery */ interface PatternOutput { mailPieces: MailAnalysis[]; householdMembers: HouseholdMember[]; mailCount: number; spamCount: number; personalCount: number; billCount: number; financialCount: number; advertisementCount: number; packageCount: number; governmentCount: number; medicalCount: number; subscriptionCount: number; charityCount: number; previewUI: unknown; } export default pattern( ({ householdMembers, overrideAuth }) => { // Directly instantiate GmailExtractor with USPS-specific settings (raw mode) // This eliminates the need for separate gmail-importer piece + wish() const extractor = GmailExtractor({ gmailQuery: "from:USPSInformeddelivery@email.informeddelivery.usps.com", resolveInlineImages: true, limit: 20, title: "USPS Mail", overrideAuth, // Pass through from USPS input (user can link google-auth here) }); // Get emails directly from the embedded extractor const allEmails = extractor.emails; // Filter for USPS emails const uspsEmails = computed(() => { return (allEmails || []).filter((e: Email) => e.from?.toLowerCase().includes(USPS_SENDER) ); }); // Count of USPS emails found const uspsEmailCount = computed(() => uspsEmails?.length || 0); // Use extractor's isConnected - it handles auth checking internally const isConnected = extractor.isConnected; // ========================================================================== // REACTIVE LLM ANALYSIS // Extract images from emails and analyze them with generateObject at pattern level // This is the correct approach - generateObject must be called reactively, not in handlers // ========================================================================== // First, extract all mail piece images from all emails // Returns array of { emailId, emailDate, imageUrl } objects const mailPieceImages = computed(() => { const images: { emailId: string; emailDate: string; imageUrl: string; imageIndex: number; }[] = []; for (const email of uspsEmails || []) { const urls = extractMailPieceImages(email.htmlContent); urls.forEach((url, idx) => { images.push({ emailId: email.id, emailDate: email.date, imageUrl: url, imageIndex: idx, }); }); } // Limit to first 10 images for now to avoid overwhelming LLM calls return images.slice(0, 10); }); // Count of images to analyze const imageCount = computed(() => mailPieceImages?.length || 0); // Analyze each image with generateObject - this is called at pattern level (reactive) // Uses .map() over the derived array to create per-item LLM calls with automatic caching const mailPieceAnalyses = mailPieceImages.map((imageInfo) => { // Get the image URL from the cell const analysis = generateObject({ // Prompt computed from imageUrl prompt: computed(() => { if (!imageInfo?.imageUrl) { if (DEBUG_USPS) { console.log(`[USPS LLM] Empty URL, returning text-only prompt`); } return undefined; // No-op while there is no URL } const url = imageInfo.imageUrl; // Debug logging (gated by DEBUG_USPS flag) if (DEBUG_USPS) { console.log( `[USPS LLM] Processing image URL (first 100 chars):`, url?.slice(0, 100), ); } // Check if it's a base64 data URL vs external URL const isBase64 = url.startsWith("data:"); if (DEBUG_USPS) { console.log( `[USPS LLM] Image type: ${isBase64 ? "base64" : "external URL"}`, ); } // NOTE: External URLs from USPS require authentication and may fail. // The LLM server cannot fetch authenticated URLs. // For now, we still try and display an error if it fails. return [ { type: "image" as const, image: url }, { type: "text" as const, text: `Analyze this scanned mail piece image from USPS Informed Delivery. Extract: 1. The recipient name (who the mail is addressed to) 2. The sender or company name (from return address if visible) 3. The type of mail - use these categories: - "personal": Greeting cards, holiday cards, wedding invitations, thank you notes, handwritten letters - mail personally sent to you, NOT business mail - "bill": Utility bills, credit card statements, invoices, payment requests - "financial": Bank statements, tax documents (1099s, W-2s), investment reports, brokerage statements - financial documents that are NOT bills - "advertisement": Flyers, coupons, promotional mailers, catalogs, marketing materials - "package": Package arrival notices, delivery confirmations - "government": IRS notices, DMV renewals, court documents, voter registration, official government correspondence - "medical": Insurance EOBs, appointment reminders, prescription notices, hospital correspondence - "subscription": Magazines, newsletters, membership renewals - "charity": Donation requests, nonprofit mailings - "other": Anything that doesn't fit the above categories 4. Whether it appears to be spam/junk mail 5. Whether it appears URGENT or time-sensitive. Consider urgent: - Government correspondence (IRS, DMV, court notices, etc.) - Legal documents or certified mail indicators - Medical correspondence (hospitals, insurance EOBs, etc.) - Late bills mentioning due dates - Bank account alerts - Time-sensitive offers with deadlines IMPORTANT: Use "personal" ONLY for greeting cards, holiday cards, and handwritten correspondence - NOT for business mail. If you cannot read the image clearly, make your best guess based on what you can see.`, }, ]; }), schema: MAIL_ANALYSIS_SCHEMA, // IMPORTANT: Must specify model explicitly for generateObject with images model: "anthropic:claude-sonnet-4-5", }); return { imageInfo, imageUrl: imageInfo.imageUrl, analysis, pending: analysis.pending, error: analysis.error, result: analysis.result, }; }); // Count pending analyses const pendingCount = computed( () => mailPieceAnalyses?.filter((a) => a?.pending)?.length || 0, ); // Count completed analyses const completedCount = computed( () => mailPieceAnalyses?.filter((a) => a?.analysis?.pending === false && a?.analysis?.result !== undefined ).length || 0, ); const mailPieces = mailPieceAnalyses.map((a) => a.result); // Derived counts from stored mailPieces const mailCount = computed(() => mailPieces?.length || 0); const spamCount = computed( () => mailPieces?.filter((p) => p?.isLikelySpam)?.length || 0, ); // Category counts const personalCount = computed( () => mailPieces?.filter((p) => p?.mailType === "personal")?.length || 0, ); const billCount = computed( () => mailPieces?.filter((p) => p?.mailType === "bill")?.length || 0, ); const financialCount = computed( () => mailPieces?.filter((p) => p?.mailType === "financial")?.length || 0, ); const advertisementCount = computed( () => mailPieces?.filter((p) => p?.mailType === "advertisement")?.length || 0, ); const packageCount = computed( () => mailPieces?.filter((p) => p?.mailType === "package")?.length || 0, ); const governmentCount = computed( () => mailPieces?.filter((p) => p?.mailType === "government")?.length || 0, ); const medicalCount = computed( () => mailPieces?.filter((p) => p?.mailType === "medical")?.length || 0, ); const subscriptionCount = computed( () => mailPieces?.filter((p) => p?.mailType === "subscription")?.length || 0, ); const charityCount = computed( () => mailPieces?.filter((p) => p?.mailType === "charity")?.length || 0, ); // Filter for urgent mail pieces (with their analysis data for display) const urgentMailAnalyses = computed(() => (mailPieceAnalyses || []).filter((a) => a?.result?.isUrgent && !a?.pending ) ); const urgentCount = computed(() => urgentMailAnalyses?.length || 0); // Unconfirmed members count const unconfirmedCount = computed( () => householdMembers?.filter((m) => !m.isConfirmed)?.length || 0, ); // Get top 3 categories for preview summary const topCategories = computed(() => { const categories = [ { name: "personal", count: personalCount }, { name: "bills", count: billCount }, { name: "financial", count: financialCount }, { name: "ads", count: advertisementCount }, { name: "packages", count: packageCount }, { name: "government", count: governmentCount }, { name: "medical", count: medicalCount }, { name: "subscriptions", count: subscriptionCount }, { name: "charity", count: charityCount }, ]; return categories .filter((c) => (c.count || 0) > 0) .sort((a, b) => (b.count || 0) - (a.count || 0)) .slice(0, 3); }); // Preview UI for compact display in lists/pickers const previewUI = (
{/* Blue badge with mail count */}
{mailCount}
{/* Label and summary */}
USPS Mail
{topCategories.map((cat, idx) => ( {idx > 0 ? " · " : ""} {cat.count} {cat.name} ))} {spamCount > 0 && ( · {spamCount} spam )}
{/* Loading/progress indicator */}
); return { [NAME]: "USPS Informed Delivery", mailPieces, householdMembers, mailCount, spamCount, personalCount, billCount, financialCount, advertisementCount, packageCount, governmentCount, medicalCount, subscriptionCount, charityCount, previewUI, [UI]: (
USPS Informed Delivery
{/* Auth UI from embedded GmailExtractor */} {extractor.ui.authStatusUI} {/* Connection Status */} {isConnected ? (
Connected to Gmail {uspsEmailCount} USPS emails found
) : null} {/* Analysis Status - reactive, no button needed */} {isConnected ? (
Analysis: {imageCount} images found {pendingCount > 0 && ( {pendingCount} analyzing... )} {completedCount} completed
) : null} {/* Stats */}
{/* Top row: Mail Pieces and Spam/Junk */}
{mailCount}
Mail Pieces
{spamCount}
Spam/Junk
{/* Category badges */}
{personalCount > 0 && ( {personalCount} Personal )} {billCount > 0 && ( {billCount} Bills )} {financialCount > 0 && ( {financialCount} Financial )} {advertisementCount > 0 && ( {advertisementCount} Ads )} {packageCount > 0 && ( {packageCount} Packages )} {governmentCount > 0 && ( {governmentCount} Government )} {medicalCount > 0 && ( {medicalCount} Medical )} {subscriptionCount > 0 && ( {subscriptionCount} Subscriptions )} {charityCount > 0 && ( {charityCount} Charity )}
{/* Household Members */} {false && (
Household Members {unconfirmedCount > 0 ? ( {unconfirmedCount} unconfirmed ) : null} {!householdMembers?.length ? (
No household members learned yet. Analyze some mail to get started.
) : null} {/* Use .map() directly on cell array to get cell references */} {householdMembers.map((member) => (
{member.name}
{member.mailCount} pieces {member.aliases?.length > 0 ? ` • Also: ${member.aliases.join(", ")}` : ""}
{!member.isConfirmed ? ( ) : null}
))}
)} { /* Possibly Urgent Section WORKAROUND: Using CSS display:none instead of conditional rendering (ifElse or &&) because .map() inside conditionals doesn't get transformed to mapWithPattern, causing raw vnode JSON to render instead of actual UI elements. See: packages/ts-transformers/ISSUES_TO_FOLLOW_UP.md Issue #5 Related: https://github.com/user/repo/commit/1b10bac4d (link subscription bug) */ }
0 ? "block" : "none", }} >
⚠️ Possibly Urgent ({urgentCount})
{mailPieceAnalyses.map((analysisItem) => (
{/* Enlarged thumbnail */}
Urgent mail piece
{/* Info */}
{analysisItem.result?.sender || "Unknown Sender"}
To: {analysisItem.result?.recipient || "Unknown"}
{analysisItem.result?.urgentReason || "Time-sensitive"}
))}
{/* Reactive Mail Piece Analysis Results */}
Mail Pieces (Live Analysis) {imageCount === 0 ? (
No mail piece images found in emails. USPS emails may not contain scanned images.
) : null} {/* Map over reactive analyses */} {mailPieceAnalyses.map((analysisItem) => (
{/* Image thumbnail */}
Mail piece
{/* Details */}
{analysisItem.pending ? (
Analyzing...
) : analysisItem.error ? (
LLM Error (image may be inaccessible)
) : (
{analysisItem.result?.recipient || "Unknown"} {analysisItem.result?.isUrgent ? ( URGENT ) : null} {analysisItem.result?.isLikelySpam ? ( SPAM ) : null}
From: {analysisItem.result?.sender || "Unknown"}
{analysisItem.result?.summary || ""}
Type:{" "} {analysisItem.result?.mailType || "unknown"} {" "} • Spam:{" "} {analysisItem.result?.spamConfidence || 0}%
)}
))}
), }; }, );