import { parseArgs } from "@std/cli/parse-args"; import { fromFileUrl, join as joinFileSystemPath, relative as relativeFileSystemPath, } from "@std/path"; import { basename as graphPathBasename, dirname as graphPathDirname, extname as graphPathExtension, join as joinGraphPath, normalize as normalizeGraphPath, } from "@std/path/posix"; import { Parser as HtmlParser } from "htmlparser2"; import { marked, type Token } from "marked"; interface MarkdownFile { readonly name: string; readonly fileSystemPath: string; readonly graphPath: string; } interface DirectoryTree { readonly name: string; readonly graphPath: string; readonly files: readonly MarkdownFile[]; readonly directories: readonly DirectoryTree[]; readonly directoryPaths: readonly string[]; readonly otherFilePaths: readonly string[]; } interface DirectedEdge { readonly source: string; readonly target: string; } interface RenderedEdge extends DirectedEdge { readonly bidirectional: boolean; } interface DocsGraph { readonly directoryTree: DirectoryTree; readonly files: readonly MarkdownFile[]; readonly includeHistory: boolean; readonly directedEdges: readonly DirectedEdge[]; readonly renderedEdges: readonly RenderedEdge[]; readonly incoming: ReadonlySet; } type OutputFormat = "dot" | "json" | "html" | "orphan"; interface CommandLineOptions { readonly format: OutputFormat; readonly includeHistory: boolean; readonly outputPath?: string; } const directoryBorderColor = "#20b2aa"; const directoryBackgroundColor = `${directoryBorderColor}1a`; const historyGraphPath = "docs/history"; const compareCodePoints = (left: string, right: string): number => { const leftCharacters = [...left]; const rightCharacters = [...right]; const sharedLength = Math.min(leftCharacters.length, rightCharacters.length); for (let index = 0; index < sharedLength; index += 1) { const difference = leftCharacters[index].codePointAt(0)! - rightCharacters[index].codePointAt(0)!; if (difference !== 0) return difference; } return leftCharacters.length - rightCharacters.length; }; // Graph paths sort by punctuation, digits, then case-insensitive ASCII letters. // Lowercase precedes uppercase when the primary paths match. const graphPathPrimaryWeight = (character: string): number => { if (character === "_") return 0; if (character === "-") return 1; if (character === ".") return 2; if (character === "/") return 3; if (character >= "0" && character <= "9") { return 10 + character.codePointAt(0)! - "0".codePointAt(0)!; } const isAsciiLetter = (character >= "a" && character <= "z") || (character >= "A" && character <= "Z"); if (isAsciiLetter) { const lowercase = character.toLowerCase(); return 30 + lowercase.codePointAt(0)! - "a".codePointAt(0)!; } return 1_000 + character.codePointAt(0)!; }; const compareGraphPaths = (left: string, right: string): number => { const leftCharacters = [...left]; const rightCharacters = [...right]; const sharedLength = Math.min(leftCharacters.length, rightCharacters.length); for (let index = 0; index < sharedLength; index += 1) { const weightDifference = graphPathPrimaryWeight(leftCharacters[index]) - graphPathPrimaryWeight(rightCharacters[index]); if (weightDifference !== 0) return weightDifference; } if (leftCharacters.length !== rightCharacters.length) { return leftCharacters.length - rightCharacters.length; } for (let index = 0; index < leftCharacters.length; index += 1) { if (leftCharacters[index] === rightCharacters[index]) continue; const leftIsLowercaseAscii = leftCharacters[index] >= "a" && leftCharacters[index] <= "z"; const rightIsLowercaseAscii = rightCharacters[index] >= "a" && rightCharacters[index] <= "z"; if (leftIsLowercaseAscii !== rightIsLowercaseAscii) { return leftIsLowercaseAscii ? -1 : 1; } return compareCodePoints(leftCharacters[index], rightCharacters[index]); } return 0; }; const readDirectoryTree = async ( fileSystemPath: string, graphPath: string, excludedDirectoryPaths: ReadonlySet, ): Promise => { const entries = [...await Array.fromAsync(Deno.readDir(fileSystemPath))] .sort((left, right) => compareCodePoints(left.name, right.name)); const files = entries .filter((entry) => entry.isFile && entry.name.endsWith(".md")) .map((entry) => ({ name: entry.name, fileSystemPath: joinFileSystemPath(fileSystemPath, entry.name), graphPath: `${graphPath}/${entry.name}`, })); const directories: DirectoryTree[] = []; const directoryPaths = [graphPath]; const otherFilePaths = entries .filter((entry) => !entry.isDirectory && !(entry.isFile && entry.name.endsWith(".md")) ) .map((entry) => `${graphPath}/${entry.name}`); for (const entry of entries) { if (!entry.isDirectory) continue; const childGraphPath = `${graphPath}/${entry.name}`; if (excludedDirectoryPaths.has(childGraphPath)) continue; const child = await readDirectoryTree( joinFileSystemPath(fileSystemPath, entry.name), childGraphPath, excludedDirectoryPaths, ); for (const directoryPath of child.directoryPaths) { directoryPaths.push(directoryPath); } for (const otherFilePath of child.otherFilePaths) { otherFilePaths.push(otherFilePath); } if (child.files.length > 0 || child.directories.length > 0) { directories.push(child); } } return { name: graphPathBasename(graphPath), graphPath, files, directories, directoryPaths, otherFilePaths, }; }; const flattenFiles = (directory: DirectoryTree): MarkdownFile[] => [ ...directory.files, ...directory.directories.flatMap(flattenFiles), ]; const graphString = (value: string) => `"${ value .replaceAll("\\", "\\\\") .replaceAll('"', '\\"') .replaceAll("\r", "\\r") .replaceAll("\n", "\\n") }"`; const htmlText = (value: string) => value .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">"); const scriptJson = (value: unknown): string => { const json = JSON.stringify(value); if (json === undefined) { throw new Error("Value cannot be represented as JSON"); } return json .replaceAll("&", "\\u0026") .replaceAll("<", "\\u003c") .replaceAll(">", "\\u003e") .replaceAll("\u2028", "\\u2028") .replaceAll("\u2029", "\\u2029"); }; const resolveLink = ( source: string, href: string, filePaths: ReadonlySet, directoryPaths: ReadonlySet, otherFilePaths: ReadonlySet, ): string | undefined => { let destination = href.trim(); if (destination.startsWith("<") && destination.endsWith(">")) { destination = destination.slice(1, -1); } if ( destination === "" || destination.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(destination) || destination.startsWith("//") ) { return undefined; } destination = destination.split("#", 1)[0].split("?", 1)[0]; if (destination === "") return undefined; try { destination = decodeURIComponent(destination); } catch { return undefined; } const terminalSegment = destination.split("/").at(-1); const usesDirectorySyntax = destination.endsWith("/") || terminalSegment === "." || terminalSegment === ".."; const normalized = destination.startsWith("/") ? normalizeGraphPath(destination.slice(1)) : normalizeGraphPath( joinGraphPath(graphPathDirname(source), destination), ); const resolved = normalized.endsWith("/") ? normalized.slice(0, -1) : normalized; if (directoryPaths.has(resolved)) { const readmePath = `${resolved}/README.md`; return filePaths.has(readmePath) ? readmePath : undefined; } if (otherFilePaths.has(resolved) || usesDirectorySyntax) { return undefined; } const candidates = graphPathExtension(resolved) === "" ? [resolved, `${resolved}.md`] : [resolved]; return candidates.find((candidate) => filePaths.has(candidate)); }; const collectDirectedEdges = async ( files: readonly MarkdownFile[], directoryPaths: ReadonlySet, otherFilePaths: ReadonlySet, ): Promise => { const filePaths = new Set(files.map(({ graphPath }) => graphPath)); const directedKeys = new Set(); for (const file of files) { const markdown = await Deno.readTextFile(file.fileSystemPath); const tokens = marked.lexer(markdown); const recordTarget = (href: string) => { const target = resolveLink( file.graphPath, href, filePaths, directoryPaths, otherFilePaths, ); if (target !== undefined) { directedKeys.add(`${file.graphPath}\0${target}`); } }; marked.walkTokens(tokens, (token: Token) => { if (token.type === "link") { recordTarget(token.href); } else if (token.type === "html") { const parser = new HtmlParser({ onopentag(name, attributes) { if (name === "a" && attributes.href !== undefined) { recordTarget(attributes.href); } }, }); parser.end(token.raw); } }); } return [...directedKeys] .map((key) => { const [source, target] = key.split("\0"); return { source, target }; }) .sort((left, right) => compareGraphPaths(left.source, right.source) || compareGraphPaths(left.target, right.target) ); }; const collapseEdges = ( directedEdges: readonly DirectedEdge[], ): RenderedEdge[] => { const directedKeys = new Set( directedEdges.map(({ source, target }) => `${source}\0${target}`), ); const emittedPairs = new Set(); const output: RenderedEdge[] = []; for (const edge of directedEdges) { const reverseKey = `${edge.target}\0${edge.source}`; if (edge.source === edge.target || !directedKeys.has(reverseKey)) { output.push({ ...edge, bidirectional: false }); continue; } const pairKey = [edge.source, edge.target] .sort(compareCodePoints) .join("\0"); if (emittedPairs.has(pairKey)) continue; emittedPairs.add(pairKey); output.push({ ...edge, bidirectional: true }); } return output; }; const nodeLine = ( file: MarkdownFile, incoming: ReadonlySet, indent: string, ): string => { if (file.name === "README.md") { const readmeLabel = `${ htmlText(graphPathBasename(graphPathDirname(file.graphPath))) }
${htmlText(file.name)}`; if (file.graphPath === "docs/README.md") { return `${indent}${ graphString(file.graphPath) } [fillcolor="lime", style="filled", label=<${readmeLabel}>];`; } return `${indent}${ graphString(file.graphPath) } [fillcolor="yellow", style="filled", label=<${readmeLabel}>];`; } if (!incoming.has(file.graphPath)) { return `${indent}${ graphString(file.graphPath) } [fillcolor="lightcoral", style="filled", label=${ graphString(file.name) }];`; } return `${indent}${graphString(file.graphPath)} [label=${ graphString(file.name) }];`; }; const renderDirectory = ( directory: DirectoryTree, incoming: ReadonlySet, depth: number, ): string[] => { const indent = " ".repeat(depth); const childIndent = " ".repeat(depth + 1); const lines = [ `${indent}subgraph ${graphString(`cluster_${directory.graphPath}`)} {`, `${childIndent}graph [label=<${ htmlText(directory.name) }>, style="rounded,filled", color=${ graphString(directoryBorderColor) }, fillcolor=${graphString(directoryBackgroundColor)}];`, ]; for (const file of directory.files) { lines.push(nodeLine(file, incoming, childIndent)); } for (const child of directory.directories) { for (const line of renderDirectory(child, incoming, depth + 1)) { lines.push(line); } } lines.push(`${indent}}`); return lines; }; const buildDocsGraph = async ( docsFileSystemPath: string, includeHistory: boolean, ): Promise => { const excludedDirectoryPaths = includeHistory ? new Set() : new Set([historyGraphPath]); const docsTree = await readDirectoryTree( docsFileSystemPath, "docs", excludedDirectoryPaths, ); const docsFiles = flattenFiles(docsTree); const directoryPaths = new Set(docsTree.directoryPaths); const otherFilePaths = new Set(docsTree.otherFilePaths); const directedEdges = await collectDirectedEdges( docsFiles, directoryPaths, otherFilePaths, ); const renderedEdges = collapseEdges(directedEdges); const incoming = new Set(directedEdges.map(({ target }) => target)); return { directoryTree: docsTree, files: docsFiles, includeHistory, directedEdges, renderedEdges, incoming, }; }; const renderDocsDot = ( graph: DocsGraph, edgeIdPrefix?: string, ): string => { const outputLines = [ "// Every scanned Markdown file under docs is a node.", graph.includeHistory ? "// docs/history is included by --history." : "// docs/history is excluded by default; pass --history to include it.", "// Every link whose resolved destination is one of those files is an edge.", "strict digraph docs {", ' graph [layout="osage", pack=48, packmode="array_i2", splines="line", fontsize=18, labeljust="l", labelloc="t"];', "", " // Links to scanned directories resolve to their README.md file.", " // Repeated links from one file to the same target appear as one edge.", " // Opposite-direction links between the same two files share one bidirectional edge.", " // Non-README files with no incoming links use a light red fill.", " // Nested README.md nodes use a yellow fill. The root README uses lime and its label is bold.", " // README labels show their directory's final component above README.md.", " // Other node labels show only the file name, without its directory path.", " // Each docs subdirectory has a light teal border and a 10%-opaque matching background, extra internal spacing, and a larger, bold label at its upper left; files directly under docs remain unclustered.", ]; for (const file of graph.directoryTree.files) { outputLines.push(nodeLine(file, graph.incoming, " ")); } outputLines.push(""); for (const directory of graph.directoryTree.directories) { for (const line of renderDirectory(directory, graph.incoming, 1)) { outputLines.push(line); } outputLines.push(""); } for (const [index, edge] of graph.renderedEdges.entries()) { const attributeValues = edge.bidirectional ? ['dir="both"', 'class="bidirectional"'] : []; if (edgeIdPrefix !== undefined) { attributeValues.push(`id=${graphString(`${edgeIdPrefix}${index}`)}`); } const attributes = attributeValues.length === 0 ? "" : ` [${attributeValues.join(", ")}]`; outputLines.push( ` ${graphString(edge.source)} -> ${ graphString(edge.target) }${attributes};`, ); } outputLines.push("}"); return `${outputLines.join("\n")}\n`; }; const renderDocsJson = (graph: DocsGraph): string => `${ JSON.stringify( { nodes: graph.files.map(({ graphPath }) => graphPath), edges: graph.directedEdges.map(({ source, target }) => ({ source, target, })), }, null, 2, ) }\n`; const renderOrphans = (graph: DocsGraph, currentDirectory: string): string => { const paths = graph.files .filter(({ graphPath }) => !graph.incoming.has(graphPath)) .map(({ fileSystemPath }) => relativeFileSystemPath(currentDirectory, fileSystemPath) ) .sort(compareCodePoints); return paths.length === 0 ? "" : `${paths.join("\n")}\n`; }; const renderDocsHtml = async ( graph: DocsGraph, dot: string, ): Promise => { // The graph renderer is needed only for HTML output. // deno-lint-ignore cf-imports/no-inline-module-import const { instance } = await import("@viz-js/viz"); const viz = await instance(); const edgeIdPrefix = "docs-edge-"; const svgDocument = viz.renderString(renderDocsDot(graph, edgeIdPrefix), { engine: "osage", format: "svg", }); const svgStart = svgDocument.indexOf(""); if (svgStart === -1 || svgEnd === -1) { throw new Error("Viz.js did not return an SVG document"); } const svg = svgDocument .slice(svgStart, svgEnd + "".length) .replace( " Docs Link Graph

Docs link graph

${graph.files.length} files · ${graph.directedEdges.length} links
Select a file to inspect its incoming and outgoing links.
${svg}
`; }; const usage = `Usage: deno run --allow-read [--allow-write=] scripts/docs-links.ts \\ (--dot | --json | --html | --orphan) [--history] [--out ] Output formats: --dot Graphviz DOT with the documentation graph styling --json JSON nodes and directed edges without presentation attributes --html Self-contained interactive HTML viewer with the DOT source inlined --orphan Paths, relative to the current directory, with no inbound links Options: --history Include docs/history in the scan --out Write to this path instead of standard output; requires matching Deno --allow-write permission --help Show this help`; const usageError = (message: string): never => { console.error(`Error: ${message}\n\n${usage}`); Deno.exit(2); }; const parseCommandLine = ( arguments_: readonly string[], ): CommandLineOptions | undefined => { const parsed = parseArgs([...arguments_], { boolean: ["dot", "json", "html", "orphan", "history", "help"], string: ["out"], alias: { h: "help", o: "out" }, }); const knownKeys = new Set([ "_", "--", "dot", "json", "html", "orphan", "history", "help", "h", "out", "o", ]); const unknown = Object.keys(parsed).find((key) => !knownKeys.has(key)); if (unknown !== undefined) usageError(`unknown option --${unknown}`); if (parsed._.length > 0) { usageError(`unexpected argument ${String(parsed._[0])}`); } if (parsed.help) { console.log(usage); return undefined; } const formats = (["dot", "json", "html", "orphan"] as const) .filter((format) => parsed[format]); if (formats.length !== 1) { usageError("choose exactly one output format"); } if (parsed.out === "") usageError("--out requires a path"); return { format: formats[0], includeHistory: parsed.history, outputPath: parsed.out, }; }; const writeOutput = async ( output: string, outputPath: string | undefined, ): Promise => { if (outputPath !== undefined) { await Deno.writeTextFile(outputPath, output); } else { const bytes = new TextEncoder().encode(output); for (let written = 0; written < bytes.length;) { written += await Deno.stdout.write(bytes.subarray(written)); } } }; if (import.meta.main) { const options = parseCommandLine(Deno.args); if (options !== undefined) { const repositoryRoot = fromFileUrl(new URL("../", import.meta.url)); const docsDirectory = joinFileSystemPath(repositoryRoot, "docs"); const graph = await buildDocsGraph( docsDirectory, options.includeHistory, ); const dot = renderDocsDot(graph); const output = options.format === "dot" ? dot : options.format === "json" ? renderDocsJson(graph) : options.format === "html" ? await renderDocsHtml(graph, dot) : renderOrphans(graph, Deno.cwd()); await writeOutput(output, options.outputPath); } }