feat(ci): check the docs against the source they describe

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

.github/workflows/ci.ymlmodified+27-1

Changes to .github/workflows/ci.yml

		⋯83 linesShow the 83 lines above this hunk@@ -84,8 +84,34 @@ jobs:
84	84	- name: Unit Test
85	85	run: bun test --timeout 300000
86	86
	87	+ check-docs:
	88	+ runs-on: ubuntu-latest
	89	+
	90	+ steps:
	91	+ - uses: actions/checkout@v6
	92	+
	93	+ - name: Install Bun
	94	+ uses: oven-sh/setup-bun@v2.2.0
	95	+
	96	+ - name: Use cached node_modules
	97	+ uses: actions/cache@v5.0.4
	98	+ with:
	99	+ path: node_modules
	100	+ key: node-modules-${{ hashFiles('**/bun.lock') }}
	101	+ restore-keys: |
	102	+ node-modules-
	103	+
	104	+ - name: Install Dependencies
	105	+ run: bun install
	106	+
	107	+ # Deliberately part of CI rather than the Docs workflow: that one is
	108	+ # path-filtered on `docs/`, so a change under `src/` that invalidates a
	109	+ # documented flag would never re-check the pages describing it.
	110	+ - name: Check docs against source
	111	+ run: bun run check:docs
	112	+
87	113	publish-commit:
88		- needs: [lint, typecheck, test]
	114	+ needs: [lint, typecheck, test, check-docs]
89	115	runs-on: ubuntu-latest
90	116
91	117	steps:

package.jsonmodified+1-0

Changes to package.json

		⋯66 linesShow the 66 lines above this hunk@@ -67,6 +67,7 @@
67	67	"changelog:generate": "bunx logsmith --output CHANGELOG.md",
68	68	"release": "bunx --bun bumpx -r",
69	69	"typecheck": "bunx tsc --noEmit",
	70	+ "check:docs": "bun scripts/check-docs-claims.ts",
70	71	"dev:docs": "bun --bun bunpress dev",
71	72	"build:docs": "bun --bun bunpress build",
72	73	"preview:docs": "bun --bun bunpress preview --outdir docs/dist",

scripts/check-docs-claims.tsadded+283-0

Changes to scripts/check-docs-claims.ts

		@@ -0,0 +1,283 @@
	1	+#!/usr/bin/env bun
	2	+/**
	3	+ * Check the documentation against the code it documents.
	4	+ *
	5	+ * The docs drifted badly once: commands that were never registered, flags that
	6	+ * `clapp` hard-errors on, and config keys invented wholesale. Nothing caught it
	7	+ * because the docs workflow is path-filtered on `docs/`, so a change under
	8	+ * `src/` never rechecks the pages describing it. This script closes that gap by
	9	+ * deriving the truth from source and failing when a page disagrees.
	10	+ *
	11	+ * Two checks run:
	12	+ *
	13	+ * - **CLI surface** — every `buddy <command> [--flags]` written in the docs must
	14	+ * resolve to a command registered in `bin/cli.ts`, with flags that command
	15	+ * actually declares.
	16	+ * - **Config keys** — every key in a Buddy config example must exist somewhere
	17	+ * in `BuddyConfig` (`src/types.ts`).
	18	+ *
	19	+ * Both checks read the truth out of source at run time, so they cannot go stale
	20	+ * the way a hand-maintained list would.
	21	+ *
	22	+ * Known blind spot: a fence that *declares* types is skipped, because a
	23	+ * documented `interface` is describing a shape rather than using one. That
	24	+ * means an invented interface in an API reference page passes. Checking a
	25	+ * documented type against a real one is a different problem than checking a
	26	+ * config literal, and this script does not attempt it.
	27	+ *
	28	+ * @example
	29	+ * ```sh
	30	+ * bun run check:docs
	31	+ * ```
	32	+ */
	33	+import { Glob } from 'bun'
	34	+
	35	+const ROOT = new URL('..', import.meta.url).pathname
	36	+
	37	+/** A documentation claim that source does not support. */
	38	+interface Problem {
	39	+ file: string
	40	+ line: number
	41	+ message: string
	42	+}
	43	+
	44	+/** Flags every command accepts, whether or not it declares them. */
	45	+const GLOBAL_FLAGS = ['--config', '--help', '-h', '--version']
	46	+
	47	+/**
	48	+ * Words that follow "buddy" in prose often enough to be worth ignoring, so a
	49	+ * sentence like "buddy scans your repo" is not read as a command invocation.
	50	+ */
	51	+const PROSE_AFTER_BUDDY = new Set([
	52	+ 'bot', 'is', 'can', 'will', 'handles', 'uses', 'scans', 'runs', 'does',
	53	+ 'and', 'or', 'to', 'the', 'a', 'as', 'in', 'on', 'for', 'with', 'from',
	54	+ 'creates', 'opens', 'posts', 'reads', 'writes', 'needs', 'supports',
	55	+])
	56	+
	57	+/**
	58	+ * Read the real command surface out of `bin/cli.ts`.
	59	+ *
	60	+ * Options are attached to the most recent `.command()` above them, which is how
	61	+ * the chained registrations in `bin/cli.ts` are actually structured.
	62	+ *
	63	+ * @returns Command name to the set of flags it accepts
	64	+ */
	65	+async function readCliSurface(): Promise<Map<string, Set<string>>> {
	66	+ const lines = (await Bun.file(`${ROOT}bin/cli.ts`).text()).split('\n')
	67	+ const commands = new Map<string, Set<string>>()
	68	+ const marks: Array<{ line: number, name: string }> = []
	69	+
	70	+ lines.forEach((line, i) => {
	71	+ const match = line.match(/^\s*\.command\((["'])(.+?)\1\s*,/)
	72	+ if (!match)
	73	+ return
	74	+
	75	+ const name = match[2].split(' ')[0]
	76	+ marks.push({ line: i, name })
	77	+ if (!commands.has(name))
	78	+ commands.set(name, new Set(GLOBAL_FLAGS))
	79	+ })
	80	+
	81	+ lines.forEach((line, i) => {
	82	+ const match = line.match(/^\s*\.option\((["'])(.+?)\1\s*,/)
	83	+ if (!match)
	84	+ return
	85	+
	86	+ let owner: string | null = null
	87	+ for (const mark of marks) {
	88	+ if (mark.line < i)
	89	+ owner = mark.name
	90	+ }
	91	+ if (!owner)
	92	+ return
	93	+
	94	+ // `'--verbose, -v'` and `'-f, --format <format>'` both declare two spellings.
	95	+ for (const token of match[2].split(',').map(part => part.trim().split(/[ <[]/)[0]).filter(Boolean))
	96	+ commands.get(owner)!.add(token)
	97	+ })
	98	+
	99	+ return commands
	100	+}
	101	+
	102	+/**
	103	+ * Files that declare the config vocabulary.
	104	+ *
	105	+ * `PackageRule` lives with the engine that consumes it rather than in
	106	+ * `types.ts`, so a rule key like `groupName` is only valid because of the
	107	+ * second entry here.
	108	+ */
	109	+const CONFIG_TYPE_FILES = ['src/types.ts', 'src/rules/engine.ts']
	110	+
	111	+/**
	112	+ * Read every property name declared by the config types.
	113	+ *
	114	+ * Flattened deliberately: `BuddyConfig` nests inline object types many levels
	115	+ * deep, and a flat vocabulary catches an invented key wherever it appears
	116	+ * without needing to resolve the shape it sits in. The trade is that a key
	117	+ * valid in one branch of the config is accepted in every branch — this check
	118	+ * exists to catch wholesale invention, not misplacement.
	119	+ *
	120	+ * @returns Every property name the config types declare
	121	+ */
	122	+async function readConfigKeys(): Promise<{ keys: Set<string>, openMaps: Set<string> }> {
	123	+ const keys = new Set<string>()
	124	+ const openMaps = new Set<string>()
	125	+
	126	+ for (const file of CONFIG_TYPE_FILES) {
	127	+ const text = await Bun.file(`${ROOT}${file}`).text()
	128	+
	129	+ for (const [, , key] of text.matchAll(/^\s*(?:readonly\s+)?(['"]?)([A-Za-z_$][\w$]*)\1\??\s*:/gm))
	130	+ keys.add(key)
	131	+
	132	+ // A `Record<string, …>` is keyed by user data — package names, scopes —
	133	+ // so its children are values, not schema keys, and must not be checked.
	134	+ for (const [, , key] of text.matchAll(/^\s*(['"]?)([A-Za-z_$][\w$]*)\1\??\s*:\s*Record</gm))
	135	+ openMaps.add(key)
	136	+ }
	137	+
	138	+ return { keys, openMaps }
	139	+}
	140	+
	141	+/**
	142	+ * Check every `buddy …` invocation written in the docs.
	143	+ *
	144	+ * @param files - Markdown files to scan
	145	+ * @param commands - The real CLI surface
	146	+ * @returns Problems found
	147	+ */
	148	+function checkCliClaims(files: Array<{ path: string, text: string }>, commands: Map<string, Set<string>>): Problem[] {
	149	+ const problems: Problem[] = []
	150	+
	151	+ for (const { path, text } of files) {
	152	+ text.split('\n').forEach((line, i) => {
	153	+ // An invocation, not a prose mention: line start, a shell prompt, or a
	154	+ // backtick immediately before the word.
	155	+ const match = line.match(/(?:^|\$\s*|`)buddy\s+([a-z][a-z-]*)((?:\s+[^`\n]*)?)/)
	156	+ if (!match)
	157	+ return
	158	+
	159	+ const [, name, rest = ''] = match
	160	+ const flags = commands.get(name)
	161	+
	162	+ if (!flags) {
	163	+ if (!PROSE_AFTER_BUDDY.has(name))
	164	+ problems.push({ file: path, line: i + 1, message: `unknown command: buddy ${name}` })
	165	+ return
	166	+ }
	167	+
	168	+ for (const [, flag] of rest.matchAll(/(?:^|\s)(--?[a-z][a-z-]*)/g)) {
	169	+ if (!flags.has(flag) && !flags.has(`--no-${flag.replace(/^--/, '')}`))
	170	+ problems.push({ file: path, line: i + 1, message: `buddy ${name}: unknown flag ${flag}` })
	171	+ }
	172	+ })
	173	+ }
	174	+
	175	+ return problems
	176	+}
	177	+
	178	+/**
	179	+ * Check config keys used in documentation examples.
	180	+ *
	181	+ * Only fenced blocks that identify themselves as Buddy config are scanned —
	182	+ * either by naming `BuddyConfig` or by sitting in a file whose whole subject is
	183	+ * configuration — so an unrelated snippet cannot raise a false alarm.
	184	+ *
	185	+ * @param files - Markdown files to scan
	186	+ * @param known - Every property name the config types declare
	187	+ * @returns Problems found
	188	+ */
	189	+function checkConfigClaims(
	190	+ files: Array<{ path: string, text: string }>,
	191	+ known: Set<string>,
	192	+ openMaps: Set<string>,
	193	+): Problem[] {
	194	+ const problems: Problem[] = []
	195	+ const configPages = /docs\/(config|api\/configuration)\.md$/
	196	+ const codeLang = /^(ts|typescript|js|javascript|json)$/
	197	+
	198	+ for (const { path, text } of files) {
	199	+ const lines = text.split('\n')
	200	+ let fenceStart = -1
	201	+ let fenceLang = ''
	202	+
	203	+ lines.forEach((line, i) => {
	204	+ const fence = line.match(/^```(\w*)/)
	205	+ if (!fence)
	206	+ return
	207	+
	208	+ if (fenceStart === -1) {
	209	+ fenceStart = i
	210	+ fenceLang = fence[1]
	211	+ return
	212	+ }
	213	+
	214	+ const body = lines.slice(fenceStart + 1, i)
	215	+ const source = body.join('\n')
	216	+
	217	+ // A fence that declares types is describing the schema, not using it.
	218	+ const declaresTypes = /^\s*(?:export\s+)?(?:interface|type)\s/m.test(source)
	219	+ const isConfig = codeLang.test(fenceLang)
	220	+ && !declaresTypes
	221	+ && (/BuddyConfig|buddy\.config/.test(source) || configPages.test(path))
	222	+
	223	+ if (isConfig) {
	224	+ // Track nesting so the children of an open map can be skipped.
	225	+ const stack: Array<{ indent: number, key: string }> = []
	226	+
	227	+ body.forEach((bodyLine, offset) => {
	228	+ const match = bodyLine.match(/^(\s*)(['"]?)([A-Za-z_$][\w$@/-]*)\2\s*:/)
	229	+ if (!match)
	230	+ return
	231	+
	232	+ const [, indent, , key] = match
	233	+ while (stack.length > 0 && stack[stack.length - 1].indent >= indent.length)
	234	+ stack.pop()
	235	+
	236	+ const insideOpenMap = stack.some(entry => openMaps.has(entry.key))
	237	+ if (!insideOpenMap && !known.has(key))
	238	+ problems.push({ file: path, line: fenceStart + offset + 2, message: `unknown config key: ${key}` })
	239	+
	240	+ if (bodyLine.trimEnd().endsWith('{'))
	241	+ stack.push({ indent: indent.length, key })
	242	+ })
	243	+ }
	244	+
	245	+ fenceStart = -1
	246	+ fenceLang = ''
	247	+ })
	248	+ }
	249	+
	250	+ return problems
	251	+}
	252	+
	253	+const [commands, { keys: configKeys, openMaps }] = await Promise.all([readCliSurface(), readConfigKeys()])
	254	+
	255	+const files: Array<{ path: string, text: string }> = []
	256	+for (const pattern of ['README.md', 'CLAUDE.md', 'docs/**/*.md']) {
	257	+ for await (const path of new Glob(pattern).scan(ROOT))
	258	+ files.push({ path, text: await Bun.file(`${ROOT}${path}`).text() })
	259	+}
	260	+
	261	+const problems = [
	262	+ ...checkCliClaims(files, commands),
	263	+ ...checkConfigClaims(files, configKeys, openMaps),
	264	+]
	265	+
	266	+if (problems.length === 0) {
	267	+ console.log(`✓ ${files.length} pages agree with ${commands.size} commands and ${configKeys.size} config keys`)
	268	+ process.exit(0)
	269	+}
	270	+
	271	+const byFile = new Map<string, Problem[]>()
	272	+for (const problem of problems)
	273	+ byFile.set(problem.file, [...(byFile.get(problem.file) ?? []), problem])
	274	+
	275	+for (const [file, found] of [...byFile].sort()) {
	276	+ console.error(`\n${file}`)
	277	+ for (const { line, message } of found)
	278	+ console.error(` ${line}: ${message}`)
	279	+}
	280	+
	281	+console.error(`\n✗ ${problems.length} claim(s) in ${byFile.size} file(s) do not match the source.`)
	282	+console.error(' Fix the documentation, or the code it describes — do not silence this check.')
	283	+process.exit(1)