ReviewOS
Reload

also looking at this

stacks/buddy

feat(ci): check the docs against the source they describe

#1403
Merged glennmichael123 wants to merge ci/check-docs-against-source into main
3 files +311 -1
scripts/check-docs-claims.tsadded+283-0
@@ -0,0 +1,283 @@
1#!/usr/bin/env bun
2/**
3 * Check the documentation against the code it documents.
4 *
5 * The docs drifted badly once: commands that were never registered, flags that
6 * `clapp` hard-errors on, and config keys invented wholesale. Nothing caught it
7 * because the docs workflow is path-filtered on `docs/`, so a change under
8 * `src/` never rechecks the pages describing it. This script closes that gap by
9 * deriving the truth from source and failing when a page disagrees.
10 *
11 * Two checks run:
12 *
13 * - **CLI surface** — every `buddy <command> [--flags]` written in the docs must
14 * resolve to a command registered in `bin/cli.ts`, with flags that command
15 * actually declares.
16 * - **Config keys** — every key in a Buddy config example must exist somewhere
17 * in `BuddyConfig` (`src/types.ts`).
18 *
19 * Both checks read the truth out of source at run time, so they cannot go stale
20 * the way a hand-maintained list would.
21 *
22 * Known blind spot: a fence that *declares* types is skipped, because a
23 * documented `interface` is describing a shape rather than using one. That
24 * means an invented interface in an API reference page passes. Checking a
25 * documented type against a real one is a different problem than checking a
26 * config literal, and this script does not attempt it.
27 *
28 * @example
29 * ```sh
30 * bun run check:docs
31 * ```
32 */
33import { Glob } from 'bun'
34
35const ROOT = new URL('..', import.meta.url).pathname
36
37/** A documentation claim that source does not support. */
38interface Problem {
39 file: string
40 line: number
41 message: string
42}
43
44/** Flags every command accepts, whether or not it declares them. */
45const GLOBAL_FLAGS = ['--config', '--help', '-h', '--version']
46
47/**
48 * Words that follow "buddy" in prose often enough to be worth ignoring, so a
49 * sentence like "buddy scans your repo" is not read as a command invocation.
50 */
51const PROSE_AFTER_BUDDY = new Set([
52 'bot', 'is', 'can', 'will', 'handles', 'uses', 'scans', 'runs', 'does',
53 'and', 'or', 'to', 'the', 'a', 'as', 'in', 'on', 'for', 'with', 'from',
54 'creates', 'opens', 'posts', 'reads', 'writes', 'needs', 'supports',
55])
56
57/**
58 * Read the real command surface out of `bin/cli.ts`.
59 *
60 * Options are attached to the most recent `.command()` above them, which is how
61 * the chained registrations in `bin/cli.ts` are actually structured.
62 *
63 * @returns Command name to the set of flags it accepts
64 */
65async function readCliSurface(): Promise<Map<string, Set<string>>> {
66 const lines = (await Bun.file(`${ROOT}bin/cli.ts`).text()).split('\n')
67 const commands = new Map<string, Set<string>>()
68 const marks: Array<{ line: number, name: string }> = []
69
70 lines.forEach((line, i) => {
71 const match = line.match(/^\s*\.command\((["'])(.+?)\1\s*,/)
72 if (!match)
73 return
74
75 const name = match[2].split(' ')[0]
76 marks.push({ line: i, name })
77 if (!commands.has(name))
78 commands.set(name, new Set(GLOBAL_FLAGS))
79 })
80
81 lines.forEach((line, i) => {
82 const match = line.match(/^\s*\.option\((["'])(.+?)\1\s*,/)
83 if (!match)
84 return
85
86 let owner: string | null = null
87 for (const mark of marks) {
88 if (mark.line < i)
89 owner = mark.name
90 }
91 if (!owner)
92 return
93
94 // `'--verbose, -v'` and `'-f, --format <format>'` both declare two spellings.
95 for (const token of match[2].split(',').map(part => part.trim().split(/[ <[]/)[0]).filter(Boolean))
96 commands.get(owner)!.add(token)
97 })
98
99 return commands
100}
101
102/**
103 * Files that declare the config vocabulary.
104 *
105 * `PackageRule` lives with the engine that consumes it rather than in
106 * `types.ts`, so a rule key like `groupName` is only valid because of the
107 * second entry here.
108 */
109const CONFIG_TYPE_FILES = ['src/types.ts', 'src/rules/engine.ts']
110
111/**
112 * Read every property name declared by the config types.
113 *
114 * Flattened deliberately: `BuddyConfig` nests inline object types many levels
115 * deep, and a flat vocabulary catches an invented key wherever it appears
116 * without needing to resolve the shape it sits in. The trade is that a key
117 * valid in one branch of the config is accepted in every branch — this check
118 * exists to catch wholesale invention, not misplacement.
119 *
120 * @returns Every property name the config types declare
121 */
122async function readConfigKeys(): Promise<{ keys: Set<string>, openMaps: Set<string> }> {
123 const keys = new Set<string>()
124 const openMaps = new Set<string>()
125
126 for (const file of CONFIG_TYPE_FILES) {
127 const text = await Bun.file(`${ROOT}${file}`).text()
128
129 for (const [, , key] of text.matchAll(/^\s*(?:readonly\s+)?(['"]?)([A-Za-z_$][\w$]*)\1\??\s*:/gm))
130 keys.add(key)
131
132 // A `Record<string, …>` is keyed by user data — package names, scopes —
133 // so its children are values, not schema keys, and must not be checked.
134 for (const [, , key] of text.matchAll(/^\s*(['"]?)([A-Za-z_$][\w$]*)\1\??\s*:\s*Record</gm))
135 openMaps.add(key)
136 }
137
138 return { keys, openMaps }
139}
140
141/**
142 * Check every `buddy …` invocation written in the docs.
143 *
144 * @param files - Markdown files to scan
145 * @param commands - The real CLI surface
146 * @returns Problems found
147 */
148function checkCliClaims(files: Array<{ path: string, text: string }>, commands: Map<string, Set<string>>): Problem[] {
149 const problems: Problem[] = []
150
151 for (const { path, text } of files) {
152 text.split('\n').forEach((line, i) => {
153 // An invocation, not a prose mention: line start, a shell prompt, or a
154 // backtick immediately before the word.
155 const match = line.match(/(?:^|\$\s*|`)buddy\s+([a-z][a-z-]*)((?:\s+[^`\n]*)?)/)
156 if (!match)
157 return
158
159 const [, name, rest = ''] = match
160 const flags = commands.get(name)
161
162 if (!flags) {
163 if (!PROSE_AFTER_BUDDY.has(name))
164 problems.push({ file: path, line: i + 1, message: `unknown command: buddy ${name}` })
165 return
166 }
167
168 for (const [, flag] of rest.matchAll(/(?:^|\s)(--?[a-z][a-z-]*)/g)) {
169 if (!flags.has(flag) && !flags.has(`--no-${flag.replace(/^--/, '')}`))
170 problems.push({ file: path, line: i + 1, message: `buddy ${name}: unknown flag ${flag}` })
171 }
172 })
173 }
174
175 return problems
176}
177
178/**
179 * Check config keys used in documentation examples.
180 *
181 * Only fenced blocks that identify themselves as Buddy config are scanned —
182 * either by naming `BuddyConfig` or by sitting in a file whose whole subject is
183 * configuration — so an unrelated snippet cannot raise a false alarm.
184 *
185 * @param files - Markdown files to scan
186 * @param known - Every property name the config types declare
187 * @returns Problems found
188 */
189function checkConfigClaims(
190 files: Array<{ path: string, text: string }>,
191 known: Set<string>,
192 openMaps: Set<string>,
193): Problem[] {
194 const problems: Problem[] = []
195 const configPages = /docs\/(config|api\/configuration)\.md$/
196 const codeLang = /^(ts|typescript|js|javascript|json)$/
197
198 for (const { path, text } of files) {
199 const lines = text.split('\n')
200 let fenceStart = -1
201 let fenceLang = ''
202
203 lines.forEach((line, i) => {
204 const fence = line.match(/^```(\w*)/)
205 if (!fence)
206 return
207
208 if (fenceStart === -1) {
209 fenceStart = i
210 fenceLang = fence[1]
211 return
212 }
213
214 const body = lines.slice(fenceStart + 1, i)
215 const source = body.join('\n')
216
217 // A fence that declares types is describing the schema, not using it.
218 const declaresTypes = /^\s*(?:export\s+)?(?:interface|type)\s/m.test(source)
219 const isConfig = codeLang.test(fenceLang)
220 && !declaresTypes
221 && (/BuddyConfig|buddy\.config/.test(source) || configPages.test(path))
222
223 if (isConfig) {
224 // Track nesting so the children of an open map can be skipped.
225 const stack: Array<{ indent: number, key: string }> = []
226
227 body.forEach((bodyLine, offset) => {
228 const match = bodyLine.match(/^(\s*)(['"]?)([A-Za-z_$][\w$@/-]*)\2\s*:/)
229 if (!match)
230 return
231
232 const [, indent, , key] = match
233 while (stack.length > 0 && stack[stack.length - 1].indent >= indent.length)
234 stack.pop()
235
236 const insideOpenMap = stack.some(entry => openMaps.has(entry.key))
237 if (!insideOpenMap && !known.has(key))
238 problems.push({ file: path, line: fenceStart + offset + 2, message: `unknown config key: ${key}` })
239
240 if (bodyLine.trimEnd().endsWith('{'))
241 stack.push({ indent: indent.length, key })
242 })
243 }
244
245 fenceStart = -1
246 fenceLang = ''
247 })
248 }
249
250 return problems
251}
252
253const [commands, { keys: configKeys, openMaps }] = await Promise.all([readCliSurface(), readConfigKeys()])
254
255const files: Array<{ path: string, text: string }> = []
256for (const pattern of ['README.md', 'CLAUDE.md', 'docs/**/*.md']) {
257 for await (const path of new Glob(pattern).scan(ROOT))
258 files.push({ path, text: await Bun.file(`${ROOT}${path}`).text() })
259}
260
261const problems = [
262 ...checkCliClaims(files, commands),
263 ...checkConfigClaims(files, configKeys, openMaps),
264]
265
266if (problems.length === 0) {
267 console.log(`✓ ${files.length} pages agree with ${commands.size} commands and ${configKeys.size} config keys`)
268 process.exit(0)
269}
270
271const byFile = new Map<string, Problem[]>()
272for (const problem of problems)
273 byFile.set(problem.file, [...(byFile.get(problem.file) ?? []), problem])
274
275for (const [file, found] of [...byFile].sort()) {
276 console.error(`\n${file}`)
277 for (const { line, message } of found)
278 console.error(` ${line}: ${message}`)
279}
280
281console.error(`\n✗ ${problems.length} claim(s) in ${byFile.size} file(s) do not match the source.`)
282console.error(' Fix the documentation, or the code it describes — do not silence this check.')
283process.exit(1)