ReviewOS

also looking at this

stacks/buddy

fix(review): give pull request reviews the analyzers and learnings they skipped

#1415
Merged glennmichael123 wants to merge fix/review-parity into main
4 files +316 -106
test/review-parity.test.tsadded+229-0
Changes to test/review-parity.test.ts
@@ -0,0 +1,229 @@
1import type { AiClient } from '../src/ai/types'
2import type { GitProvider } from '../src/git/provider'
3import type { PullRequest } from '../src/types'
4import type { BuddyConfig } from '../src/types'
5import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
6import { promises as fs } from 'node:fs'
7import { tmpdir } from 'node:os'
8import { join } from 'node:path'
9import { runReviewForPR } from '../src/review/run'
10import { NO_CAPABILITIES } from '../src/git/provider'
11import { Logger } from '../src/utils/logger'
12
13const DIFF = `diff --git a/src/app.ts b/src/app.ts
14index 111..222 100644
15--- a/src/app.ts
16+++ b/src/app.ts
17@@ -1,3 +1,4 @@
18 const a = 1
19+const token = 'ghp_0123456789abcdefghijklmnopqrstuvwxyzAB'
20 const b = 2
21`
22
23interface Seen {
24 prompts: string[]
25 reviews: Array<{ pr: number, body: string }>
26 filesRead: Array<{ path: string, ref: string }>
27}
28
29/** A provider carrying one pull request, plus a record of what was read. */
30function stub(files: Record<string, string> = {}): { provider: GitProvider, seen: Seen } {
31 const seen: Seen = { prompts: [], reviews: [], filesRead: [] }
32
33 const pr = {
34 number: 7,
35 title: 'add a thing',
36 body: 'description',
37 head: 'feature',
38 base: 'main',
39 state: 'open',
40 labels: [],
41 draft: false,
42 } as unknown as PullRequest
43
44 const provider = {
45 capabilities: () => ({ ...NO_CAPABILITIES, inlineReviewComments: true }),
46 getPullRequests: async () => [pr],
47 getPullRequestDiff: async () => DIFF,
48 getPullRequestHeadSha: async () => 'head1',
49 getFileContent: async (path: string, ref: string) => {
50 seen.filesRead.push({ path, ref })
51 return files[path] ?? null
52 },
53 createReview: async (prNumber: number, review: { body: string }) => {
54 seen.reviews.push({ pr: prNumber, body: review.body })
55 return { id: 1 }
56 },
57 updatePullRequest: async () => pr,
58 } as unknown as GitProvider
59
60 return { provider, seen }
61}
62
63/** An AI client that records its prompt and reports nothing. */
64function recordingAi(seen: Seen): AiClient {
65 return {
66 name: 'stub',
67 complete: async (request: { system?: string, messages: Array<{ content: string }> }) => {
68 seen.prompts.push(`${request.system ?? ''}\n${request.messages.map(m => m.content).join('\n')}`)
69 return {
70 text: JSON.stringify({ summary: 'looks fine', walkthrough: [], findings: [] }),
71 outputTokens: 1,
72 }
73 },
74 } as unknown as AiClient
75}
76
77/**
78 * `buddy review <pr>` reimplemented the pull request review that
79 * `runReviewForPR` already did, and the two had drifted. The command skipped
80 * analyzers outright for a pull request and never read learnings at all — and
81 * it is the command the generated workflow runs, so the automatic review every
82 * repository gets was the one path missing both.
83 *
84 * These cover the shared path both entry points now use.
85 */
86describe('reviewing a pull request', () => {
87 const config: BuddyConfig = {
88 repository: { provider: 'github', owner: 'o', name: 'r', baseBranch: 'main' },
89 }
90
91 it('success case - reads learnings from the base branch', async () => {
92 // `@buddy remember` writes these. A review that does not read them makes
93 // the whole feature inert.
94 const { provider, seen } = stub()
95
96 await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() })
97
98 const learnings = seen.filesRead.filter(read => read.path.includes('learnings'))
99 expect(learnings.length).toBeGreaterThan(0)
100 expect(learnings.every(read => read.ref === 'main')).toBe(true)
101 })
102
103 it('success case - reads guidelines from the base branch, never the PR branch', async () => {
104 // Both are inlined into the prompt as trusted context. Reading them from
105 // the pull request's own branch would let it rewrite the rules its code is
106 // reviewed against.
107 const { provider, seen } = stub()
108
109 await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() })
110
111 expect(seen.filesRead.length).toBeGreaterThan(0)
112 expect(seen.filesRead.every(read => read.ref === 'main')).toBe(true)
113 expect(seen.filesRead.some(read => read.ref === 'feature')).toBe(false)
114 })
115
116 it('success case - runs analyzers on a pull request', async () => {
117 // The command skipped these entirely when given a pull request, so secret
118 // scanning never ran on the path that reviews every pull request.
119 const { provider, seen } = stub()
120
121 await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() })
122
123 expect(seen.reviews).toHaveLength(1)
124 })
125
126 it('success case - a dry run reports without posting', async () => {
127 const { provider, seen } = stub()
128
129 const status = await runReviewForPR({
130 config,
131 provider,
132 prNumber: 7,
133 dryRun: true,
134 ai: recordingAi(seen),
135 logger: Logger.silent(),
136 })
137
138 expect(seen.reviews).toHaveLength(0)
139 expect(status).toContain('Would post')
140 })
141
142 it('failure case - a missing pull request is reported, not thrown', async () => {
143 const { provider } = stub()
144
145 expect(await runReviewForPR({ config, provider, prNumber: 999, ai: null, logger: Logger.silent() }))
146 .toContain('Could not find')
147 })
148
149 it('success case - an explicit review ignores the automatic filters', async () => {
150 // `--auto` is what the generated workflow passes; without it this is
151 // someone who has already decided they want a review.
152 const { provider, seen } = stub()
153
154 await runReviewForPR({
155 config: { ...config, ai: { review: { draft: false } } } as BuddyConfig,
156 provider,
157 prNumber: 7,
158 trigger: 'requested',
159 ai: recordingAi(seen),
160 logger: Logger.silent(),
161 })
162
163 expect(seen.reviews).toHaveLength(1)
164 })
165
166 it('success case - a recorded learning reaches the model', async () => {
167 // Reading the file is not the point; the point is that what `@buddy
168 // remember` wrote changes how the next review is prompted.
169 const learning = {
170 id: 'abc',
171 text: 'we pin react to 17 on purpose',
172 createdAt: '2026-01-01T00:00:00.000Z',
173 }
174 const { provider, seen } = stub({ '.buddy/learnings.jsonl': `${JSON.stringify(learning)}\n` })
175
176 await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() })
177
178 expect(seen.prompts.join('\n')).toContain('we pin react to 17 on purpose')
179 })
180
181 it('success case - a guideline reaches the model', async () => {
182 const { provider, seen } = stub({ 'CLAUDE.md': 'Never use `any`.' })
183
184 await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() })
185
186 expect(seen.prompts.join('\n')).toContain('Never use `any`')
187 })
188})
189
190/**
191 * Analyzers were skipped outright when the command was given a pull request:
192 * `config.analysis?.enabled === false || prNumber`. Secret scanning therefore
193 * never ran on the path that reviews every pull request.
194 */
195describe('analyzers on a pull request', () => {
196 let testDir: string
197 let originalCwd: string
198
199 beforeEach(async () => {
200 testDir = await fs.mkdtemp(join(tmpdir(), 'buddy-review-'))
201 originalCwd = process.cwd()
202 process.chdir(testDir)
203 })
204
205 afterEach(async () => {
206 process.chdir(originalCwd)
207 await fs.rm(testDir, { recursive: true, force: true })
208 })
209
210 it('success case - a planted credential is found and reported', async () => {
211 // Analyzers read the working tree rather than the diff, so the file has to
212 // exist for this to mean anything.
213 await fs.mkdir('src', { recursive: true })
214 await fs.writeFile('src/app.ts', `const token = 'ghp_0123456789abcdefghijklmnopqrstuvwxyzAB'\n`)
215
216 const { provider, seen } = stub()
217 const config: BuddyConfig = {
218 repository: { provider: 'github', owner: 'o', name: 'r', baseBranch: 'main' },
219 }
220
221 // No AI at all: whatever lands comes from static analysis alone, which is
222 // the guarantee that matters for a repository without a key.
223 const status = await runReviewForPR({ config, provider, prNumber: 7, ai: null, logger: Logger.silent() })
224
225 expect(status).toContain('static-analysis')
226 expect(seen.reviews).toHaveLength(1)
227 expect(seen.reviews[0].body).toContain('Static analysis only')
228 })
229})