also looking at this
fix(review): give pull request reviews the analyzers and learnings they skipped
#1415
4 files
+316
-106
| @@ -0,0 +1,229 @@ | ||
| 1 | import type { AiClient } from '../src/ai/types' | |
| 2 | import type { GitProvider } from '../src/git/provider' | |
| 3 | import type { PullRequest } from '../src/types' | |
| 4 | import type { BuddyConfig } from '../src/types' | |
| 5 | import { afterEach, beforeEach, describe, expect, it } from 'bun:test' | |
| 6 | import { promises as fs } from 'node:fs' | |
| 7 | import { tmpdir } from 'node:os' | |
| 8 | import { join } from 'node:path' | |
| 9 | import { runReviewForPR } from '../src/review/run' | |
| 10 | import { NO_CAPABILITIES } from '../src/git/provider' | |
| 11 | import { Logger } from '../src/utils/logger' | |
| 12 | ||
| 13 | const DIFF = `diff --git a/src/app.ts b/src/app.ts | |
| 14 | index 111..222 100644 | |
| 15 | --- a/src/app.ts | |
| 16 | +++ b/src/app.ts | |
| 17 | @@ -1,3 +1,4 @@ | |
| 18 | const a = 1 | |
| 19 | +const token = 'ghp_0123456789abcdefghijklmnopqrstuvwxyzAB' | |
| 20 | const b = 2 | |
| 21 | ` | |
| 22 | ||
| 23 | interface Seen { | |
| 24 | prompts: string[] | |
| 25 | reviews: Array<{ pr: number, body: string }> | |
| 26 | filesRead: Array<{ path: string, ref: string }> | |
| 27 | } | |
| 28 | ||
| 29 | /** A provider carrying one pull request, plus a record of what was read. */ | |
| 30 | function stub(files: Record<string, string> = {}): { provider: GitProvider, seen: Seen } { | |
| 31 | const seen: Seen = { prompts: [], reviews: [], filesRead: [] } | |
| 32 | ||
| 33 | const pr = { | |
| 34 | number: 7, | |
| 35 | title: 'add a thing', | |
| 36 | body: 'description', | |
| 37 | head: 'feature', | |
| 38 | base: 'main', | |
| 39 | state: 'open', | |
| 40 | labels: [], | |
| 41 | draft: false, | |
| 42 | } as unknown as PullRequest | |
| 43 | ||
| 44 | const provider = { | |
| 45 | capabilities: () => ({ ...NO_CAPABILITIES, inlineReviewComments: true }), | |
| 46 | getPullRequests: async () => [pr], | |
| 47 | getPullRequestDiff: async () => DIFF, | |
| 48 | getPullRequestHeadSha: async () => 'head1', | |
| 49 | getFileContent: async (path: string, ref: string) => { | |
| 50 | seen.filesRead.push({ path, ref }) | |
| 51 | return files[path] ?? null | |
| 52 | }, | |
| 53 | createReview: async (prNumber: number, review: { body: string }) => { | |
| 54 | seen.reviews.push({ pr: prNumber, body: review.body }) | |
| 55 | return { id: 1 } | |
| 56 | }, | |
| 57 | updatePullRequest: async () => pr, | |
| 58 | } as unknown as GitProvider | |
| 59 | ||
| 60 | return { provider, seen } | |
| 61 | } | |
| 62 | ||
| 63 | /** An AI client that records its prompt and reports nothing. */ | |
| 64 | function recordingAi(seen: Seen): AiClient { | |
| 65 | return { | |
| 66 | name: 'stub', | |
| 67 | complete: async (request: { system?: string, messages: Array<{ content: string }> }) => { | |
| 68 | seen.prompts.push(`${request.system ?? ''}\n${request.messages.map(m => m.content).join('\n')}`) | |
| 69 | return { | |
| 70 | text: JSON.stringify({ summary: 'looks fine', walkthrough: [], findings: [] }), | |
| 71 | outputTokens: 1, | |
| 72 | } | |
| 73 | }, | |
| 74 | } as unknown as AiClient | |
| 75 | } | |
| 76 | ||
| 77 | /** | |
| 78 | * `buddy review <pr>` reimplemented the pull request review that | |
| 79 | * `runReviewForPR` already did, and the two had drifted. The command skipped | |
| 80 | * analyzers outright for a pull request and never read learnings at all — and | |
| 81 | * it is the command the generated workflow runs, so the automatic review every | |
| 82 | * repository gets was the one path missing both. | |
| 83 | * | |
| 84 | * These cover the shared path both entry points now use. | |
| 85 | */ | |
| 86 | describe('reviewing a pull request', () => { | |
| 87 | const config: BuddyConfig = { | |
| 88 | repository: { provider: 'github', owner: 'o', name: 'r', baseBranch: 'main' }, | |
| 89 | } | |
| 90 | ||
| 91 | it('success case - reads learnings from the base branch', async () => { | |
| 92 | // `@buddy remember` writes these. A review that does not read them makes | |
| 93 | // the whole feature inert. | |
| 94 | const { provider, seen } = stub() | |
| 95 | ||
| 96 | await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() }) | |
| 97 | ||
| 98 | const learnings = seen.filesRead.filter(read => read.path.includes('learnings')) | |
| 99 | expect(learnings.length).toBeGreaterThan(0) | |
| 100 | expect(learnings.every(read => read.ref === 'main')).toBe(true) | |
| 101 | }) | |
| 102 | ||
| 103 | it('success case - reads guidelines from the base branch, never the PR branch', async () => { | |
| 104 | // Both are inlined into the prompt as trusted context. Reading them from | |
| 105 | // the pull request's own branch would let it rewrite the rules its code is | |
| 106 | // reviewed against. | |
| 107 | const { provider, seen } = stub() | |
| 108 | ||
| 109 | await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() }) | |
| 110 | ||
| 111 | expect(seen.filesRead.length).toBeGreaterThan(0) | |
| 112 | expect(seen.filesRead.every(read => read.ref === 'main')).toBe(true) | |
| 113 | expect(seen.filesRead.some(read => read.ref === 'feature')).toBe(false) | |
| 114 | }) | |
| 115 | ||
| 116 | it('success case - runs analyzers on a pull request', async () => { | |
| 117 | // The command skipped these entirely when given a pull request, so secret | |
| 118 | // scanning never ran on the path that reviews every pull request. | |
| 119 | const { provider, seen } = stub() | |
| 120 | ||
| 121 | await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() }) | |
| 122 | ||
| 123 | expect(seen.reviews).toHaveLength(1) | |
| 124 | }) | |
| 125 | ||
| 126 | it('success case - a dry run reports without posting', async () => { | |
| 127 | const { provider, seen } = stub() | |
| 128 | ||
| 129 | const status = await runReviewForPR({ | |
| 130 | config, | |
| 131 | provider, | |
| 132 | prNumber: 7, | |
| 133 | dryRun: true, | |
| 134 | ai: recordingAi(seen), | |
| 135 | logger: Logger.silent(), | |
| 136 | }) | |
| 137 | ||
| 138 | expect(seen.reviews).toHaveLength(0) | |
| 139 | expect(status).toContain('Would post') | |
| 140 | }) | |
| 141 | ||
| 142 | it('failure case - a missing pull request is reported, not thrown', async () => { | |
| 143 | const { provider } = stub() | |
| 144 | ||
| 145 | expect(await runReviewForPR({ config, provider, prNumber: 999, ai: null, logger: Logger.silent() })) | |
| 146 | .toContain('Could not find') | |
| 147 | }) | |
| 148 | ||
| 149 | it('success case - an explicit review ignores the automatic filters', async () => { | |
| 150 | // `--auto` is what the generated workflow passes; without it this is | |
| 151 | // someone who has already decided they want a review. | |
| 152 | const { provider, seen } = stub() | |
| 153 | ||
| 154 | await runReviewForPR({ | |
| 155 | config: { ...config, ai: { review: { draft: false } } } as BuddyConfig, | |
| 156 | provider, | |
| 157 | prNumber: 7, | |
| 158 | trigger: 'requested', | |
| 159 | ai: recordingAi(seen), | |
| 160 | logger: Logger.silent(), | |
| 161 | }) | |
| 162 | ||
| 163 | expect(seen.reviews).toHaveLength(1) | |
| 164 | }) | |
| 165 | ||
| 166 | it('success case - a recorded learning reaches the model', async () => { | |
| 167 | // Reading the file is not the point; the point is that what `@buddy | |
| 168 | // remember` wrote changes how the next review is prompted. | |
| 169 | const learning = { | |
| 170 | id: 'abc', | |
| 171 | text: 'we pin react to 17 on purpose', | |
| 172 | createdAt: '2026-01-01T00:00:00.000Z', | |
| 173 | } | |
| 174 | const { provider, seen } = stub({ '.buddy/learnings.jsonl': `${JSON.stringify(learning)}\n` }) | |
| 175 | ||
| 176 | await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() }) | |
| 177 | ||
| 178 | expect(seen.prompts.join('\n')).toContain('we pin react to 17 on purpose') | |
| 179 | }) | |
| 180 | ||
| 181 | it('success case - a guideline reaches the model', async () => { | |
| 182 | const { provider, seen } = stub({ 'CLAUDE.md': 'Never use `any`.' }) | |
| 183 | ||
| 184 | await runReviewForPR({ config, provider, prNumber: 7, ai: recordingAi(seen), logger: Logger.silent() }) | |
| 185 | ||
| 186 | expect(seen.prompts.join('\n')).toContain('Never use `any`') | |
| 187 | }) | |
| 188 | }) | |
| 189 | ||
| 190 | /** | |
| 191 | * Analyzers were skipped outright when the command was given a pull request: | |
| 192 | * `config.analysis?.enabled === false || prNumber`. Secret scanning therefore | |
| 193 | * never ran on the path that reviews every pull request. | |
| 194 | */ | |
| 195 | describe('analyzers on a pull request', () => { | |
| 196 | let testDir: string | |
| 197 | let originalCwd: string | |
| 198 | ||
| 199 | beforeEach(async () => { | |
| 200 | testDir = await fs.mkdtemp(join(tmpdir(), 'buddy-review-')) | |
| 201 | originalCwd = process.cwd() | |
| 202 | process.chdir(testDir) | |
| 203 | }) | |
| 204 | ||
| 205 | afterEach(async () => { | |
| 206 | process.chdir(originalCwd) | |
| 207 | await fs.rm(testDir, { recursive: true, force: true }) | |
| 208 | }) | |
| 209 | ||
| 210 | it('success case - a planted credential is found and reported', async () => { | |
| 211 | // Analyzers read the working tree rather than the diff, so the file has to | |
| 212 | // exist for this to mean anything. | |
| 213 | await fs.mkdir('src', { recursive: true }) | |
| 214 | await fs.writeFile('src/app.ts', `const token = 'ghp_0123456789abcdefghijklmnopqrstuvwxyzAB'\n`) | |
| 215 | ||
| 216 | const { provider, seen } = stub() | |
| 217 | const config: BuddyConfig = { | |
| 218 | repository: { provider: 'github', owner: 'o', name: 'r', baseBranch: 'main' }, | |
| 219 | } | |
| 220 | ||
| 221 | // No AI at all: whatever lands comes from static analysis alone, which is | |
| 222 | // the guarantee that matters for a repository without a key. | |
| 223 | const status = await runReviewForPR({ config, provider, prNumber: 7, ai: null, logger: Logger.silent() }) | |
| 224 | ||
| 225 | expect(status).toContain('static-analysis') | |
| 226 | expect(seen.reviews).toHaveLength(1) | |
| 227 | expect(seen.reviews[0].body).toContain('Static analysis only') | |
| 228 | }) | |
| 229 | }) | |