ReviewOS

also looking at this

stacks/buddy

fix(review): read what publishing reported, honour reviewSuggestions, and un-strand two flags

#1435
Merged glennmichael123 wants to merge fix/review-low-tier into main
5 files +267 -10
test/review-outcomes.test.tsadded+191-0
Changes to test/review-outcomes.test.ts
@@ -0,0 +1,191 @@
1import type { AiClient } from '../src/ai/types'
2import type { GitProvider, ReviewSubmission, ReviewSubmissionResult } from '../src/git/provider'
3import type { BuddyConfig, PullRequest } from '../src/types'
4import { describe, expect, it } from 'bun:test'
5import { NO_CAPABILITIES } from '../src/git/provider'
6import { reviewPullRequest } from '../src/review/run'
7import { Logger } from '../src/utils/logger'
8
9const DIFF = `diff --git a/src/app.ts b/src/app.ts
10index 111..222 100644
11--- a/src/app.ts
12+++ b/src/app.ts
13@@ -1,2 +1,4 @@
14 const a = 1
15+const b = 2
16+const c = 3
17 const d = 4
18`
19
20interface Seen {
21 reviews: ReviewSubmission[]
22 bodyUpdates: number
23}
24
25/**
26 * A provider carrying one pull request, with a scriptable review outcome.
27 *
28 * @param options - Capability toggle and what createReview should report
29 */
30function stub(options: {
31 suggestions?: boolean
32 submission?: (review: ReviewSubmission) => ReviewSubmissionResult
33} = {}): { provider: GitProvider, seen: Seen } {
34 const seen: Seen = { reviews: [], bodyUpdates: 0 }
35
36 const pr = {
37 number: 7,
38 title: 'add a thing',
39 body: 'description',
40 head: 'feature',
41 base: 'main',
42 state: 'open',
43 labels: [],
44 draft: false,
45 } as unknown as PullRequest
46
47 const provider = {
48 capabilities: () => ({
49 ...NO_CAPABILITIES,
50 inlineReviewComments: true,
51 reviewSuggestions: options.suggestions ?? true,
52 }),
53 getPullRequests: async () => [pr],
54 getPullRequestDiff: async () => DIFF,
55 getPullRequestHeadSha: async () => 'head1',
56 getFileContent: async () => null,
57 createReview: async (_prNumber: number, review: ReviewSubmission) => {
58 seen.reviews.push(review)
59 return options.submission
60 ? options.submission(review)
61 : { posted: true, inlineComments: review.comments?.length ?? 0 }
62 },
63 updatePullRequest: async () => {
64 seen.bodyUpdates++
65 return pr
66 },
67 } as unknown as GitProvider
68
69 return { provider, seen }
70}
71
72/** An AI client that reports these findings for any diff. */
73function reportingAi(findings: unknown[]): AiClient {
74 return {
75 name: 'stub',
76 complete: async () => ({
77 // The engine reads `json`, not `text` — a stub answering in prose is
78 // indistinguishable from a review that found nothing.
79 json: { summary: 'looked at it', walkthrough: [], findings },
80 outputTokens: 1,
81 }),
82 } as unknown as AiClient
83}
84
85const FINDING = {
86 path: 'src/app.ts',
87 line: 2,
88 severity: 'minor',
89 category: 'style',
90 message: 'Name this better.',
91 suggestion: 'const better = 2',
92}
93
94const config: BuddyConfig = { analysis: { enabled: false } }
95
96/**
97 * `createReview` reports what publishing achieved — `ReviewSubmissionResult`
98 * was declared and implemented by all three providers, and the one caller
99 * ignored it: a review that failed to post was reported as posted, and
100 * inline comments the platform dropped vanished without a trace.
101 */
102describe('review submission outcomes', () => {
103 it('failure case - a review that was not posted says so and records nothing', async () => {
104 const { provider, seen } = stub({ submission: () => ({ posted: false, inlineComments: 0 }) })
105
106 const { status } = await reviewPullRequest({
107 config,
108 provider,
109 prNumber: 7,
110 ai: reportingAi([FINDING]),
111 logger: Logger.silent(),
112 })
113
114 expect(status).toBe('The review could not be posted.')
115 // Persisting the state anyway would make every later run skip a review
116 // nobody can see.
117 expect(seen.bodyUpdates).toBe(0)
118 })
119
120 it('failure case - dropped inline comments are named in the status', async () => {
121 const { provider } = stub({ submission: () => ({ posted: true, inlineComments: 1 }) })
122
123 const { status } = await reviewPullRequest({
124 config,
125 provider,
126 prNumber: 7,
127 ai: reportingAi([
128 FINDING,
129 { ...FINDING, line: 3, message: 'This one anchors nowhere.' },
130 ]),
131 logger: Logger.silent(),
132 })
133
134 expect(status).toContain('1 of 2 inline comment(s) could not be placed on the diff.')
135 })
136
137 it('success case - a fully placed review reports no shortfall', async () => {
138 const { provider, seen } = stub()
139
140 const { status } = await reviewPullRequest({
141 config,
142 provider,
143 prNumber: 7,
144 ai: reportingAi([FINDING]),
145 logger: Logger.silent(),
146 })
147
148 expect(status).toContain('1 finding(s) posted.')
149 expect(status).not.toContain('could not be placed')
150 expect(seen.bodyUpdates).toBe(1)
151 })
152})
153
154/**
155 * `ProviderCapabilities.reviewSuggestions` was declared, documented and set
156 * by every provider — and read by nothing, so Bitbucket received literal
157 * ```suggestion fences promising a one-click apply it does not have.
158 */
159describe('suggestion blocks follow the capability', () => {
160 it('success case - a capable platform gets the suggestion block', async () => {
161 const { provider, seen } = stub({ suggestions: true })
162
163 await reviewPullRequest({
164 config,
165 provider,
166 prNumber: 7,
167 ai: reportingAi([FINDING]),
168 logger: Logger.silent(),
169 })
170
171 const body = seen.reviews[0]?.comments?.[0]?.body ?? ''
172 expect(body).toContain('```suggestion')
173 })
174
175 it('failure case - a platform without suggestions gets a plain fence', async () => {
176 const { provider, seen } = stub({ suggestions: false })
177
178 await reviewPullRequest({
179 config,
180 provider,
181 prNumber: 7,
182 ai: reportingAi([FINDING]),
183 logger: Logger.silent(),
184 })
185
186 const body = seen.reviews[0]?.comments?.[0]?.body ?? ''
187 expect(body).not.toContain('```suggestion')
188 expect(body).toContain('Suggested replacement:')
189 expect(body).toContain(FINDING.suggestion)
190 })
191})