also looking at this
fix(review): read what publishing reported, honour reviewSuggestions, and un-strand two flags
#1435
5 files
+267
-10
| @@ -0,0 +1,191 @@ | ||
| 1 | import type { AiClient } from '../src/ai/types' | |
| 2 | import type { GitProvider, ReviewSubmission, ReviewSubmissionResult } from '../src/git/provider' | |
| 3 | import type { BuddyConfig, PullRequest } from '../src/types' | |
| 4 | import { describe, expect, it } from 'bun:test' | |
| 5 | import { NO_CAPABILITIES } from '../src/git/provider' | |
| 6 | import { reviewPullRequest } from '../src/review/run' | |
| 7 | import { Logger } from '../src/utils/logger' | |
| 8 | ||
| 9 | const DIFF = `diff --git a/src/app.ts b/src/app.ts | |
| 10 | index 111..222 100644 | |
| 11 | --- a/src/app.ts | |
| 12 | +++ b/src/app.ts | |
| 13 | @@ -1,2 +1,4 @@ | |
| 14 | const a = 1 | |
| 15 | +const b = 2 | |
| 16 | +const c = 3 | |
| 17 | const d = 4 | |
| 18 | ` | |
| 19 | ||
| 20 | interface Seen { | |
| 21 | reviews: ReviewSubmission[] | |
| 22 | bodyUpdates: number | |
| 23 | } | |
| 24 | ||
| 25 | /** | |
| 26 | * A provider carrying one pull request, with a scriptable review outcome. | |
| 27 | * | |
| 28 | * @param options - Capability toggle and what createReview should report | |
| 29 | */ | |
| 30 | function stub(options: { | |
| 31 | suggestions?: boolean | |
| 32 | submission?: (review: ReviewSubmission) => ReviewSubmissionResult | |
| 33 | } = {}): { provider: GitProvider, seen: Seen } { | |
| 34 | const seen: Seen = { reviews: [], bodyUpdates: 0 } | |
| 35 | ||
| 36 | const pr = { | |
| 37 | number: 7, | |
| 38 | title: 'add a thing', | |
| 39 | body: 'description', | |
| 40 | head: 'feature', | |
| 41 | base: 'main', | |
| 42 | state: 'open', | |
| 43 | labels: [], | |
| 44 | draft: false, | |
| 45 | } as unknown as PullRequest | |
| 46 | ||
| 47 | const provider = { | |
| 48 | capabilities: () => ({ | |
| 49 | ...NO_CAPABILITIES, | |
| 50 | inlineReviewComments: true, | |
| 51 | reviewSuggestions: options.suggestions ?? true, | |
| 52 | }), | |
| 53 | getPullRequests: async () => [pr], | |
| 54 | getPullRequestDiff: async () => DIFF, | |
| 55 | getPullRequestHeadSha: async () => 'head1', | |
| 56 | getFileContent: async () => null, | |
| 57 | createReview: async (_prNumber: number, review: ReviewSubmission) => { | |
| 58 | seen.reviews.push(review) | |
| 59 | return options.submission | |
| 60 | ? options.submission(review) | |
| 61 | : { posted: true, inlineComments: review.comments?.length ?? 0 } | |
| 62 | }, | |
| 63 | updatePullRequest: async () => { | |
| 64 | seen.bodyUpdates++ | |
| 65 | return pr | |
| 66 | }, | |
| 67 | } as unknown as GitProvider | |
| 68 | ||
| 69 | return { provider, seen } | |
| 70 | } | |
| 71 | ||
| 72 | /** An AI client that reports these findings for any diff. */ | |
| 73 | function reportingAi(findings: unknown[]): AiClient { | |
| 74 | return { | |
| 75 | name: 'stub', | |
| 76 | complete: async () => ({ | |
| 77 | // The engine reads `json`, not `text` — a stub answering in prose is | |
| 78 | // indistinguishable from a review that found nothing. | |
| 79 | json: { summary: 'looked at it', walkthrough: [], findings }, | |
| 80 | outputTokens: 1, | |
| 81 | }), | |
| 82 | } as unknown as AiClient | |
| 83 | } | |
| 84 | ||
| 85 | const FINDING = { | |
| 86 | path: 'src/app.ts', | |
| 87 | line: 2, | |
| 88 | severity: 'minor', | |
| 89 | category: 'style', | |
| 90 | message: 'Name this better.', | |
| 91 | suggestion: 'const better = 2', | |
| 92 | } | |
| 93 | ||
| 94 | const config: BuddyConfig = { analysis: { enabled: false } } | |
| 95 | ||
| 96 | /** | |
| 97 | * `createReview` reports what publishing achieved — `ReviewSubmissionResult` | |
| 98 | * was declared and implemented by all three providers, and the one caller | |
| 99 | * ignored it: a review that failed to post was reported as posted, and | |
| 100 | * inline comments the platform dropped vanished without a trace. | |
| 101 | */ | |
| 102 | describe('review submission outcomes', () => { | |
| 103 | it('failure case - a review that was not posted says so and records nothing', async () => { | |
| 104 | const { provider, seen } = stub({ submission: () => ({ posted: false, inlineComments: 0 }) }) | |
| 105 | ||
| 106 | const { status } = await reviewPullRequest({ | |
| 107 | config, | |
| 108 | provider, | |
| 109 | prNumber: 7, | |
| 110 | ai: reportingAi([FINDING]), | |
| 111 | logger: Logger.silent(), | |
| 112 | }) | |
| 113 | ||
| 114 | expect(status).toBe('The review could not be posted.') | |
| 115 | // Persisting the state anyway would make every later run skip a review | |
| 116 | // nobody can see. | |
| 117 | expect(seen.bodyUpdates).toBe(0) | |
| 118 | }) | |
| 119 | ||
| 120 | it('failure case - dropped inline comments are named in the status', async () => { | |
| 121 | const { provider } = stub({ submission: () => ({ posted: true, inlineComments: 1 }) }) | |
| 122 | ||
| 123 | const { status } = await reviewPullRequest({ | |
| 124 | config, | |
| 125 | provider, | |
| 126 | prNumber: 7, | |
| 127 | ai: reportingAi([ | |
| 128 | FINDING, | |
| 129 | { ...FINDING, line: 3, message: 'This one anchors nowhere.' }, | |
| 130 | ]), | |
| 131 | logger: Logger.silent(), | |
| 132 | }) | |
| 133 | ||
| 134 | expect(status).toContain('1 of 2 inline comment(s) could not be placed on the diff.') | |
| 135 | }) | |
| 136 | ||
| 137 | it('success case - a fully placed review reports no shortfall', async () => { | |
| 138 | const { provider, seen } = stub() | |
| 139 | ||
| 140 | const { status } = await reviewPullRequest({ | |
| 141 | config, | |
| 142 | provider, | |
| 143 | prNumber: 7, | |
| 144 | ai: reportingAi([FINDING]), | |
| 145 | logger: Logger.silent(), | |
| 146 | }) | |
| 147 | ||
| 148 | expect(status).toContain('1 finding(s) posted.') | |
| 149 | expect(status).not.toContain('could not be placed') | |
| 150 | expect(seen.bodyUpdates).toBe(1) | |
| 151 | }) | |
| 152 | }) | |
| 153 | ||
| 154 | /** | |
| 155 | * `ProviderCapabilities.reviewSuggestions` was declared, documented and set | |
| 156 | * by every provider — and read by nothing, so Bitbucket received literal | |
| 157 | * ```suggestion fences promising a one-click apply it does not have. | |
| 158 | */ | |
| 159 | describe('suggestion blocks follow the capability', () => { | |
| 160 | it('success case - a capable platform gets the suggestion block', async () => { | |
| 161 | const { provider, seen } = stub({ suggestions: true }) | |
| 162 | ||
| 163 | await reviewPullRequest({ | |
| 164 | config, | |
| 165 | provider, | |
| 166 | prNumber: 7, | |
| 167 | ai: reportingAi([FINDING]), | |
| 168 | logger: Logger.silent(), | |
| 169 | }) | |
| 170 | ||
| 171 | const body = seen.reviews[0]?.comments?.[0]?.body ?? '' | |
| 172 | expect(body).toContain('```suggestion') | |
| 173 | }) | |
| 174 | ||
| 175 | it('failure case - a platform without suggestions gets a plain fence', async () => { | |
| 176 | const { provider, seen } = stub({ suggestions: false }) | |
| 177 | ||
| 178 | await reviewPullRequest({ | |
| 179 | config, | |
| 180 | provider, | |
| 181 | prNumber: 7, | |
| 182 | ai: reportingAi([FINDING]), | |
| 183 | logger: Logger.silent(), | |
| 184 | }) | |
| 185 | ||
| 186 | const body = seen.reviews[0]?.comments?.[0]?.body ?? '' | |
| 187 | expect(body).not.toContain('```suggestion') | |
| 188 | expect(body).toContain('Suggested replacement:') | |
| 189 | expect(body).toContain(FINDING.suggestion) | |
| 190 | }) | |
| 191 | }) | |