ReviewOS

also looking at this

stacks/buddy

fix: report skipped gates as skipped, and honour ai.majorUpgrades.maxAttempts

#1433
Merged glennmichael123 wants to merge fix/gate-neutral into main
9 files +257 -8
test/upgrade-attempts.test.tsadded+120-0
Changes to test/upgrade-attempts.test.ts
@@ -0,0 +1,120 @@
1import type { AiClient } from '../src/ai/types'
2import type { AgentRunResult } from '../src/agent/types'
3import type { PackageUpdate } from '../src/types'
4import { describe, expect, it } from 'bun:test'
5import { attemptMajorUpgrade } from '../src/upgrades/migrate'
6import { Logger } from '../src/utils/logger'
7
8const PLAN = {
9 changes: [{ description: 'the legacy import was removed', action: 'import from the root', affectedFiles: [], automatable: true }],
10 confidence: 'high',
11 effort: 2,
12 risks: [],
13}
14
15/** An analysis client that always returns the same plan. */
16const ai = {
17 provider: 'anthropic',
18 model: 'test',
19 tokensUsed: 0,
20 async complete() {
21 return { text: JSON.stringify(PLAN), toolCalls: [], json: PLAN, stopReason: 'end', usage: { inputTokens: 1, outputTokens: 1 }, model: 'test' }
22 },
23} as unknown as AiClient
24
25const update: PackageUpdate = {
26 name: 'react',
27 currentVersion: '17.0.2',
28 newVersion: '19.0.0',
29 updateType: 'major',
30 dependencyType: 'dependencies',
31 file: 'package.json',
32} as PackageUpdate
33
34/** A scripted agent: one result per call, recording the tasks it was given. */
35function scriptedAgent(results: Array<Partial<AgentRunResult>>) {
36 const tasks: string[] = []
37 let call = 0
38 const agent = async (_ai: unknown, options: { task: string }): Promise<AgentRunResult> => {
39 tasks.push(options.task)
40 const scripted = results[Math.min(call++, results.length - 1)]
41 return { output: '', stopReason: 'completed', transcript: [], outputTokens: 1, toolCalls: 1, ...scripted }
42 }
43 return { agent: agent as never, tasks }
44}
45
46function attempt(maxAttempts: number | undefined, results: Array<Partial<AgentRunResult>>) {
47 const { agent, tasks } = scriptedAgent(results)
48 return attemptMajorUpgrade({
49 update,
50 releaseNotes: 'removed the legacy API',
51 files: [],
52 workspace: '/tmp',
53 baseBranch: 'main',
54 ai,
55 autoMigrate: true,
56 ...(maxAttempts === undefined ? {} : { maxAttempts }),
57 agent,
58 logger: Logger.silent(),
59 }).then(result => ({ result, tasks }))
60}
61
62/**
63 * `ai.majorUpgrades.maxAttempts` was declared, documented as "maximum agent
64 * attempts per upgrade", and the agent ran exactly once whatever it said.
65 */
66describe('migration attempts', () => {
67 it('success case - a run that completes takes one attempt', async () => {
68 const { result, tasks } = await attempt(3, [{ stopReason: 'completed' }])
69
70 expect(tasks).toHaveLength(1)
71 expect(result.outcome?.attempts).toBe(1)
72 expect(result.status).toBe('migrated')
73 })
74
75 it('success case - a run that stops short is retried up to the limit', async () => {
76 const { result, tasks } = await attempt(3, [
77 { stopReason: 'max_tool_calls', output: 'got halfway' },
78 { stopReason: 'completed' },
79 ])
80
81 expect(tasks).toHaveLength(2)
82 expect(result.outcome?.attempts).toBe(2)
83 expect(result.status).toBe('migrated')
84 })
85
86 it('success case - the retry is told what happened and to continue', async () => {
87 // Starting over would redo the work that did land; the second run is
88 // handed the first one's output and pointed at the workspace as it is.
89 const { tasks } = await attempt(2, [
90 { stopReason: 'timeout', output: 'renamed two imports' },
91 { stopReason: 'completed' },
92 ])
93
94 expect(tasks[1]).toContain('renamed two imports')
95 expect(tasks[1]).toContain('Continue from the workspace')
96 })
97
98 it('failure case - the limit is respected', async () => {
99 const { result, tasks } = await attempt(2, [{ stopReason: 'max_tool_calls' }])
100
101 expect(tasks).toHaveLength(2)
102 expect(result.status).toBe('migration-failed')
103 expect(result.draft).toBe(true)
104 })
105
106 it('edge case - unset means one attempt, as before', async () => {
107 const { tasks } = await attempt(undefined, [{ stopReason: 'error' }])
108
109 expect(tasks).toHaveLength(1)
110 })
111
112 it('edge case - applied stays true if any attempt changed code', async () => {
113 const { result } = await attempt(2, [
114 { stopReason: 'max_tool_calls', toolCalls: 3 },
115 { stopReason: 'error', toolCalls: 0 },
116 ])
117
118 expect(result.outcome?.applied).toBe(true)
119 })
120})