also looking at this
fix: report skipped gates as skipped, and honour ai.majorUpgrades.maxAttempts
#1433
9 files
+257
-8
| @@ -0,0 +1,120 @@ | ||
| 1 | import type { AiClient } from '../src/ai/types' | |
| 2 | import type { AgentRunResult } from '../src/agent/types' | |
| 3 | import type { PackageUpdate } from '../src/types' | |
| 4 | import { describe, expect, it } from 'bun:test' | |
| 5 | import { attemptMajorUpgrade } from '../src/upgrades/migrate' | |
| 6 | import { Logger } from '../src/utils/logger' | |
| 7 | ||
| 8 | const PLAN = { | |
| 9 | changes: [{ description: 'the legacy import was removed', action: 'import from the root', affectedFiles: [], automatable: true }], | |
| 10 | confidence: 'high', | |
| 11 | effort: 2, | |
| 12 | risks: [], | |
| 13 | } | |
| 14 | ||
| 15 | /** An analysis client that always returns the same plan. */ | |
| 16 | const ai = { | |
| 17 | provider: 'anthropic', | |
| 18 | model: 'test', | |
| 19 | tokensUsed: 0, | |
| 20 | async complete() { | |
| 21 | return { text: JSON.stringify(PLAN), toolCalls: [], json: PLAN, stopReason: 'end', usage: { inputTokens: 1, outputTokens: 1 }, model: 'test' } | |
| 22 | }, | |
| 23 | } as unknown as AiClient | |
| 24 | ||
| 25 | const update: PackageUpdate = { | |
| 26 | name: 'react', | |
| 27 | currentVersion: '17.0.2', | |
| 28 | newVersion: '19.0.0', | |
| 29 | updateType: 'major', | |
| 30 | dependencyType: 'dependencies', | |
| 31 | file: 'package.json', | |
| 32 | } as PackageUpdate | |
| 33 | ||
| 34 | /** A scripted agent: one result per call, recording the tasks it was given. */ | |
| 35 | function scriptedAgent(results: Array<Partial<AgentRunResult>>) { | |
| 36 | const tasks: string[] = [] | |
| 37 | let call = 0 | |
| 38 | const agent = async (_ai: unknown, options: { task: string }): Promise<AgentRunResult> => { | |
| 39 | tasks.push(options.task) | |
| 40 | const scripted = results[Math.min(call++, results.length - 1)] | |
| 41 | return { output: '', stopReason: 'completed', transcript: [], outputTokens: 1, toolCalls: 1, ...scripted } | |
| 42 | } | |
| 43 | return { agent: agent as never, tasks } | |
| 44 | } | |
| 45 | ||
| 46 | function attempt(maxAttempts: number | undefined, results: Array<Partial<AgentRunResult>>) { | |
| 47 | const { agent, tasks } = scriptedAgent(results) | |
| 48 | return attemptMajorUpgrade({ | |
| 49 | update, | |
| 50 | releaseNotes: 'removed the legacy API', | |
| 51 | files: [], | |
| 52 | workspace: '/tmp', | |
| 53 | baseBranch: 'main', | |
| 54 | ai, | |
| 55 | autoMigrate: true, | |
| 56 | ...(maxAttempts === undefined ? {} : { maxAttempts }), | |
| 57 | agent, | |
| 58 | logger: Logger.silent(), | |
| 59 | }).then(result => ({ result, tasks })) | |
| 60 | } | |
| 61 | ||
| 62 | /** | |
| 63 | * `ai.majorUpgrades.maxAttempts` was declared, documented as "maximum agent | |
| 64 | * attempts per upgrade", and the agent ran exactly once whatever it said. | |
| 65 | */ | |
| 66 | describe('migration attempts', () => { | |
| 67 | it('success case - a run that completes takes one attempt', async () => { | |
| 68 | const { result, tasks } = await attempt(3, [{ stopReason: 'completed' }]) | |
| 69 | ||
| 70 | expect(tasks).toHaveLength(1) | |
| 71 | expect(result.outcome?.attempts).toBe(1) | |
| 72 | expect(result.status).toBe('migrated') | |
| 73 | }) | |
| 74 | ||
| 75 | it('success case - a run that stops short is retried up to the limit', async () => { | |
| 76 | const { result, tasks } = await attempt(3, [ | |
| 77 | { stopReason: 'max_tool_calls', output: 'got halfway' }, | |
| 78 | { stopReason: 'completed' }, | |
| 79 | ]) | |
| 80 | ||
| 81 | expect(tasks).toHaveLength(2) | |
| 82 | expect(result.outcome?.attempts).toBe(2) | |
| 83 | expect(result.status).toBe('migrated') | |
| 84 | }) | |
| 85 | ||
| 86 | it('success case - the retry is told what happened and to continue', async () => { | |
| 87 | // Starting over would redo the work that did land; the second run is | |
| 88 | // handed the first one's output and pointed at the workspace as it is. | |
| 89 | const { tasks } = await attempt(2, [ | |
| 90 | { stopReason: 'timeout', output: 'renamed two imports' }, | |
| 91 | { stopReason: 'completed' }, | |
| 92 | ]) | |
| 93 | ||
| 94 | expect(tasks[1]).toContain('renamed two imports') | |
| 95 | expect(tasks[1]).toContain('Continue from the workspace') | |
| 96 | }) | |
| 97 | ||
| 98 | it('failure case - the limit is respected', async () => { | |
| 99 | const { result, tasks } = await attempt(2, [{ stopReason: 'max_tool_calls' }]) | |
| 100 | ||
| 101 | expect(tasks).toHaveLength(2) | |
| 102 | expect(result.status).toBe('migration-failed') | |
| 103 | expect(result.draft).toBe(true) | |
| 104 | }) | |
| 105 | ||
| 106 | it('edge case - unset means one attempt, as before', async () => { | |
| 107 | const { tasks } = await attempt(undefined, [{ stopReason: 'error' }]) | |
| 108 | ||
| 109 | expect(tasks).toHaveLength(1) | |
| 110 | }) | |
| 111 | ||
| 112 | it('edge case - applied stays true if any attempt changed code', async () => { | |
| 113 | const { result } = await attempt(2, [ | |
| 114 | { stopReason: 'max_tool_calls', toolCalls: 3 }, | |
| 115 | { stopReason: 'error', toolCalls: 0 }, | |
| 116 | ]) | |
| 117 | ||
| 118 | expect(result.outcome?.applied).toBe(true) | |
| 119 | }) | |
| 120 | }) | |