ReviewOS

also looking at this

stacks/uplink

fix(app): an installed database gains the columns a new build needs

#43
Merged glennmichael123 wants to merge fix/sqlite-store-migrations into main
4 files +440 -37
tests/unit/uplink/sqlite-store.test.tsmodified+222-2
Changes to tests/unit/uplink/sqlite-store.test.ts
@@ -1,12 +1,13 @@
11import type { UplinkConfig } from '../../../app/Uplink/config'
22import { afterEach, describe, expect, it } from 'bun:test'
3import { mkdtempSync, rmSync } from 'node:fs'
3import { Database } from 'bun:sqlite'
4import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
45import { tmpdir } from 'node:os'
56import { join } from 'node:path'
67import { loadConfig } from '../../../app/Uplink/config'
78import { MessagesDb } from '../../../app/Uplink/messages-db'
89import { cleanToken, readSettings, settingsEnv, tokenLooksValid, writeSettings } from '../../../app/Uplink/settings'
9import { SqliteStore } from '../../../app/Uplink/sqlite-store'
10import { migrate, RUN_COLUMNS, schemaVersion, SqliteStore, TABLES } from '../../../app/Uplink/sqlite-store'
1011import { Uplink } from '../../../app/Uplink/uplink'
1112import { FakeChatDb, FakeEngine, FakeSender, ME, settle } from './fixtures'
1213
@@ -64,6 +65,225 @@ describe('SqliteStore', () => {
6465 })
6566})
6667
68/**
69 * The shape Uplink 0.1.3 wrote, which is what is on every Mac that already
70 * runs it. Frozen on purpose: it is the oldest database a new build has to
71 * open, so it is the thing an upgrade has to be proved against.
72 */
73const SHIPPED = {
74 state: 'key TEXT PRIMARY KEY, value TEXT NOT NULL',
75 conversations: 'chat_guid TEXT PRIMARY KEY, handle TEXT NOT NULL, service TEXT NOT NULL DEFAULT \'iMessage\', session_id TEXT, cwd TEXT, last_active_at INTEGER, more_text TEXT',
76 runs: 'id INTEGER PRIMARY KEY AUTOINCREMENT, chat_guid TEXT NOT NULL, message_guid TEXT NOT NULL, prompt TEXT NOT NULL, cwd TEXT, status TEXT NOT NULL DEFAULT \'queued\', reply TEXT, error TEXT, session_id TEXT, cost_cents INTEGER, duration_ms INTEGER, last_activity TEXT, started_at INTEGER, finished_at INTEGER',
77}
78
79function write(path: string, tables: Record<string, string>): void {
80 const db = new Database(path, { create: true })
81 for (const [name, columns] of Object.entries(tables))
82 db.run(`CREATE TABLE ${name} (${columns})`)
83 db.close()
84}
85
86function columnsOf(path: string, table: string): string[] {
87 const db = new Database(path)
88 const names = (db.query(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(column => column.name)
89 db.close()
90 return names
91}
92
93/**
94 * Compared as a set, not a list. `ALTER TABLE ADD COLUMN` appends, so a column
95 * written into the middle of `TABLES` is last on an upgraded database and
96 * mid-list on a new one. Both are correct, and an order-sensitive assertion
97 * would fail the upgrade that worked.
98 */
99function sameColumns(path: string, table: { name: string, columns: Record<string, string> }): void {
100 expect([...columnsOf(path, table.name)].sort()).toEqual(Object.keys(table.columns).sort())
101}
102
103function versionOf(path: string): number {
104 const db = new Database(path)
105 const version = schemaVersion(db)
106 db.close()
107 return version
108}
109
110describe('SqliteStore migrations', () => {
111 it('creates a database that is already current', () => {
112 const path = join(tempDir(), 'uplink.sqlite')
113 const store = new SqliteStore(path)
114 expect(store.schemaError).toBeNull()
115 store.close()
116
117 for (const table of TABLES)
118 expect(columnsOf(path, table.name)).toEqual(Object.keys(table.columns))
119
120 expect(versionOf(path)).toBeGreaterThan(0)
121 })
122
123 /**
124 * The bug this exists for. `CREATE TABLE IF NOT EXISTS` is a no-op against a
125 * database that is already there, so before this every column added to the
126 * schema was a column an installed copy would never get, and its first INSERT
127 * after the update would fail. Here `runs` is missing the four columns a
128 * newer build names and `conversations` is missing entirely.
129 */
130 it('adds what an installed database is missing, without losing its rows', async () => {
131 const path = join(tempDir(), 'uplink.sqlite')
132 write(path, {
133 state: SHIPPED.state,
134 runs: 'id INTEGER PRIMARY KEY AUTOINCREMENT, chat_guid TEXT NOT NULL, message_guid TEXT NOT NULL, prompt TEXT NOT NULL, cwd TEXT, status TEXT NOT NULL DEFAULT \'queued\', reply TEXT, error TEXT, session_id TEXT, cost_cents INTEGER',
135 })
136 const older = new Database(path)
137 older.run('INSERT INTO runs (chat_guid, message_guid, prompt, cwd, status, reply) VALUES (?, ?, ?, ?, ?, ?)', ['g', 'm', 'answered before the update', '/tmp', 'done', 'hi'])
138 older.close()
139
140 const store = new SqliteStore(path)
141 expect(store.schemaError).toBeNull()
142
143 // The history it already had is still there.
144 expect(store.recentRuns()[0]).toMatchObject({ prompt: 'answered before the update', status: 'done', reply: 'hi' })
145
146 // And the columns it did not have are usable, which is the INSERT that
147 // used to fail on every text.
148 const id = await store.createRun({ chatGuid: 'g', messageGuid: 'm2', prompt: 'after', cwd: '/tmp', status: 'queued' })
149 await store.updateRun(id, { status: 'done', durationMs: 7, lastActivity: 'Thinking', startedAt: 1, finishedAt: 8 })
150 expect(store.recentRuns()[0]).toMatchObject({ id, durationMs: 7, lastActivity: 'Thinking', startedAt: 1, finishedAt: 8 })
151
152 // A table that did not exist at all is created rather than altered.
153 await store.saveConversation({ chatGuid: 'g', handle: ME, service: 'iMessage', sessionId: 's', cwd: '/tmp', lastActiveAt: 5, moreText: null })
154 expect(await store.conversation('g')).toMatchObject({ sessionId: 's' })
155 store.close()
156 })
157
158 /**
159 * The guard that makes the mechanism worth having. SQLite will not ALTER in
160 * a PRIMARY KEY or UNIQUE column at all, and refuses a NOT NULL column with
161 * no default, or a non-constant default, **once the table has rows**. A
162 * column written any of those ways reaches every new install and no existing
163 * one, which is the failure this whole file is about, one level up.
164 *
165 * The rows are the point. On an empty table SQLite 3.51 accepts both
166 * `NOT NULL` with no default and `DEFAULT (datetime())`, so a version of
167 * this test that migrated an empty 0.1.3 database would pass while every
168 * real Mac, which has a history, failed.
169 *
170 * It passes trivially while the schema is unchanged and fails the day
171 * someone adds a column that could not travel.
172 */
173 it('can add every column to a shipped database that is already in use', () => {
174 // A table added after 0.1.3 would be created rather than altered, and this
175 // test would quietly stop covering it. Add it to SHIPPED, or state here
176 // why it does not need the guard.
177 expect(TABLES.map(table => table.name).sort()).toEqual(Object.keys(SHIPPED).sort())
178
179 const path = join(tempDir(), 'uplink.sqlite')
180 write(path, SHIPPED)
181 const used = new Database(path)
182 used.run('INSERT INTO state (key, value) VALUES (?, ?)', ['cursor', '1'])
183 used.run('INSERT INTO conversations (chat_guid, handle) VALUES (?, ?)', ['g', ME])
184 used.run('INSERT INTO runs (chat_guid, message_guid, prompt) VALUES (?, ?, ?)', ['g', 'm', 'p'])
185 used.close()
186
187 expect(versionOf(path)).toBe(0)
188
189 const store = new SqliteStore(path)
190 expect(store.schemaError).toBeNull()
191 store.close()
192
193 for (const table of TABLES)
194 sameColumns(path, table)
195 // Stamped on an upgraded database, not only on a new one. Without this a
196 // migration could leave the version at 0 and every other test stay green.
197 const fresh = join(tempDir(), 'fresh.sqlite')
198 new SqliteStore(fresh).close()
199 expect(versionOf(path)).toBe(versionOf(fresh))
200 })
201
202 /**
203 * The claim in this file's header is that a column is added in one place.
204 * TypeScript only half enforces it: `RUN_COLUMNS` is a `Record` over
205 * `RunRecord`, so those two cannot drift, but nothing ties either to
206 * `TABLES`. A column named in one and not the other is exactly #9 again, so
207 * it is pinned here instead of in the type system.
208 */
209 it('names the same runs columns in TABLES and in RUN_COLUMNS', () => {
210 const declared = Object.keys(TABLES.find(table => table.name === 'runs')!.columns)
211 // `id` is the key SQLite assigns; nothing writes it.
212 expect(Object.values(RUN_COLUMNS).sort()).toEqual(declared.filter(name => name !== 'id').sort())
213 })
214
215 /**
216 * During an update two Uplinks hold this file for a moment, and the loser of
217 * a write lock abandons it for an in-memory database and loses the history.
218 * A launch that changes nothing must therefore not ask for a write lock, and
219 * almost every launch changes nothing.
220 *
221 * Not covered here: the launch that does have work to do and meets a lock.
222 * That one waits on `PRAGMA busy_timeout`, and proving it needs a second
223 * process, because SQLite's wait blocks this one's event loop.
224 */
225 it('opens a current database while another process is writing to it', async () => {
226 const path = join(tempDir(), 'uplink.sqlite')
227 new SqliteStore(path).close()
228
229 const other = new Database(path)
230 other.exec('BEGIN IMMEDIATE')
231 other.run('INSERT INTO state (key, value) VALUES (?, ?)', ['cursor', '1'])
232
233 // Opening is the assertion: it neither threw nor fell back to memory.
234 const store = new SqliteStore(path)
235 expect(store.schemaError).toBeNull()
236
237 other.exec('ROLLBACK')
238 other.close()
239
240 // And it is the real file, not a replacement that forgets on quit.
241 expect(await store.createRun({ chatGuid: 'g', messageGuid: 'm', prompt: 'p', cwd: '/tmp', status: 'queued' })).toBeGreaterThan(0)
242 store.close()
243 expect(new SqliteStore(path).recentRuns()[0]).toMatchObject({ prompt: 'p' })
244 })
245
246 it('runs twice over the same database without changing it', () => {
247 const path = join(tempDir(), 'uplink.sqlite')
248 new SqliteStore(path).close()
249 const after = TABLES.map(table => columnsOf(path, table.name))
250 new SqliteStore(path).close()
251 expect(TABLES.map(table => columnsOf(path, table.name))).toEqual(after)
252 })
253
254 /**
255 * The store is built at the top of the desktop agent, under launchd, so a
256 * throw here is a restart loop with no menubar to explain it. A ruined file
257 * costs the history, not the app.
258 */
259 it('keeps answering texts when the file cannot be opened', async () => {
260 const path = join(tempDir(), 'uplink.sqlite')
261 writeFileSync(path, 'this is not a database')
262
263 const store = new SqliteStore(path)
264 expect(store.schemaError).toContain('not a database')
265 expect(store.schemaError).toContain('not saving them')
266
267 const id = await store.createRun({ chatGuid: 'g', messageGuid: 'm', prompt: 'still works', cwd: '/tmp', status: 'queued' })
268 expect(store.recentRuns()[0]).toMatchObject({ id, prompt: 'still works' })
269 store.close()
270 })
271
272 it('leaves a half-applied change out of the database entirely', () => {
273 const db = new Database(':memory:')
274 db.run('CREATE TABLE runs (id INTEGER PRIMARY KEY)')
275 // UNIQUE is one SQLite never allows in an ALTER, so this stands in for a
276 // migration that fails partway: the table before it was created first.
277 expect(() => migrate(db, [
278 { name: 'created_first', columns: { a: 'TEXT' } },
279 { name: 'runs', columns: { id: 'INTEGER PRIMARY KEY', nope: 'TEXT UNIQUE' } },
280 ])).toThrow()
281 expect(db.query('SELECT name FROM sqlite_master WHERE type = ?').all('table')).toEqual([{ name: 'runs' }])
282 expect(schemaVersion(db)).toBe(0)
283 db.close()
284 })
285})
286
67287describe('settings', () => {
68288 it('fills in defaults and survives a broken file', () => {
69289 const path = join(tempDir(), 'settings.json')