also looking at this
fix(app): an installed database gains the columns a new build needs
#43
4 files
+440
-37
| @@ -1,12 +1,13 @@ | ||
| 1 | 1 | import type { UplinkConfig } from '../../../app/Uplink/config' |
| 2 | 2 | import { afterEach, describe, expect, it } from 'bun:test' |
| 3 | import { mkdtempSync, rmSync } from 'node:fs' | |
| 3 | import { Database } from 'bun:sqlite' | |
| 4 | import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' | |
| 4 | 5 | import { tmpdir } from 'node:os' |
| 5 | 6 | import { join } from 'node:path' |
| 6 | 7 | import { loadConfig } from '../../../app/Uplink/config' |
| 7 | 8 | import { MessagesDb } from '../../../app/Uplink/messages-db' |
| 8 | 9 | import { cleanToken, readSettings, settingsEnv, tokenLooksValid, writeSettings } from '../../../app/Uplink/settings' |
| 9 | import { SqliteStore } from '../../../app/Uplink/sqlite-store' | |
| 10 | import { migrate, RUN_COLUMNS, schemaVersion, SqliteStore, TABLES } from '../../../app/Uplink/sqlite-store' | |
| 10 | 11 | import { Uplink } from '../../../app/Uplink/uplink' |
| 11 | 12 | import { FakeChatDb, FakeEngine, FakeSender, ME, settle } from './fixtures' |
| 12 | 13 | |
| @@ -64,6 +65,225 @@ describe('SqliteStore', () => { | ||
| 64 | 65 | }) |
| 65 | 66 | }) |
| 66 | 67 | |
| 68 | /** | |
| 69 | * The shape Uplink 0.1.3 wrote, which is what is on every Mac that already | |
| 70 | * runs it. Frozen on purpose: it is the oldest database a new build has to | |
| 71 | * open, so it is the thing an upgrade has to be proved against. | |
| 72 | */ | |
| 73 | const SHIPPED = { | |
| 74 | state: 'key TEXT PRIMARY KEY, value TEXT NOT NULL', | |
| 75 | conversations: 'chat_guid TEXT PRIMARY KEY, handle TEXT NOT NULL, service TEXT NOT NULL DEFAULT \'iMessage\', session_id TEXT, cwd TEXT, last_active_at INTEGER, more_text TEXT', | |
| 76 | runs: 'id INTEGER PRIMARY KEY AUTOINCREMENT, chat_guid TEXT NOT NULL, message_guid TEXT NOT NULL, prompt TEXT NOT NULL, cwd TEXT, status TEXT NOT NULL DEFAULT \'queued\', reply TEXT, error TEXT, session_id TEXT, cost_cents INTEGER, duration_ms INTEGER, last_activity TEXT, started_at INTEGER, finished_at INTEGER', | |
| 77 | } | |
| 78 | ||
| 79 | function write(path: string, tables: Record<string, string>): void { | |
| 80 | const db = new Database(path, { create: true }) | |
| 81 | for (const [name, columns] of Object.entries(tables)) | |
| 82 | db.run(`CREATE TABLE ${name} (${columns})`) | |
| 83 | db.close() | |
| 84 | } | |
| 85 | ||
| 86 | function columnsOf(path: string, table: string): string[] { | |
| 87 | const db = new Database(path) | |
| 88 | const names = (db.query(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(column => column.name) | |
| 89 | db.close() | |
| 90 | return names | |
| 91 | } | |
| 92 | ||
| 93 | /** | |
| 94 | * Compared as a set, not a list. `ALTER TABLE ADD COLUMN` appends, so a column | |
| 95 | * written into the middle of `TABLES` is last on an upgraded database and | |
| 96 | * mid-list on a new one. Both are correct, and an order-sensitive assertion | |
| 97 | * would fail the upgrade that worked. | |
| 98 | */ | |
| 99 | function sameColumns(path: string, table: { name: string, columns: Record<string, string> }): void { | |
| 100 | expect([...columnsOf(path, table.name)].sort()).toEqual(Object.keys(table.columns).sort()) | |
| 101 | } | |
| 102 | ||
| 103 | function versionOf(path: string): number { | |
| 104 | const db = new Database(path) | |
| 105 | const version = schemaVersion(db) | |
| 106 | db.close() | |
| 107 | return version | |
| 108 | } | |
| 109 | ||
| 110 | describe('SqliteStore migrations', () => { | |
| 111 | it('creates a database that is already current', () => { | |
| 112 | const path = join(tempDir(), 'uplink.sqlite') | |
| 113 | const store = new SqliteStore(path) | |
| 114 | expect(store.schemaError).toBeNull() | |
| 115 | store.close() | |
| 116 | ||
| 117 | for (const table of TABLES) | |
| 118 | expect(columnsOf(path, table.name)).toEqual(Object.keys(table.columns)) | |
| 119 | ||
| 120 | expect(versionOf(path)).toBeGreaterThan(0) | |
| 121 | }) | |
| 122 | ||
| 123 | /** | |
| 124 | * The bug this exists for. `CREATE TABLE IF NOT EXISTS` is a no-op against a | |
| 125 | * database that is already there, so before this every column added to the | |
| 126 | * schema was a column an installed copy would never get, and its first INSERT | |
| 127 | * after the update would fail. Here `runs` is missing the four columns a | |
| 128 | * newer build names and `conversations` is missing entirely. | |
| 129 | */ | |
| 130 | it('adds what an installed database is missing, without losing its rows', async () => { | |
| 131 | const path = join(tempDir(), 'uplink.sqlite') | |
| 132 | write(path, { | |
| 133 | state: SHIPPED.state, | |
| 134 | runs: 'id INTEGER PRIMARY KEY AUTOINCREMENT, chat_guid TEXT NOT NULL, message_guid TEXT NOT NULL, prompt TEXT NOT NULL, cwd TEXT, status TEXT NOT NULL DEFAULT \'queued\', reply TEXT, error TEXT, session_id TEXT, cost_cents INTEGER', | |
| 135 | }) | |
| 136 | const older = new Database(path) | |
| 137 | older.run('INSERT INTO runs (chat_guid, message_guid, prompt, cwd, status, reply) VALUES (?, ?, ?, ?, ?, ?)', ['g', 'm', 'answered before the update', '/tmp', 'done', 'hi']) | |
| 138 | older.close() | |
| 139 | ||
| 140 | const store = new SqliteStore(path) | |
| 141 | expect(store.schemaError).toBeNull() | |
| 142 | ||
| 143 | // The history it already had is still there. | |
| 144 | expect(store.recentRuns()[0]).toMatchObject({ prompt: 'answered before the update', status: 'done', reply: 'hi' }) | |
| 145 | ||
| 146 | // And the columns it did not have are usable, which is the INSERT that | |
| 147 | // used to fail on every text. | |
| 148 | const id = await store.createRun({ chatGuid: 'g', messageGuid: 'm2', prompt: 'after', cwd: '/tmp', status: 'queued' }) | |
| 149 | await store.updateRun(id, { status: 'done', durationMs: 7, lastActivity: 'Thinking', startedAt: 1, finishedAt: 8 }) | |
| 150 | expect(store.recentRuns()[0]).toMatchObject({ id, durationMs: 7, lastActivity: 'Thinking', startedAt: 1, finishedAt: 8 }) | |
| 151 | ||
| 152 | // A table that did not exist at all is created rather than altered. | |
| 153 | await store.saveConversation({ chatGuid: 'g', handle: ME, service: 'iMessage', sessionId: 's', cwd: '/tmp', lastActiveAt: 5, moreText: null }) | |
| 154 | expect(await store.conversation('g')).toMatchObject({ sessionId: 's' }) | |
| 155 | store.close() | |
| 156 | }) | |
| 157 | ||
| 158 | /** | |
| 159 | * The guard that makes the mechanism worth having. SQLite will not ALTER in | |
| 160 | * a PRIMARY KEY or UNIQUE column at all, and refuses a NOT NULL column with | |
| 161 | * no default, or a non-constant default, **once the table has rows**. A | |
| 162 | * column written any of those ways reaches every new install and no existing | |
| 163 | * one, which is the failure this whole file is about, one level up. | |
| 164 | * | |
| 165 | * The rows are the point. On an empty table SQLite 3.51 accepts both | |
| 166 | * `NOT NULL` with no default and `DEFAULT (datetime())`, so a version of | |
| 167 | * this test that migrated an empty 0.1.3 database would pass while every | |
| 168 | * real Mac, which has a history, failed. | |
| 169 | * | |
| 170 | * It passes trivially while the schema is unchanged and fails the day | |
| 171 | * someone adds a column that could not travel. | |
| 172 | */ | |
| 173 | it('can add every column to a shipped database that is already in use', () => { | |
| 174 | // A table added after 0.1.3 would be created rather than altered, and this | |
| 175 | // test would quietly stop covering it. Add it to SHIPPED, or state here | |
| 176 | // why it does not need the guard. | |
| 177 | expect(TABLES.map(table => table.name).sort()).toEqual(Object.keys(SHIPPED).sort()) | |
| 178 | ||
| 179 | const path = join(tempDir(), 'uplink.sqlite') | |
| 180 | write(path, SHIPPED) | |
| 181 | const used = new Database(path) | |
| 182 | used.run('INSERT INTO state (key, value) VALUES (?, ?)', ['cursor', '1']) | |
| 183 | used.run('INSERT INTO conversations (chat_guid, handle) VALUES (?, ?)', ['g', ME]) | |
| 184 | used.run('INSERT INTO runs (chat_guid, message_guid, prompt) VALUES (?, ?, ?)', ['g', 'm', 'p']) | |
| 185 | used.close() | |
| 186 | ||
| 187 | expect(versionOf(path)).toBe(0) | |
| 188 | ||
| 189 | const store = new SqliteStore(path) | |
| 190 | expect(store.schemaError).toBeNull() | |
| 191 | store.close() | |
| 192 | ||
| 193 | for (const table of TABLES) | |
| 194 | sameColumns(path, table) | |
| 195 | // Stamped on an upgraded database, not only on a new one. Without this a | |
| 196 | // migration could leave the version at 0 and every other test stay green. | |
| 197 | const fresh = join(tempDir(), 'fresh.sqlite') | |
| 198 | new SqliteStore(fresh).close() | |
| 199 | expect(versionOf(path)).toBe(versionOf(fresh)) | |
| 200 | }) | |
| 201 | ||
| 202 | /** | |
| 203 | * The claim in this file's header is that a column is added in one place. | |
| 204 | * TypeScript only half enforces it: `RUN_COLUMNS` is a `Record` over | |
| 205 | * `RunRecord`, so those two cannot drift, but nothing ties either to | |
| 206 | * `TABLES`. A column named in one and not the other is exactly #9 again, so | |
| 207 | * it is pinned here instead of in the type system. | |
| 208 | */ | |
| 209 | it('names the same runs columns in TABLES and in RUN_COLUMNS', () => { | |
| 210 | const declared = Object.keys(TABLES.find(table => table.name === 'runs')!.columns) | |
| 211 | // `id` is the key SQLite assigns; nothing writes it. | |
| 212 | expect(Object.values(RUN_COLUMNS).sort()).toEqual(declared.filter(name => name !== 'id').sort()) | |
| 213 | }) | |
| 214 | ||
| 215 | /** | |
| 216 | * During an update two Uplinks hold this file for a moment, and the loser of | |
| 217 | * a write lock abandons it for an in-memory database and loses the history. | |
| 218 | * A launch that changes nothing must therefore not ask for a write lock, and | |
| 219 | * almost every launch changes nothing. | |
| 220 | * | |
| 221 | * Not covered here: the launch that does have work to do and meets a lock. | |
| 222 | * That one waits on `PRAGMA busy_timeout`, and proving it needs a second | |
| 223 | * process, because SQLite's wait blocks this one's event loop. | |
| 224 | */ | |
| 225 | it('opens a current database while another process is writing to it', async () => { | |
| 226 | const path = join(tempDir(), 'uplink.sqlite') | |
| 227 | new SqliteStore(path).close() | |
| 228 | ||
| 229 | const other = new Database(path) | |
| 230 | other.exec('BEGIN IMMEDIATE') | |
| 231 | other.run('INSERT INTO state (key, value) VALUES (?, ?)', ['cursor', '1']) | |
| 232 | ||
| 233 | // Opening is the assertion: it neither threw nor fell back to memory. | |
| 234 | const store = new SqliteStore(path) | |
| 235 | expect(store.schemaError).toBeNull() | |
| 236 | ||
| 237 | other.exec('ROLLBACK') | |
| 238 | other.close() | |
| 239 | ||
| 240 | // And it is the real file, not a replacement that forgets on quit. | |
| 241 | expect(await store.createRun({ chatGuid: 'g', messageGuid: 'm', prompt: 'p', cwd: '/tmp', status: 'queued' })).toBeGreaterThan(0) | |
| 242 | store.close() | |
| 243 | expect(new SqliteStore(path).recentRuns()[0]).toMatchObject({ prompt: 'p' }) | |
| 244 | }) | |
| 245 | ||
| 246 | it('runs twice over the same database without changing it', () => { | |
| 247 | const path = join(tempDir(), 'uplink.sqlite') | |
| 248 | new SqliteStore(path).close() | |
| 249 | const after = TABLES.map(table => columnsOf(path, table.name)) | |
| 250 | new SqliteStore(path).close() | |
| 251 | expect(TABLES.map(table => columnsOf(path, table.name))).toEqual(after) | |
| 252 | }) | |
| 253 | ||
| 254 | /** | |
| 255 | * The store is built at the top of the desktop agent, under launchd, so a | |
| 256 | * throw here is a restart loop with no menubar to explain it. A ruined file | |
| 257 | * costs the history, not the app. | |
| 258 | */ | |
| 259 | it('keeps answering texts when the file cannot be opened', async () => { | |
| 260 | const path = join(tempDir(), 'uplink.sqlite') | |
| 261 | writeFileSync(path, 'this is not a database') | |
| 262 | ||
| 263 | const store = new SqliteStore(path) | |
| 264 | expect(store.schemaError).toContain('not a database') | |
| 265 | expect(store.schemaError).toContain('not saving them') | |
| 266 | ||
| 267 | const id = await store.createRun({ chatGuid: 'g', messageGuid: 'm', prompt: 'still works', cwd: '/tmp', status: 'queued' }) | |
| 268 | expect(store.recentRuns()[0]).toMatchObject({ id, prompt: 'still works' }) | |
| 269 | store.close() | |
| 270 | }) | |
| 271 | ||
| 272 | it('leaves a half-applied change out of the database entirely', () => { | |
| 273 | const db = new Database(':memory:') | |
| 274 | db.run('CREATE TABLE runs (id INTEGER PRIMARY KEY)') | |
| 275 | // UNIQUE is one SQLite never allows in an ALTER, so this stands in for a | |
| 276 | // migration that fails partway: the table before it was created first. | |
| 277 | expect(() => migrate(db, [ | |
| 278 | { name: 'created_first', columns: { a: 'TEXT' } }, | |
| 279 | { name: 'runs', columns: { id: 'INTEGER PRIMARY KEY', nope: 'TEXT UNIQUE' } }, | |
| 280 | ])).toThrow() | |
| 281 | expect(db.query('SELECT name FROM sqlite_master WHERE type = ?').all('table')).toEqual([{ name: 'runs' }]) | |
| 282 | expect(schemaVersion(db)).toBe(0) | |
| 283 | db.close() | |
| 284 | }) | |
| 285 | }) | |
| 286 | ||
| 67 | 287 | describe('settings', () => { |
| 68 | 288 | it('fills in defaults and survives a broken file', () => { |
| 69 | 289 | const path = join(tempDir(), 'settings.json') |