also looking at this
fix(app): an installed database gains the columns a new build needs
#43
4 files
+440
-37
| @@ -1,52 +1,107 @@ | ||
| 1 | 1 | import type { ConversationRecord, RunRecord, Store } from './store' |
| 2 | 2 | import { Database } from 'bun:sqlite' |
| 3 | 3 | import { mkdirSync } from 'node:fs' |
| 4 | import { homedir } from 'node:os' | |
| 4 | 5 | import { dirname } from 'node:path' |
| 5 | 6 | |
| 6 | 7 | /** |
| 7 | 8 | * The store the downloadable app uses: one SQLite file in |
| 8 | * ~/Library/Application Support/Uplink, no framework, no migrations to run. | |
| 9 | * ~/Library/Application Support/Uplink, no framework and no migration runner. | |
| 9 | 10 | * |
| 10 | 11 | * The Stacks app keeps its model-backed store (`model-store.ts`), whose tables |
| 11 | 12 | * the dashboard reads. This one exists because a downloaded Uplink.app has no |
| 12 | 13 | * Stacks project around it. Column names and units match the models - cost is |
| 13 | 14 | * integer cents - so the two stay describable in one sentence. |
| 15 | * | |
| 16 | * `CREATE TABLE IF NOT EXISTS` does nothing on a Mac that already has the | |
| 17 | * file, so a column added to a schema string would never reach an installed | |
| 18 | * copy: the first text after the update would fail on the INSERT, for every | |
| 19 | * existing user, on every message. That is why the tables are described as | |
| 20 | * data below rather than as one block of SQL. The same description creates a | |
| 21 | * new database and tells an old one which columns it is missing. | |
| 22 | * | |
| 23 | * `TABLES` is not the only place a `runs` column is named: `RUN_COLUMNS` maps | |
| 24 | * it to its field and `recentRuns` reads it back. TypeScript ties | |
| 25 | * `RUN_COLUMNS` to `RunRecord` but cannot tie either to `TABLES`, so a test | |
| 26 | * does it instead. Adding a column means editing both, and the suite says so | |
| 27 | * rather than a customer's Mac. | |
| 14 | 28 | */ |
| 15 | 29 | |
| 16 | 30 | export interface RunRow extends RunRecord { |
| 17 | 31 | id: number |
| 18 | 32 | } |
| 19 | 33 | |
| 20 | const SCHEMA = ` | |
| 21 | CREATE TABLE IF NOT EXISTS state (key TEXT PRIMARY KEY, value TEXT NOT NULL); | |
| 22 | CREATE TABLE IF NOT EXISTS conversations ( | |
| 23 | chat_guid TEXT PRIMARY KEY, | |
| 24 | handle TEXT NOT NULL, | |
| 25 | service TEXT NOT NULL DEFAULT 'iMessage', | |
| 26 | session_id TEXT, | |
| 27 | cwd TEXT, | |
| 28 | last_active_at INTEGER, | |
| 29 | more_text TEXT | |
| 30 | ); | |
| 31 | CREATE TABLE IF NOT EXISTS runs ( | |
| 32 | id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 33 | chat_guid TEXT NOT NULL, | |
| 34 | message_guid TEXT NOT NULL, | |
| 35 | prompt TEXT NOT NULL, | |
| 36 | cwd TEXT, | |
| 37 | status TEXT NOT NULL DEFAULT 'queued', | |
| 38 | reply TEXT, | |
| 39 | error TEXT, | |
| 40 | session_id TEXT, | |
| 41 | cost_cents INTEGER, | |
| 42 | duration_ms INTEGER, | |
| 43 | last_activity TEXT, | |
| 44 | started_at INTEGER, | |
| 45 | finished_at INTEGER | |
| 46 | ); | |
| 47 | ` | |
| 48 | ||
| 49 | const RUN_COLUMNS: Record<keyof RunRecord, string> = { | |
| 34 | /** | |
| 35 | * Stamped into `PRAGMA user_version`. Nothing branches on it today: the | |
| 36 | * columns themselves are reconciled. It is recorded so a database can say how | |
| 37 | * old it is, and so the day a change needs more than a new column - a backfill, | |
| 38 | * a rename - has a version to key off. | |
| 39 | */ | |
| 40 | const SCHEMA_VERSION = 1 | |
| 41 | ||
| 42 | export interface TableSpec { | |
| 43 | name: string | |
| 44 | /** | |
| 45 | * Definitions in creation order. A column added here later is also added to | |
| 46 | * databases that already exist, so it has to be one SQLite can ALTER in: | |
| 47 | * nullable or carrying a constant default, and never PRIMARY KEY or UNIQUE. | |
| 48 | * | |
| 49 | * Only missing columns are added. Changing a definition here, a default or a | |
| 50 | * type, reaches new databases and no existing one, which is the original bug | |
| 51 | * in a quieter form. A change like that needs its own step, not an edit. | |
| 52 | * | |
| 53 | * Worth knowing before trusting a green test: SQLite only enforces the | |
| 54 | * NOT NULL and constant-default halves once a table has rows. An empty one | |
| 55 | * accepts both. So a new column can pass on a fresh database and fail on | |
| 56 | * every Mac that has been running, which is this file's own bug wearing a | |
| 57 | * different hat. `tests/unit/uplink/sqlite-store.test.ts` migrates a | |
| 58 | * populated database for that reason. | |
| 59 | */ | |
| 60 | columns: Record<string, string> | |
| 61 | } | |
| 62 | ||
| 63 | export const TABLES: TableSpec[] = [ | |
| 64 | { | |
| 65 | name: 'state', | |
| 66 | columns: { | |
| 67 | key: 'TEXT PRIMARY KEY', | |
| 68 | value: 'TEXT NOT NULL', | |
| 69 | }, | |
| 70 | }, | |
| 71 | { | |
| 72 | name: 'conversations', | |
| 73 | columns: { | |
| 74 | chat_guid: 'TEXT PRIMARY KEY', | |
| 75 | handle: 'TEXT NOT NULL', | |
| 76 | service: 'TEXT NOT NULL DEFAULT \'iMessage\'', | |
| 77 | session_id: 'TEXT', | |
| 78 | cwd: 'TEXT', | |
| 79 | last_active_at: 'INTEGER', | |
| 80 | more_text: 'TEXT', | |
| 81 | }, | |
| 82 | }, | |
| 83 | { | |
| 84 | name: 'runs', | |
| 85 | columns: { | |
| 86 | id: 'INTEGER PRIMARY KEY AUTOINCREMENT', | |
| 87 | chat_guid: 'TEXT NOT NULL', | |
| 88 | message_guid: 'TEXT NOT NULL', | |
| 89 | prompt: 'TEXT NOT NULL', | |
| 90 | cwd: 'TEXT', | |
| 91 | status: 'TEXT NOT NULL DEFAULT \'queued\'', | |
| 92 | reply: 'TEXT', | |
| 93 | error: 'TEXT', | |
| 94 | session_id: 'TEXT', | |
| 95 | cost_cents: 'INTEGER', | |
| 96 | duration_ms: 'INTEGER', | |
| 97 | last_activity: 'TEXT', | |
| 98 | started_at: 'INTEGER', | |
| 99 | finished_at: 'INTEGER', | |
| 100 | }, | |
| 101 | }, | |
| 102 | ] | |
| 103 | ||
| 104 | export const RUN_COLUMNS: Record<keyof RunRecord, string> = { | |
| 50 | 105 | chatGuid: 'chat_guid', |
| 51 | 106 | messageGuid: 'message_guid', |
| 52 | 107 | prompt: 'prompt', |
| @@ -62,14 +117,98 @@ const RUN_COLUMNS: Record<keyof RunRecord, string> = { | ||
| 62 | 117 | finishedAt: 'finished_at', |
| 63 | 118 | } |
| 64 | 119 | |
| 120 | /** | |
| 121 | * Bring a database up to the shape `TABLES` describes: create the tables it | |
| 122 | * does not have, and add to the ones it does any column it is missing. | |
| 123 | * | |
| 124 | * Forward only, and idempotent, so it runs on every start. The work is decided | |
| 125 | * by reading first, and a database that is already current returns before the | |
| 126 | * transaction opens. That matters because opening one takes a write lock, and | |
| 127 | * during an update two Uplinks hold the same file for a moment: a start that | |
| 128 | * wrote on every launch would lose that race, and the caller answers a lost | |
| 129 | * race by abandoning the file. Almost every launch is the no-work case. | |
| 130 | * | |
| 131 | * When there is work it is one transaction, which in SQLite covers the | |
| 132 | * `ALTER TABLE`s and the version stamp together: a database is either fully | |
| 133 | * updated or untouched, never left halfway by a Mac that went to sleep. | |
| 134 | */ | |
| 135 | export function migrate(db: Database, tables: TableSpec[] = TABLES): void { | |
| 136 | if (isCurrent(db, tables)) | |
| 137 | return | |
| 138 | ||
| 139 | db.transaction(() => { | |
| 140 | for (const table of tables) { | |
| 141 | const present = columnsOf(db, table.name) | |
| 142 | const columns = Object.entries(table.columns) | |
| 143 | ||
| 144 | if (present.size === 0) { | |
| 145 | db.run(`CREATE TABLE ${table.name} (${columns.map(([name, definition]) => `${name} ${definition}`).join(', ')})`) | |
| 146 | continue | |
| 147 | } | |
| 148 | for (const [name, definition] of columns) { | |
| 149 | if (!present.has(name)) | |
| 150 | db.run(`ALTER TABLE ${table.name} ADD COLUMN ${name} ${definition}`) | |
| 151 | } | |
| 152 | } | |
| 153 | // Interpolated, not bound: PRAGMA takes no parameter. | |
| 154 | db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`) | |
| 155 | })() | |
| 156 | } | |
| 157 | ||
| 158 | /** | |
| 159 | * Whether the database already has every table and column `tables` names, and | |
| 160 | * carries this build's version. Reads only, so it is safe to ask while another | |
| 161 | * process is writing. | |
| 162 | */ | |
| 163 | function isCurrent(db: Database, tables: TableSpec[]): boolean { | |
| 164 | if (schemaVersion(db) !== SCHEMA_VERSION) | |
| 165 | return false | |
| 166 | return tables.every((table) => { | |
| 167 | const present = columnsOf(db, table.name) | |
| 168 | return present.size > 0 && Object.keys(table.columns).every(name => present.has(name)) | |
| 169 | }) | |
| 170 | } | |
| 171 | ||
| 172 | /** Empty means the table does not exist, which is also how a new file answers. */ | |
| 173 | function columnsOf(db: Database, table: string): Set<string> { | |
| 174 | return new Set((db.query(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>) | |
| 175 | .map(column => column.name)) | |
| 176 | } | |
| 177 | ||
| 178 | export function schemaVersion(db: Database): number { | |
| 179 | return (db.query('PRAGMA user_version').get() as { user_version: number }).user_version | |
| 180 | } | |
| 181 | ||
| 65 | 182 | export class SqliteStore implements Store { |
| 66 | 183 | private db: Database |
| 67 | 184 | |
| 185 | /** | |
| 186 | * Why this Uplink is running without its database, if it is. | |
| 187 | * | |
| 188 | * The store is built at the top of the desktop agent, under launchd, where | |
| 189 | * an uncaught throw is a restart loop and the menubar never appears to say | |
| 190 | * why. A corrupt file, a full disk or a migration SQLite refuses would all | |
| 191 | * end there. Answering texts without remembering them is worth more than an | |
| 192 | * app that is not running, so the file is abandoned for an in-memory | |
| 193 | * database and the popover explains it. | |
| 194 | */ | |
| 195 | readonly schemaError: string | null = null | |
| 196 | ||
| 68 | 197 | constructor(path: string) { |
| 69 | mkdirSync(dirname(path), { recursive: true }) | |
| 70 | this.db = new Database(path, { create: true }) | |
| 71 | this.db.exec('PRAGMA journal_mode = WAL;') | |
| 72 | this.db.exec(SCHEMA) | |
| 198 | try { | |
| 199 | mkdirSync(dirname(path), { recursive: true }) | |
| 200 | this.db = open(path) | |
| 201 | } | |
| 202 | catch (error) { | |
| 203 | // Written for whoever reads it in the popover, not for whoever wrote | |
| 204 | // this: what is happening, and the one thing they can do about it. | |
| 205 | this.schemaError = [ | |
| 206 | 'Uplink is answering texts but not saving them.', | |
| 207 | `Its history file could not be opened (${message(error)}).`, | |
| 208 | `Quit and reopen Uplink to try again, or delete ${tilde(path)} to start a fresh history.`, | |
| 209 | ].join(' ') | |
| 210 | this.db = open(':memory:') | |
| 211 | } | |
| 73 | 212 | } |
| 74 | 213 | |
| 75 | 214 | close(): void { |
| @@ -147,3 +286,31 @@ export class SqliteStore implements Store { | ||
| 147 | 286 | })) |
| 148 | 287 | } |
| 149 | 288 | } |
| 289 | ||
| 290 | function open(path: string): Database { | |
| 291 | const db = new Database(path, { create: true }) | |
| 292 | try { | |
| 293 | // A second Uplink on the same file is the normal case during an update. | |
| 294 | // WAL is what lets it read while this one writes, and the timeout is what | |
| 295 | // stops the one real upgrade from failing because the other one had the | |
| 296 | // lock for a moment. Without it SQLite gives up at once and the caller | |
| 297 | // reads that as a broken file. | |
| 298 | db.exec('PRAGMA journal_mode = WAL;') | |
| 299 | db.exec('PRAGMA busy_timeout = 5000;') | |
| 300 | migrate(db) | |
| 301 | return db | |
| 302 | } | |
| 303 | catch (error) { | |
| 304 | db.close() | |
| 305 | throw error | |
| 306 | } | |
| 307 | } | |
| 308 | ||
| 309 | function message(error: unknown): string { | |
| 310 | return error instanceof Error ? error.message : String(error) | |
| 311 | } | |
| 312 | ||
| 313 | function tilde(path: string): string { | |
| 314 | const home = homedir() | |
| 315 | return path.startsWith(`${home}/`) ? `~${path.slice(home.length)}` : path | |
| 316 | } | |