ReviewOS

also looking at this

stacks/uplink

fix(app): an installed database gains the columns a new build needs

#43
Merged glennmichael123 wants to merge fix/sqlite-store-migrations into main
4 files +440 -37
app/Uplink/sqlite-store.tsmodified+202-35
Changes to app/Uplink/sqlite-store.ts
@@ -1,52 +1,107 @@
11import type { ConversationRecord, RunRecord, Store } from './store'
22import { Database } from 'bun:sqlite'
33import { mkdirSync } from 'node:fs'
4import { homedir } from 'node:os'
45import { dirname } from 'node:path'
56
67/**
78 * The store the downloadable app uses: one SQLite file in
8 * ~/Library/Application Support/Uplink, no framework, no migrations to run.
9 * ~/Library/Application Support/Uplink, no framework and no migration runner.
910 *
1011 * The Stacks app keeps its model-backed store (`model-store.ts`), whose tables
1112 * the dashboard reads. This one exists because a downloaded Uplink.app has no
1213 * Stacks project around it. Column names and units match the models - cost is
1314 * integer cents - so the two stay describable in one sentence.
15 *
16 * `CREATE TABLE IF NOT EXISTS` does nothing on a Mac that already has the
17 * file, so a column added to a schema string would never reach an installed
18 * copy: the first text after the update would fail on the INSERT, for every
19 * existing user, on every message. That is why the tables are described as
20 * data below rather than as one block of SQL. The same description creates a
21 * new database and tells an old one which columns it is missing.
22 *
23 * `TABLES` is not the only place a `runs` column is named: `RUN_COLUMNS` maps
24 * it to its field and `recentRuns` reads it back. TypeScript ties
25 * `RUN_COLUMNS` to `RunRecord` but cannot tie either to `TABLES`, so a test
26 * does it instead. Adding a column means editing both, and the suite says so
27 * rather than a customer's Mac.
1428 */
1529
1630export interface RunRow extends RunRecord {
1731 id: number
1832}
1933
20const SCHEMA = `
21 CREATE TABLE IF NOT EXISTS state (key TEXT PRIMARY KEY, value TEXT NOT NULL);
22 CREATE TABLE IF NOT EXISTS conversations (
23 chat_guid TEXT PRIMARY KEY,
24 handle TEXT NOT NULL,
25 service TEXT NOT NULL DEFAULT 'iMessage',
26 session_id TEXT,
27 cwd TEXT,
28 last_active_at INTEGER,
29 more_text TEXT
30 );
31 CREATE TABLE IF NOT EXISTS runs (
32 id INTEGER PRIMARY KEY AUTOINCREMENT,
33 chat_guid TEXT NOT NULL,
34 message_guid TEXT NOT NULL,
35 prompt TEXT NOT NULL,
36 cwd TEXT,
37 status TEXT NOT NULL DEFAULT 'queued',
38 reply TEXT,
39 error TEXT,
40 session_id TEXT,
41 cost_cents INTEGER,
42 duration_ms INTEGER,
43 last_activity TEXT,
44 started_at INTEGER,
45 finished_at INTEGER
46 );
47`
48
49const RUN_COLUMNS: Record<keyof RunRecord, string> = {
34/**
35 * Stamped into `PRAGMA user_version`. Nothing branches on it today: the
36 * columns themselves are reconciled. It is recorded so a database can say how
37 * old it is, and so the day a change needs more than a new column - a backfill,
38 * a rename - has a version to key off.
39 */
40const SCHEMA_VERSION = 1
41
42export interface TableSpec {
43 name: string
44 /**
45 * Definitions in creation order. A column added here later is also added to
46 * databases that already exist, so it has to be one SQLite can ALTER in:
47 * nullable or carrying a constant default, and never PRIMARY KEY or UNIQUE.
48 *
49 * Only missing columns are added. Changing a definition here, a default or a
50 * type, reaches new databases and no existing one, which is the original bug
51 * in a quieter form. A change like that needs its own step, not an edit.
52 *
53 * Worth knowing before trusting a green test: SQLite only enforces the
54 * NOT NULL and constant-default halves once a table has rows. An empty one
55 * accepts both. So a new column can pass on a fresh database and fail on
56 * every Mac that has been running, which is this file's own bug wearing a
57 * different hat. `tests/unit/uplink/sqlite-store.test.ts` migrates a
58 * populated database for that reason.
59 */
60 columns: Record<string, string>
61}
62
63export const TABLES: TableSpec[] = [
64 {
65 name: 'state',
66 columns: {
67 key: 'TEXT PRIMARY KEY',
68 value: 'TEXT NOT NULL',
69 },
70 },
71 {
72 name: 'conversations',
73 columns: {
74 chat_guid: 'TEXT PRIMARY KEY',
75 handle: 'TEXT NOT NULL',
76 service: 'TEXT NOT NULL DEFAULT \'iMessage\'',
77 session_id: 'TEXT',
78 cwd: 'TEXT',
79 last_active_at: 'INTEGER',
80 more_text: 'TEXT',
81 },
82 },
83 {
84 name: 'runs',
85 columns: {
86 id: 'INTEGER PRIMARY KEY AUTOINCREMENT',
87 chat_guid: 'TEXT NOT NULL',
88 message_guid: 'TEXT NOT NULL',
89 prompt: 'TEXT NOT NULL',
90 cwd: 'TEXT',
91 status: 'TEXT NOT NULL DEFAULT \'queued\'',
92 reply: 'TEXT',
93 error: 'TEXT',
94 session_id: 'TEXT',
95 cost_cents: 'INTEGER',
96 duration_ms: 'INTEGER',
97 last_activity: 'TEXT',
98 started_at: 'INTEGER',
99 finished_at: 'INTEGER',
100 },
101 },
102]
103
104export const RUN_COLUMNS: Record<keyof RunRecord, string> = {
50105 chatGuid: 'chat_guid',
51106 messageGuid: 'message_guid',
52107 prompt: 'prompt',
@@ -62,14 +117,98 @@ const RUN_COLUMNS: Record<keyof RunRecord, string> = {
62117 finishedAt: 'finished_at',
63118}
64119
120/**
121 * Bring a database up to the shape `TABLES` describes: create the tables it
122 * does not have, and add to the ones it does any column it is missing.
123 *
124 * Forward only, and idempotent, so it runs on every start. The work is decided
125 * by reading first, and a database that is already current returns before the
126 * transaction opens. That matters because opening one takes a write lock, and
127 * during an update two Uplinks hold the same file for a moment: a start that
128 * wrote on every launch would lose that race, and the caller answers a lost
129 * race by abandoning the file. Almost every launch is the no-work case.
130 *
131 * When there is work it is one transaction, which in SQLite covers the
132 * `ALTER TABLE`s and the version stamp together: a database is either fully
133 * updated or untouched, never left halfway by a Mac that went to sleep.
134 */
135export function migrate(db: Database, tables: TableSpec[] = TABLES): void {
136 if (isCurrent(db, tables))
137 return
138
139 db.transaction(() => {
140 for (const table of tables) {
141 const present = columnsOf(db, table.name)
142 const columns = Object.entries(table.columns)
143
144 if (present.size === 0) {
145 db.run(`CREATE TABLE ${table.name} (${columns.map(([name, definition]) => `${name} ${definition}`).join(', ')})`)
146 continue
147 }
148 for (const [name, definition] of columns) {
149 if (!present.has(name))
150 db.run(`ALTER TABLE ${table.name} ADD COLUMN ${name} ${definition}`)
151 }
152 }
153 // Interpolated, not bound: PRAGMA takes no parameter.
154 db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`)
155 })()
156}
157
158/**
159 * Whether the database already has every table and column `tables` names, and
160 * carries this build's version. Reads only, so it is safe to ask while another
161 * process is writing.
162 */
163function isCurrent(db: Database, tables: TableSpec[]): boolean {
164 if (schemaVersion(db) !== SCHEMA_VERSION)
165 return false
166 return tables.every((table) => {
167 const present = columnsOf(db, table.name)
168 return present.size > 0 && Object.keys(table.columns).every(name => present.has(name))
169 })
170}
171
172/** Empty means the table does not exist, which is also how a new file answers. */
173function columnsOf(db: Database, table: string): Set<string> {
174 return new Set((db.query(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>)
175 .map(column => column.name))
176}
177
178export function schemaVersion(db: Database): number {
179 return (db.query('PRAGMA user_version').get() as { user_version: number }).user_version
180}
181
65182export class SqliteStore implements Store {
66183 private db: Database
67184
185 /**
186 * Why this Uplink is running without its database, if it is.
187 *
188 * The store is built at the top of the desktop agent, under launchd, where
189 * an uncaught throw is a restart loop and the menubar never appears to say
190 * why. A corrupt file, a full disk or a migration SQLite refuses would all
191 * end there. Answering texts without remembering them is worth more than an
192 * app that is not running, so the file is abandoned for an in-memory
193 * database and the popover explains it.
194 */
195 readonly schemaError: string | null = null
196
68197 constructor(path: string) {
69 mkdirSync(dirname(path), { recursive: true })
70 this.db = new Database(path, { create: true })
71 this.db.exec('PRAGMA journal_mode = WAL;')
72 this.db.exec(SCHEMA)
198 try {
199 mkdirSync(dirname(path), { recursive: true })
200 this.db = open(path)
201 }
202 catch (error) {
203 // Written for whoever reads it in the popover, not for whoever wrote
204 // this: what is happening, and the one thing they can do about it.
205 this.schemaError = [
206 'Uplink is answering texts but not saving them.',
207 `Its history file could not be opened (${message(error)}).`,
208 `Quit and reopen Uplink to try again, or delete ${tilde(path)} to start a fresh history.`,
209 ].join(' ')
210 this.db = open(':memory:')
211 }
73212 }
74213
75214 close(): void {
@@ -147,3 +286,31 @@ export class SqliteStore implements Store {
147286 }))
148287 }
149288}
289
290function open(path: string): Database {
291 const db = new Database(path, { create: true })
292 try {
293 // A second Uplink on the same file is the normal case during an update.
294 // WAL is what lets it read while this one writes, and the timeout is what
295 // stops the one real upgrade from failing because the other one had the
296 // lock for a moment. Without it SQLite gives up at once and the caller
297 // reads that as a broken file.
298 db.exec('PRAGMA journal_mode = WAL;')
299 db.exec('PRAGMA busy_timeout = 5000;')
300 migrate(db)
301 return db
302 }
303 catch (error) {
304 db.close()
305 throw error
306 }
307}
308
309function message(error: unknown): string {
310 return error instanceof Error ? error.message : String(error)
311}
312
313function tilde(path: string): string {
314 const home = homedir()
315 return path.startsWith(`${home}/`) ? `~${path.slice(home.length)}` : path
316}