also looking at this
fix(fleet): carry the on-box database when a site moves
#178
4 files
+295
-0
| @@ -202,6 +202,37 @@ command. Background units are enabled but not started on the target until the | ||
| 202 | 202 | source is drained, so the two boxes can never both run a scheduler against one |
| 203 | 203 | dataset. |
| 204 | 204 | |
| 205 | #### The database | |
| 206 | ||
| 207 | A **SQLite** database rides along in the tree: it lives under `shared/`, which is | |
| 208 | the whole point of `sharedPaths`. An **external** database (RDS, a managed host) | |
| 209 | needs nothing either — the target reaches the same endpoint the source did. | |
| 210 | ||
| 211 | An **on-box Postgres or MySQL** database is different. It lives in the engine's | |
| 212 | own data directory, which belongs to the box rather than to the site, so the move | |
| 213 | carries it explicitly: dump on the source while background work is stopped, carry | |
| 214 | the file, then create the role and database on the target and load it — using the | |
| 215 | same dump, setup, and restore scripts `cloud db:backup`/`db:restore` and | |
| 216 | provisioning use, so a moved database is built exactly like a provisioned one. | |
| 217 | ||
| 218 | The dump is loaded **before** the app starts on the target, so its first request | |
| 219 | finds its data. | |
| 220 | ||
| 221 | Two rules keep this safe: | |
| 222 | ||
| 223 | - If the project has an on-box database and the move has no way to carry it, the | |
| 224 | move **refuses to run**. Moving the tree alone would pass every check in the | |
| 225 | plan — the app starts, answers its health gate, takes the DNS cutover — and | |
| 226 | then serve production an empty database. | |
| 227 | - If the target already has a database of that name **with tables in it**, the | |
| 228 | move refuses. Loading a dump over someone else's data is the one genuinely | |
| 229 | destructive thing this operation could do, so it is a precondition the operator | |
| 230 | resolves rather than a step behind a confirmation flag. | |
| 231 | ||
| 232 | Like the tree snapshot, the dump is re-taken on a resume rather than skipped: one | |
| 233 | from an earlier attempt predates whatever the source has committed since, and | |
| 234 | shipping stale rows is worse than dumping twice. | |
| 235 | ||
| 205 | 236 | The archive travels through the machine running the command rather than directly |
| 206 | 237 | between the boxes: a direct hop would need the target to hold a credential for |
| 207 | 238 | the source, which is the same credential-radius problem consolidation already |
| @@ -5,11 +5,15 @@ import type { SiteMoveEffects } from '../../src/operations/site-move' | ||
| 5 | 5 | import { existsSync } from 'node:fs' |
| 6 | 6 | import { readFile, writeFile } from 'node:fs/promises' |
| 7 | 7 | import * as cli from '../../src/utils/cli' |
| 8 | import { resolveAppDatabase } from '@ts-cloud/core' | |
| 8 | 9 | import { initializeDashboardControlPlane } from '../../src/deploy/dashboard-control-plane' |
| 9 | 10 | import { createDnsProvider } from '../../src/dns' |
| 10 | 11 | import { normalizePublicIpv6, reconcileAddressRecords, verifyAddressRecord } from '../../src/deploy/server-dns' |
| 11 | 12 | import { addSiteToCloudConfig } from '../../src/deploy/site-config-editor' |
| 12 | 13 | import { siteInstallBase } from '../../src/deploy/site-target' |
| 14 | import { buildBackupScript } from '../../src/deploy/dashboard-database' | |
| 15 | import { buildDatabaseSetupScript, isLocalDatabase } from '../../src/drivers/shared/db-provision' | |
| 16 | import { buildBackupRestoreScript } from '../../src/drivers/shared/backups' | |
| 13 | 17 | import { buildRpxConfig, buildRpxFragmentRefreshScript } from '../../src/drivers/shared/rpx-gateway' |
| 14 | 18 | import { FleetStore, SystemFleetSshTransport } from '../../src/fleet' |
| 15 | 19 | import { applyPlan, formatPlan, resolvePlan } from '../../src/operations/plan' |
| @@ -114,6 +118,16 @@ async function runSiteMove(siteName: string, options: SiteMoveCommandOptions): P | ||
| 114 | 118 | const zoneFor = (fqdn: string): string => |
| 115 | 119 | configuredZone && fqdn.endsWith(configuredZone) ? configuredZone : fqdn.split('.').slice(-2).join('.') |
| 116 | 120 | |
| 121 | // An ON-BOX engine database has to be carried separately: it lives in the | |
| 122 | // engine's data directory, not in the site tree. An external one needs | |
| 123 | // nothing — the target reaches the same endpoint the source did. SQLite | |
| 124 | // needs nothing either; it is under `shared/` and rides along in the tree. | |
| 125 | const appDatabase = resolveAppDatabase(config) | |
| 126 | const onBoxDatabase = | |
| 127 | appDatabase?.name && isLocalDatabase(appDatabase) ? { name: appDatabase.name } : undefined | |
| 128 | const dumpPath = `/tmp/ts-cloud-move-${slug}-${siteName}.sql.gz` | |
| 129 | const engine = (appDatabase?.engine ?? 'mysql') as 'mysql' | 'mariadb' | 'postgres' | |
| 130 | ||
| 117 | 131 | const effects: SiteMoveEffects = { |
| 118 | 132 | runOnSource: (script) => execOn(transport, source, script), |
| 119 | 133 | runOnTarget: (script) => execOn(transport, target, script), |
| @@ -165,6 +179,60 @@ async function runSiteMove(siteName: string, options: SiteMoveCommandOptions): P | ||
| 165 | 179 | ) |
| 166 | 180 | return published ? target.endpoint : undefined |
| 167 | 181 | }, |
| 182 | ...(onBoxDatabase | |
| 183 | ? { | |
| 184 | database: { | |
| 185 | // The SAME dump the backup command takes, written to a known path | |
| 186 | // so the transfer and the restore can find it without parsing. | |
| 187 | dump: async () => { | |
| 188 | await execOn( | |
| 189 | transport, | |
| 190 | source, | |
| 191 | [ | |
| 192 | 'set -euo pipefail', | |
| 193 | 'eval "$(cd /opt/pantry && pantry env 2>/dev/null)" || true', | |
| 194 | ...buildBackupScript(engine, onBoxDatabase.name, '/tmp', appDatabase), | |
| 195 | `mv -f "$(ls -1t /tmp/${onBoxDatabase.name}-*.sql.gz | head -1)" ${dumpPath}`, | |
| 196 | ].join('\n'), | |
| 197 | ) | |
| 198 | }, | |
| 199 | dumpStaged: async () => { | |
| 200 | const result = await transport.exec(target, `test -s ${dumpPath} && echo staged || true`) | |
| 201 | return result.stdout.includes('staged') | |
| 202 | }, | |
| 203 | transferDump: async () => { | |
| 204 | const local = `${process.cwd()}/.ts-cloud-move-${slug}-${siteName}.sql.gz` | |
| 205 | await copyFile(source, `${source.sshUser}@${source.endpoint}:${dumpPath}`, local) | |
| 206 | await copyFile(target, local, `${target.sshUser}@${target.endpoint}:${dumpPath}`) | |
| 207 | await Bun.file(local).delete().catch(() => {}) | |
| 208 | }, | |
| 209 | restore: async () => { | |
| 210 | await execOn( | |
| 211 | transport, | |
| 212 | target, | |
| 213 | [ | |
| 214 | // Create the role + database first, with the SAME idempotent | |
| 215 | // script provisioning runs, then load the dump into it. | |
| 216 | ...buildDatabaseSetupScript(appDatabase, config.infrastructure?.compute?.managedServices ?? {}), | |
| 217 | ...buildBackupRestoreScript(appDatabase, { from: dumpPath }), | |
| 218 | `rm -f ${dumpPath}`, | |
| 219 | ].join('\n'), | |
| 220 | ) | |
| 221 | }, | |
| 222 | targetHasData: async () => { | |
| 223 | const query = | |
| 224 | engine === 'postgres' | |
| 225 | ? `psql -tAc "select count(*) from information_schema.tables where table_schema='public'" -d ${onBoxDatabase.name} 2>/dev/null || echo 0` | |
| 226 | : `mysql -N -B -e "select count(*) from information_schema.tables where table_schema='${onBoxDatabase.name}'" 2>/dev/null || echo 0` | |
| 227 | const result = await transport.exec( | |
| 228 | target, | |
| 229 | `eval "$(cd /opt/pantry && pantry env 2>/dev/null)" || true\n${query}`, | |
| 230 | ) | |
| 231 | return Number.parseInt(result.stdout.trim().split('\n').pop() ?? '0', 10) > 0 | |
| 232 | }, | |
| 233 | }, | |
| 234 | } | |
| 235 | : {}), | |
| 168 | 236 | cutoverDns: async () => { |
| 169 | 237 | if (!domain) return [] |
| 170 | 238 | if (!dnsName) return [`No DNS provider configured — point ${domain} at ${target.endpoint} manually.`] |
| @@ -191,6 +259,7 @@ async function runSiteMove(siteName: string, options: SiteMoveCommandOptions): P | ||
| 191 | 259 | targetAddress: target.endpoint, |
| 192 | 260 | port: site.port, |
| 193 | 261 | healthCheckPath: site.healthCheck?.path, |
| 262 | database: onBoxDatabase, | |
| 194 | 263 | }, |
| 195 | 264 | effects, |
| 196 | 265 | ) |
| @@ -285,3 +285,101 @@ describe('remote scripts', () => { | ||
| 285 | 285 | expect(siteMoveArchivePath('hq', 'bughq')).toBe('/tmp/ts-cloud-move-hq-bughq.tar.gz') |
| 286 | 286 | }) |
| 287 | 287 | }) |
| 288 | ||
| 289 | /** | |
| 290 | * A SQLite database rides along in the tree — it lives under `shared/`, which is | |
| 291 | * the whole point of `sharedPaths`. A Postgres or MySQL database does not: it | |
| 292 | * lives in the engine's own data directory, which belongs to the box rather than | |
| 293 | * the site. Moving the tree alone would pass every check in the plan and then | |
| 294 | * serve production an empty database. | |
| 295 | */ | |
| 296 | describe('planSiteMove with an on-box database', () => { | |
| 297 | function dbWorld() { | |
| 298 | const base = world() | |
| 299 | const db = { dumped: false, staged: false, restored: false, targetData: false } | |
| 300 | const effects = { | |
| 301 | ...base.effects, | |
| 302 | database: { | |
| 303 | dump: async () => { db.dumped = true }, | |
| 304 | transferDump: async () => { db.staged = true }, | |
| 305 | dumpStaged: async () => db.staged, | |
| 306 | restore: async () => { db.restored = true }, | |
| 307 | targetHasData: async () => db.targetData, | |
| 308 | }, | |
| 309 | } | |
| 310 | return { state: base.state, db, effects } | |
| 311 | } | |
| 312 | ||
| 313 | const withDb = { ...options, database: { name: 'bughq' } } | |
| 314 | ||
| 315 | it('refuses the move when there is no way to carry the database', async () => { | |
| 316 | await expect(planSiteMove(withDb, world().effects)).rejects.toThrow('no way to carry it') | |
| 317 | }) | |
| 318 | ||
| 319 | /** Loading a dump over data already there is the one destructive thing here. */ | |
| 320 | it('refuses when the target already holds a database of that name', async () => { | |
| 321 | const { db, effects } = dbWorld() | |
| 322 | db.targetData = true | |
| 323 | await expect(planSiteMove(withDb, effects)).rejects.toThrow('already has a database') | |
| 324 | }) | |
| 325 | ||
| 326 | it('carries the database, and loads it before the app starts', async () => { | |
| 327 | const p = await planSiteMove(withDb, dbWorld().effects) | |
| 328 | const ids = p.steps.map(step => step.id) | |
| 329 | expect(ids).toEqual([ | |
| 330 | 'pause-workers', | |
| 331 | 'database-dump', | |
| 332 | 'snapshot', | |
| 333 | 'transfer', | |
| 334 | 'database-transfer', | |
| 335 | 'database-restore', | |
| 336 | 'restore', | |
| 337 | 'health', | |
| 338 | 'gateway', | |
| 339 | 'dns', | |
| 340 | 'drain-source', | |
| 341 | ]) | |
| 342 | // The dump is taken while background work is stopped, and loaded before the | |
| 343 | // app on the target can serve a request against it. | |
| 344 | expect(ids.indexOf('database-dump')).toBeGreaterThan(ids.indexOf('pause-workers')) | |
| 345 | expect(ids.indexOf('database-restore')).toBeLessThan(ids.indexOf('restore')) | |
| 346 | }) | |
| 347 | ||
| 348 | it('moves the data end to end', async () => { | |
| 349 | const { state, db, effects } = dbWorld() | |
| 350 | const p = await planSiteMove(withDb, effects) | |
| 351 | expect((await applyPlan(p, await resolvePlan(p))).success).toBe(true) | |
| 352 | expect(db).toMatchObject({ dumped: true, staged: true, restored: true }) | |
| 353 | expect(state.sourceRunning).toBe(false) | |
| 354 | }) | |
| 355 | ||
| 356 | /** A dump from an earlier attempt predates what the source has committed since. */ | |
| 357 | it('re-dumps on a resume rather than shipping stale rows', async () => { | |
| 358 | const { effects } = dbWorld() | |
| 359 | const p = await planSiteMove(withDb, effects) | |
| 360 | await applyPlan(p, await resolvePlan(p)) | |
| 361 | const again = await resolvePlan(await planSiteMove(withDb, effects)) | |
| 362 | expect(again.find(item => item.step.id === 'database-dump')?.state).toBe('pending') | |
| 363 | // The transfer, though, resumes — the dump is already staged. | |
| 364 | expect(again.find(item => item.step.id === 'database-transfer')?.state).toBe('satisfied') | |
| 365 | }) | |
| 366 | ||
| 367 | /** An external database is reached at the same endpoint from either box. */ | |
| 368 | it('adds no database steps when the project has no on-box database', async () => { | |
| 369 | const p = await planSiteMove(options, world().effects) | |
| 370 | expect(p.steps.map(step => step.id).some(id => id.startsWith('database-'))).toBe(false) | |
| 371 | }) | |
| 372 | ||
| 373 | it('leaves the source serving when the restore fails', async () => { | |
| 374 | const { state, effects } = dbWorld() | |
| 375 | const p = await planSiteMove(withDb, { | |
| 376 | ...effects, | |
| 377 | database: { ...effects.database, restore: async () => { throw new Error('role does not exist') } }, | |
| 378 | }) | |
| 379 | const outcome = await applyPlan(p, await resolvePlan(p)) | |
| 380 | expect(outcome.success).toBe(false) | |
| 381 | expect(outcome.steps.at(-1)?.id).toBe('database-restore') | |
| 382 | expect(state.published).toBe('203.0.113.1') | |
| 383 | expect(state.sourceRunning).toBe(true) | |
| 384 | }) | |
| 385 | }) | |
| @@ -243,6 +243,41 @@ export interface SiteMoveEffects { | ||
| 243 | 243 | refreshTargetGateway: () => Promise<void> |
| 244 | 244 | /** Does the target's gateway already route this site? */ |
| 245 | 245 | targetRoutesSite: () => Promise<boolean> |
| 246 | /** | |
| 247 | * How to carry an ON-BOX database, when the project has one. | |
| 248 | * | |
| 249 | * A SQLite database rides along in the tree already — it lives under | |
| 250 | * `shared/`, which is the whole point of `sharedPaths`. An engine database | |
| 251 | * does not: Postgres and MySQL keep their data in the engine's own directory, | |
| 252 | * which belongs to the box rather than to the site. Moving the tree and not | |
| 253 | * the database would cut DNS over to a target serving an empty app, so the | |
| 254 | * plan REFUSES to be built when a project has one of these and no way to | |
| 255 | * carry it — see {@link SiteMoveOptions.database}. | |
| 256 | * | |
| 257 | * Absent when the database is external (RDS, a managed host): the target | |
| 258 | * connects to the same endpoint the source did, and there is nothing to move. | |
| 259 | */ | |
| 260 | database?: SiteMoveDatabaseEffects | |
| 261 | } | |
| 262 | ||
| 263 | /** Carrying an on-box engine database between boxes, in resumable pieces. */ | |
| 264 | export interface SiteMoveDatabaseEffects { | |
| 265 | /** Dump it on the source. */ | |
| 266 | dump: () => Promise<void> | |
| 267 | /** Carry the dump to the target. */ | |
| 268 | transferDump: () => Promise<void> | |
| 269 | /** Is the dump already staged on the target? Lets the transfer resume. */ | |
| 270 | dumpStaged: () => Promise<boolean> | |
| 271 | /** Create the role + database on the target, then load the dump into it. */ | |
| 272 | restore: () => Promise<void> | |
| 273 | /** | |
| 274 | * Does the target ALREADY hold a database of this name with tables in it? | |
| 275 | * | |
| 276 | * Checked as a precondition, not as a step: loading a dump over someone | |
| 277 | * else's data is the one genuinely destructive thing a move could do, and the | |
| 278 | * answer decides whether the operation may run at all. | |
| 279 | */ | |
| 280 | targetHasData: () => Promise<boolean> | |
| 246 | 281 | } |
| 247 | 282 | |
| 248 | 283 | export interface SiteMoveOptions { |
| @@ -259,6 +294,13 @@ export interface SiteMoveOptions { | ||
| 259 | 294 | port?: number |
| 260 | 295 | /** Health-check path, defaulting to `/`. */ |
| 261 | 296 | healthCheckPath?: string |
| 297 | /** | |
| 298 | * The on-box engine database this site's project owns, when it has one — | |
| 299 | * `resolveAppDatabase(config)` narrowed by `isLocalDatabase`. Naming it here | |
| 300 | * is what makes the move refuse to run without a way to carry it, rather than | |
| 301 | * moving the app and quietly leaving its data behind. | |
| 302 | */ | |
| 303 | database?: { name: string } | |
| 262 | 304 | } |
| 263 | 305 | |
| 264 | 306 | /** |
| @@ -273,6 +315,27 @@ export async function planSiteMove(options: SiteMoveOptions, effects: SiteMoveEf | ||
| 273 | 315 | |
| 274 | 316 | if (from === to) throw new Error(`'${siteName}' is already on ${to}.`) |
| 275 | 317 | |
| 318 | // A project with an on-box engine database and no way to carry it must not | |
| 319 | // move at all. Moving the tree alone would pass every check in this plan — | |
| 320 | // the app starts, answers its health gate, takes the DNS cutover — and serve | |
| 321 | // an empty database to production. Refusing here is the only honest answer. | |
| 322 | if (options.database && !effects.database) | |
| 323 | throw new Error( | |
| 324 | `'${siteName}' uses the on-box database '${options.database.name}', which lives in the engine's own ` | |
| 325 | + 'data directory rather than in the site tree, and this move has no way to carry it. ' | |
| 326 | + 'Moving the site without it would cut traffic over to an empty database.', | |
| 327 | ) | |
| 328 | ||
| 329 | // Loading a dump over data already on the target is the one genuinely | |
| 330 | // destructive thing this operation could do, so it is a precondition something | |
| 331 | // the operator resolves, not a step with a confirmation flag on it. | |
| 332 | if (options.database && effects.database && (await effects.database.targetHasData())) | |
| 333 | throw new Error( | |
| 334 | `${to} already has a database named '${options.database.name}' with tables in it. ` | |
| 335 | + 'Restoring this site\'s dump over it would destroy that data. ' | |
| 336 | + 'Rename or drop it on the target first, or point this project at a different database name.', | |
| 337 | ) | |
| 338 | ||
| 276 | 339 | const steps: OperationStep[] = [ |
| 277 | 340 | { |
| 278 | 341 | id: 'pause-workers', |
| @@ -282,6 +345,20 @@ export async function planSiteMove(options: SiteMoveOptions, effects: SiteMoveEf | ||
| 282 | 345 | await effects.runOnSource(buildPauseWorkersScript(slug, siteName)) |
| 283 | 346 | }, |
| 284 | 347 | }, |
| 348 | ...(effects.database | |
| 349 | ? [ | |
| 350 | { | |
| 351 | id: 'database-dump', | |
| 352 | title: `Dump the '${options.database?.name}' database on ${from}`, | |
| 353 | // Never skipped, for the same reason the tree snapshot is not: a | |
| 354 | // dump from an earlier attempt predates whatever the source has | |
| 355 | // committed since, and shipping stale rows is worse than dumping | |
| 356 | // twice. | |
| 357 | satisfied: async () => false, | |
| 358 | apply: () => effects.database!.dump(), | |
| 359 | } satisfies OperationStep, | |
| 360 | ] | |
| 361 | : []), | |
| 285 | 362 | { |
| 286 | 363 | id: 'snapshot', |
| 287 | 364 | title: `Archive the site's tree and units on ${from}`, |
| @@ -301,6 +378,26 @@ export async function planSiteMove(options: SiteMoveOptions, effects: SiteMoveEf | ||
| 301 | 378 | satisfied: () => effects.archiveStaged(), |
| 302 | 379 | apply: () => effects.transferArchive(), |
| 303 | 380 | }, |
| 381 | ...(effects.database | |
| 382 | ? [ | |
| 383 | { | |
| 384 | id: 'database-transfer', | |
| 385 | title: `Carry the database dump to ${to}`, | |
| 386 | change: { from, to }, | |
| 387 | satisfied: () => effects.database!.dumpStaged(), | |
| 388 | apply: () => effects.database!.transferDump(), | |
| 389 | } satisfies OperationStep, | |
| 390 | { | |
| 391 | id: 'database-restore', | |
| 392 | title: `Create and load '${options.database?.name}' on ${to}`, | |
| 393 | // Before the app starts, so its first request finds its data — | |
| 394 | // and never skipped, because the precondition above already | |
| 395 | // established the target holds nothing this could overwrite. | |
| 396 | satisfied: async () => false, | |
| 397 | apply: () => effects.database!.restore(), | |
| 398 | } satisfies OperationStep, | |
| 399 | ] | |
| 400 | : []), | |
| 304 | 401 | { |
| 305 | 402 | id: 'restore', |
| 306 | 403 | title: `Unpack the site on ${to} and start it`, |