also looking at this
fix(db): bound on-demand database dumps, and publish them atomically
#200
2 files
+188
-25
| @@ -428,17 +428,71 @@ export async function createDatabaseUser( | ||
| 428 | 428 | export const DB_BACKUP_DIR = '/var/backups/ts-cloud/databases' |
| 429 | 429 | |
| 430 | 430 | /** |
| 431 | * Script that dumps a single database to a timestamped, gzipped file. The name | |
| 432 | * is a validated SQL identifier (no shell metacharacters), so it is safe to | |
| 433 | * embed directly. The timestamp is computed on the box. Postgres connects over | |
| 434 | * the local unix socket for a co-located engine, or TCP with credentials for | |
| 435 | * an external host (see {@link pgAdminCommand}). | |
| 431 | * Local retention for on-box dumps, mirroring the bounds the scheduled path | |
| 432 | * already honours ({@link ComputeBackupConfig.retentionCount} / | |
| 433 | * {@link ComputeBackupConfig.retentionDays}). | |
| 434 | */ | |
| 435 | export interface DbBackupRetention { | |
| 436 | /** Keep the newest N dumps of this database. @default 5 */ | |
| 437 | count?: number | |
| 438 | /** Delete this database's dumps older than N days. @default 30 */ | |
| 439 | days?: number | |
| 440 | } | |
| 441 | ||
| 442 | /** Applied when `compute.backups` configures no bounds of its own. */ | |
| 443 | export const DEFAULT_DB_BACKUP_RETENTION = { count: 5, days: 30 } as const | |
| 444 | ||
| 445 | /** | |
| 446 | * Retention for an on-demand dump. Falls back to the defaults rather than to | |
| 447 | * "unbounded" even when scheduled backups are disabled: `compute.backups` turns | |
| 448 | * the *schedule* off, and a dashboard button that leaves dumps on the box | |
| 449 | * forever is the bug this resolves, not a documented mode. | |
| 450 | */ | |
| 451 | export function resolveDbBackupRetention(config: CloudConfig): DbBackupRetention { | |
| 452 | const backups = config.infrastructure?.compute?.backups | |
| 453 | return { | |
| 454 | count: backups?.retentionCount ?? DEFAULT_DB_BACKUP_RETENTION.count, | |
| 455 | days: backups?.retentionDays ?? DEFAULT_DB_BACKUP_RETENTION.days, | |
| 456 | } | |
| 457 | } | |
| 458 | ||
| 459 | /** | |
| 460 | * Prune one database's dumps. Scoped to `<name>-*.sql.gz` so a shared box does | |
| 461 | * not let one database's retention delete another's, and emitted only on the | |
| 462 | * success branch of {@link buildBackupScript}. | |
| 463 | */ | |
| 464 | function buildDbBackupRetentionScript(destDir: string, name: string, retention: DbBackupRetention): string[] { | |
| 465 | const count = Math.max(1, Math.floor(retention.count ?? DEFAULT_DB_BACKUP_RETENTION.count)) | |
| 466 | const days = Math.max(1, Math.floor(retention.days ?? DEFAULT_DB_BACKUP_RETENTION.days)) | |
| 467 | const pattern = `${name}-*.sql.gz` | |
| 468 | return [ | |
| 469 | // Newest first, so `tail -n +N+1` is everything past the newest N. The dump | |
| 470 | // just taken is the newest, so a count of at least 1 always keeps it. | |
| 471 | ` ls -1t ${destDir}/${pattern} 2>/dev/null | tail -n +${count + 1} | xargs -r rm -f`, | |
| 472 | ` find ${destDir} -xdev -maxdepth 1 -type f -name ${JSON.stringify(pattern)} -mtime +${days} -delete 2>/dev/null || true`, | |
| 473 | // Scratch files from a dump whose process was killed before either branch | |
| 474 | // ran. An hour bounds a concurrent dump that is still writing. | |
| 475 | ` find ${destDir} -xdev -maxdepth 1 -type f -name ${JSON.stringify(`.${name}-*.part`)} -mmin +60 -delete 2>/dev/null || true`, | |
| 476 | ] | |
| 477 | } | |
| 478 | ||
| 479 | /** | |
| 480 | * Script that dumps a single database to a timestamped, gzipped file, then | |
| 481 | * prunes older dumps of that database. The name is a validated SQL identifier | |
| 482 | * (no shell metacharacters), so it is safe to embed directly. The timestamp is | |
| 483 | * computed on the box. Postgres connects over the local unix socket for a | |
| 484 | * co-located engine, or TCP with credentials for an external host (see | |
| 485 | * {@link pgAdminCommand}). | |
| 486 | * | |
| 487 | * Pass `retention: false` for a caller that owns the destination directory's | |
| 488 | * lifetime itself (a dump staged for a one-shot move, say). | |
| 436 | 489 | */ |
| 437 | 490 | export function buildBackupScript( |
| 438 | 491 | engine: DbEngine, |
| 439 | 492 | name: string, |
| 440 | 493 | destDir: string = DB_BACKUP_DIR, |
| 441 | 494 | database?: DatabaseConfig, |
| 495 | retention: DbBackupRetention | false = {}, | |
| 442 | 496 | ): string[] { |
| 443 | 497 | // The timestamp is taken ONCE into a variable rather than embedded as a |
| 444 | 498 | // command substitution. Written inline, `$(date …)` runs again in every |
| @@ -448,22 +502,41 @@ export function buildBackupScript( | ||
| 448 | 502 | // find, which is the worst way for a backup to fail. |
| 449 | 503 | const stamp = 'TS_CLOUD_BACKUP_STAMP="$(date +%Y%m%d-%H%M%S)"' |
| 450 | 504 | const file = `${destDir}/${name}-$TS_CLOUD_BACKUP_STAMP.sql.gz` |
| 451 | const mkdir = `mkdir -p ${destDir}` | |
| 452 | if (engine === 'postgres') | |
| 453 | return [ | |
| 454 | mkdir, | |
| 455 | stamp, | |
| 456 | `${pgAdminCommand(database, 'pg_dump')} ${name} | gzip > "${file}"`, | |
| 457 | `echo "BACKUP=${file}"`, | |
| 458 | `ls -l "${file}"`, | |
| 459 | ] | |
| 505 | // Dump to a scratch name and rename only on success. Redirecting straight to | |
| 506 | // `${file}` opens (and truncates) the final name before the dump is known to | |
| 507 | // have worked, so two dumps of one database in the same second — the second | |
| 508 | // failing — destroyed the first: same stamp, same path. The stamp is only | |
| 509 | // second-granular, and a retry after a failure is exactly when that happens. | |
| 510 | // `.part` is outside the `*.sql.gz` glob, so a scratch file is never listed | |
| 511 | // as a backup nor counted by retention. | |
| 512 | const part = `${destDir}/.${name}-$TS_CLOUD_BACKUP_STAMP.part` | |
| 460 | 513 | const sock = engine === 'mariadb' ? SOCKETS.mariadb : SOCKETS.mysql |
| 514 | const dump = engine === 'postgres' | |
| 515 | ? `${pgAdminCommand(database, 'pg_dump')} ${name} | gzip > "${part}"` | |
| 516 | : `mysqldump --socket=${sock} -u root ${name} | gzip > "${part}"` | |
| 517 | ||
| 461 | 518 | return [ |
| 462 | mkdir, | |
| 519 | `mkdir -p ${destDir}`, | |
| 463 | 520 | stamp, |
| 464 | `mysqldump --socket=${sock} -u root ${name} | gzip > "${file}"`, | |
| 465 | `echo "BACKUP=${file}"`, | |
| 466 | `ls -l "${file}"`, | |
| 521 | // Branch explicitly on the dump rather than letting the script run on. | |
| 522 | // `runDb` executes these under `set -uo pipefail` with NO `-e`, so a failed | |
| 523 | // dump does not stop what follows — and `gzip` exits 0 even when its | |
| 524 | // upstream died, so pipefail is the only thing that tells a real dump from | |
| 525 | // a 20-byte gzip header. Without this gate, retention would count that | |
| 526 | // header as the newest backup and delete a good one to make room for it: | |
| 527 | // adding a bound would then destroy data that accumulating never did. | |
| 528 | `if ${dump}; then`, | |
| 529 | ` mv -f "${part}" "${file}"`, | |
| 530 | ` echo "BACKUP=${file}"`, | |
| 531 | ` ls -l "${file}"`, | |
| 532 | ...(retention === false ? [] : buildDbBackupRetentionScript(destDir, name, retention)), | |
| 533 | 'else', | |
| 534 | // A partial file is worse than none: `buildListBackupsScript` would report | |
| 535 | // it as a backup, and `ls -1t` would rank it newest. | |
| 536 | ` rm -f "${part}"`, | |
| 537 | ` echo "[ts-cloud] backup failed for ${name}; no dump was kept" >&2`, | |
| 538 | ' exit 1', | |
| 539 | 'fi', | |
| 467 | 540 | ] |
| 468 | 541 | } |
| 469 | 542 | |
| @@ -502,7 +575,7 @@ export async function backupDatabase( | ||
| 502 | 575 | const r = await runDb( |
| 503 | 576 | config, |
| 504 | 577 | environment, |
| 505 | buildBackupScript(engine, name, DB_BACKUP_DIR, resolveAppDatabase(config)), | |
| 578 | buildBackupScript(engine, name, DB_BACKUP_DIR, resolveAppDatabase(config), resolveDbBackupRetention(config)), | |
| 506 | 579 | `ts-cloud db:backup ${name}`, |
| 507 | 580 | ) |
| 508 | 581 | return { ...r, database: name } |