ReviewOS
Reload

also looking at this

stacks/ts-cloud

test(fleet): prove the database and hostname halves of a transfer on real machines

#184
Merged glennmichael123 wants to merge test/move-database-and-rename-e2e into main
4 files +363 -3
packages/ts-cloud/test/integration/site-move-database-e2e.test.tsadded+216-0
@@ -0,0 +1,216 @@
1/**
2 * End-to-end check that a moved site's on-box database really arrives.
3 *
4 * This is the part of `site:move` with the least margin for error. The tree and
5 * the units can be re-shipped by a deploy if something goes wrong; the rows
6 * cannot. And it is the part unit tests can say least about, because what has to
7 * be true is not "the right string was built" but "pg_dump wrote something psql
8 * could read back into a database that setup had just created".
9 *
10 * So this stands up Postgres on two machines and drives the ACTUAL builders the
11 * move composes — `buildBackupScript` for the dump, `buildDatabaseSetupScript`
12 * for the role and schema, `buildBackupRestoreScript` for the load — then reads
13 * the rows back out of the target.
14 *
15 * Local `trust` auth on the unix socket is not a fudge for the test: it is what
16 * the pantry Postgres the drivers provision actually configures, and it is why
17 * `pgAdminCommand` omits `-h` for a co-located engine.
18 *
19 * @see https://github.com/stacksjs/ts-cloud/issues/167
20 */
21
22import type { DatabaseConfig } from '@ts-cloud/core'
23import { afterAll, beforeAll, describe, expect, it } from 'bun:test'
24import { mkdtemp, rm, writeFile } from 'node:fs/promises'
25import { tmpdir } from 'node:os'
26import { join } from 'node:path'
27import { buildBackupScript } from '../../src/deploy/dashboard-database'
28import { buildBackupRestoreScript } from '../../src/drivers/shared/backups'
29import { buildDatabaseSetupScript } from '../../src/drivers/shared/db-provision'
30
31const SOURCE = 'ts-cloud-db-source'
32const TARGET = 'ts-cloud-db-target'
33const IMAGE = 'ts-cloud-db-box:22.04'
34const DB_NAME = 'bughq'
35const DUMP = '/tmp/ts-cloud-move-hq-bughq.sql.gz'
36
37const database: DatabaseConfig = {
38 engine: 'postgres',
39 name: DB_NAME,
40 username: 'bughq_app',
41 password: 'not-a-real-password',
42}
43
44async function available(command: string[]): Promise<boolean> {
45 const result = await Bun.$`${command}`.quiet().nothrow()
46 return result.exitCode === 0
47}
48
49const canRun = await available(['docker', 'info'])
50
51async function exec(box: string, script: string): Promise<{ stdout: string, code: number }> {
52 const child = Bun.spawn(['docker', 'exec', '-i', box, 'bash', '-s'], {
53 stdin: new Blob([script]),
54 stdout: 'pipe',
55 stderr: 'pipe',
56 })
57 const [code, stdout, stderr] = await Promise.all([
58 child.exited,
59 new Response(child.stdout).text(),
60 new Response(child.stderr).text(),
61 ])
62 return { stdout: stdout + stderr, code }
63}
64
65async function run(box: string, script: string): Promise<string> {
66 const result = await exec(box, script)
67 if (result.code !== 0) throw new Error(`${box} exited ${result.code}:\n${result.stdout}`)
68 return result.stdout
69}
70
71/** Query the box's Postgres the way an operator would, and return the raw value. */
72async function query(box: string, sql: string, db = DB_NAME): Promise<string> {
73 const out = await run(box, `psql -tA -U postgres -d ${db} -c ${JSON.stringify(sql)}`)
74 return out.trim()
75}
76
77describe.skipIf(!canRun)('a moved site\'s database (docker)', () => {
78 let workspace: string
79
80 async function boot(name: string): Promise<void> {
81 await Bun.$`docker rm -f ${name}`.quiet().nothrow()
82 await Bun.$`docker run -d --name ${name} ${IMAGE} sleep infinity`.quiet()
83 // `trust` on the local socket — what pantry's postgres configures, and what
84 // lets `psql -U postgres` connect passwordless as root.
85 await run(name, [
86 'set -eu',
87 'PGDIR=$(ls -d /etc/postgresql/*/main | head -1)',
88 // Replaced outright rather than patched: Ubuntu ships several `local`
89 // lines (postgres and all), and editing only one leaves peer auth in
90 // force for the superuser the admin commands connect as.
91 'cat > "$PGDIR/pg_hba.conf" <<EOF',
92 'local all all trust',
93 'host all all 127.0.0.1/32 trust',
94 'host all all ::1/128 trust',
95 'EOF',
96 'CLUSTER=$(ls /etc/postgresql | head -1)',
97 'pg_ctlcluster "$CLUSTER" main start || pg_ctlcluster "$CLUSTER" main reload',
98 'for i in $(seq 1 30); do pg_isready -q && break; sleep 1; done',
99 'pg_isready',
100 ].join('\n'))
101 }
102
103 beforeAll(async () => {
104 workspace = await mkdtemp(join(tmpdir(), 'ts-cloud-db-e2e-'))
105 const dockerfile = join(workspace, 'Dockerfile')
106 await writeFile(
107 dockerfile,
108 [
109 'FROM ubuntu:22.04',
110 'ENV DEBIAN_FRONTEND=noninteractive',
111 'RUN apt-get update && apt-get install -y postgresql gzip \\',
112 ' && apt-get clean && rm -rf /var/lib/apt/lists/*',
113 // The builders put the engine client on PATH via `pantry env`, which is a
114 // no-op here; the apt client already is.
115 'ENV PATH=/usr/lib/postgresql/14/bin:$PATH',
116 ].join('\n'),
117 )
118 await Bun.$`docker build -q -f ${dockerfile} -t ${IMAGE} ${workspace}`.quiet()
119 await boot(SOURCE)
120 await boot(TARGET)
121
122 // The source's live database, with rows worth losing.
123 await run(SOURCE, [
124 'set -eu',
125 ...buildDatabaseSetupScript(database, { postgres: true }),
126 ].join('\n'))
127 // Seeded AS THE APP ROLE, because that is who runs migrations in practice.
128 // Creating the tables as the superuser instead would leave the app unable to
129 // read its own data on the source too, and quietly turn the ownership check
130 // below into a test of nothing.
131 const asApp = `PGPASSWORD='${database.password}' psql -h 127.0.0.1 -U ${database.username} -d ${DB_NAME}`
132 await run(SOURCE, [
133 'set -eu',
134 `${asApp} -c "CREATE TABLE incidents (id serial primary key, title text not null);"`,
135 `${asApp} -c "INSERT INTO incidents (title) VALUES ('edge outage'), ('db failover'), ('cert expiry');"`,
136 ].join('\n'))
137 }, 900_000)
138
139 afterAll(async () => {
140 await Bun.$`docker rm -f ${SOURCE}`.quiet().nothrow()
141 await Bun.$`docker rm -f ${TARGET}`.quiet().nothrow()
142 await rm(workspace, { recursive: true, force: true })
143 })
144
145 it('creates the role and database from the same script provisioning runs', async () => {
146 expect(await query(SOURCE, 'SELECT count(*) FROM incidents')).toBe('3')
147 const role = await query(SOURCE, `SELECT count(*) FROM pg_roles WHERE rolname = '${database.username}'`, 'postgres')
148 expect(role).toBe('1')
149 })
150
151 /** Idempotent by design — a re-run must not fail or disturb the data. */
152 it('is idempotent, so a resumed move does not break the database', async () => {
153 await run(SOURCE, ['set -eu', ...buildDatabaseSetupScript(database, { postgres: true })].join('\n'))
154 expect(await query(SOURCE, 'SELECT count(*) FROM incidents')).toBe('3')
155 })
156
157 it('dumps the database to a file the move can carry', async () => {
158 await run(SOURCE, [
159 'set -euo pipefail',
160 ...buildBackupScript('postgres', DB_NAME, '/tmp', database),
161 `mv -f "$(ls -1t /tmp/${DB_NAME}-*.sql.gz | head -1)" ${DUMP}`,
162 ].join('\n'))
163 const size = await run(SOURCE, `stat -c %s ${DUMP}`)
164 expect(Number(size.trim())).toBeGreaterThan(0)
165 // A real dump, not an error page written to the file.
166 const head = await run(SOURCE, `gunzip -c ${DUMP} | head -40`)
167 expect(head).toContain('CREATE TABLE')
168 expect(head.toLowerCase()).toContain('incidents')
169 })
170
171 it('starts from a target that does not have the database at all', async () => {
172 const exists = await query(TARGET, `SELECT count(*) FROM pg_database WHERE datname = '${DB_NAME}'`, 'postgres')
173 expect(exists).toBe('0')
174 })
175
176 /**
177 * The whole point: rows written on one machine, read back on another, through
178 * the builders the move actually composes.
179 */
180 it('restores every row on the target', async () => {
181 const local = join(workspace, 'dump.sql.gz')
182 await Bun.$`docker cp ${`${SOURCE}:${DUMP}`} ${local}`.quiet()
183 await Bun.$`docker cp ${local} ${`${TARGET}:${DUMP}`}`.quiet()
184
185 await run(TARGET, [
186 ...buildDatabaseSetupScript(database, { postgres: true }),
187 ...buildBackupRestoreScript(database, { from: DUMP }),
188 ].join('\n'))
189
190 expect(await query(TARGET, 'SELECT count(*) FROM incidents')).toBe('3')
191 const titles = await query(TARGET, 'SELECT title FROM incidents ORDER BY id')
192 expect(titles.split('\n')).toEqual(['edge outage', 'db failover', 'cert expiry'])
193 })
194
195 /**
196 * The app connects as its own role, not as the superuser the restore ran as.
197 * `pg_dump` records ownership; if the restore lost it, the app would come up
198 * against a database full of tables it cannot read — which looks exactly like
199 * an empty database until someone reads the logs.
200 */
201 it('preserves table ownership, so the application role can still read', async () => {
202 const owner = await query(TARGET, "SELECT tableowner FROM pg_tables WHERE tablename = 'incidents'")
203 expect(owner).toBe(database.username!)
204
205 const out = await run(
206 TARGET,
207 `PGPASSWORD='${database.password}' psql -tA -h 127.0.0.1 -U ${database.username} -d ${DB_NAME} `
208 + `-c 'SELECT count(*) FROM incidents'`,
209 )
210 expect(out.trim()).toBe('3')
211 })
212
213 it('leaves the source untouched, so the move is still reversible', async () => {
214 expect(await query(SOURCE, 'SELECT count(*) FROM incidents')).toBe('3')
215 })
216})