ReviewOS
Reload

also looking at this

stacks/ts-cloud

test(fleet): prove the database and hostname halves of a transfer on real machines

#184
Merged glennmichael123 wants to merge test/move-database-and-rename-e2e into main
4 files +363 -3

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

packages/ts-cloud/src/operations/server-rename.test.tsmodified+12-1
@@ -149,6 +149,17 @@ describe('buildSetHostnameScript', () => {
149149 it('replaces the existing 127.0.1.1 line rather than appending a second', () => {
150150 const script = buildSetHostnameScript('hq-production-server')
151151 expect(script).toContain('s/^127\\.0\\.1\\.1.*/127.0.1.1\\thq-production-server/')
152 expect(script).toContain('if grep -q "127.0.1.1" /etc/hosts; then')
152 expect(script).toContain('if grep -q "^127.0.1.1" /etc/hosts; then')
153 })
154
155 /**
156 * `sed -i` renames a temp file over the target, which fails outright when
157 * /etc/hosts is a bind mount — every container — or is hard-linked. Writing
158 * the content back in place keeps the inode and works either way.
159 */
160 it('rewrites /etc/hosts in place rather than renaming over it', () => {
161 const script = buildSetHostnameScript('hq-production-server')
162 expect(script).not.toContain('sed -i')
163 expect(script).toContain('> /etc/hosts')
153164 })
154165})
packages/ts-cloud/src/operations/server-rename.tsmodified+9-2
@@ -177,8 +177,15 @@ export function buildSetHostnameScript(next: string): string {
177177 `hostnamectl set-hostname ${quoted} 2>/dev/null || { echo ${quoted} > /etc/hostname && hostname ${quoted}; }`,
178178 // Replace the old name where it stands rather than appending: a second
179179 // 127.0.1.1 line would leave the box resolving its own name two ways.
180 `if grep -q "127.0.1.1" /etc/hosts; then`,
181 ` sed -i "s/^127\\.0\\.1\\.1.*/127.0.1.1\\t${next}/" /etc/hosts`,
180 //
181 // Rewritten through a variable and truncating redirect rather than `sed -i`,
182 // which renames a temp file over the target and so fails outright when
183 // /etc/hosts is a bind mount (every container), is hard-linked, or lives on
184 // a filesystem the rename cannot cross. Writing in place keeps the inode and
185 // works in all of those.
186 `if grep -q "^127.0.1.1" /etc/hosts; then`,
187 ` TS_CLOUD_HOSTS="$(sed -E "s/^127\\.0\\.1\\.1.*/127.0.1.1\\t${next}/" /etc/hosts)"`,
188 ` printf '%s\\n' "$TS_CLOUD_HOSTS" > /etc/hosts`,
182189 'else',
183190 ` printf '127.0.1.1\\t%s\\n' ${quoted} >> /etc/hosts`,
184191 'fi',
packages/ts-cloud/test/integration/server-rename-e2e.test.tsadded+126-0
@@ -0,0 +1,126 @@
1/**
2 * End-to-end check of the one part of `server:rename` that touches a machine.
3 *
4 * Three of the four records a rename updates are data — a provider API call, a
5 * JSON state file, a row in the inventory — and unit tests cover them properly.
6 * The fourth is shell run on the box, and it is the one with somewhere to go
7 * wrong: `hostnamectl` is not always available, `/etc/hosts` may or may not
8 * already carry a `127.0.1.1` line, and getting either wrong leaves a box whose
9 * own name does not resolve — which surfaces later as `sudo` hanging, not as a
10 * failed rename.
11 *
12 * Both paths are exercised: with systemd present (`hostnamectl`) and without it
13 * (the `/etc/hostname` fallback the script falls back to).
14 *
15 * @see https://github.com/stacksjs/ts-cloud/issues/167
16 */
17
18import { afterAll, beforeAll, describe, expect, it } from 'bun:test'
19import { mkdtemp, rm, writeFile } from 'node:fs/promises'
20import { tmpdir } from 'node:os'
21import { join } from 'node:path'
22import { buildSetHostnameScript } from '../../src/operations/server-rename'
23
24const PLAIN = 'ts-cloud-rename-plain'
25const IMAGE = 'ts-cloud-rename-box:22.04'
26const NEW_NAME = 'hq-production-server'
27
28async function available(command: string[]): Promise<boolean> {
29 const result = await Bun.$`${command}`.quiet().nothrow()
30 return result.exitCode === 0
31}
32
33const canRun = await available(['docker', 'info'])
34
35async function exec(box: string, script: string): Promise<{ stdout: string, code: number }> {
36 const child = Bun.spawn(['docker', 'exec', '-i', box, 'bash', '-s'], {
37 stdin: new Blob([script]),
38 stdout: 'pipe',
39 stderr: 'pipe',
40 })
41 const [code, stdout, stderr] = await Promise.all([
42 child.exited,
43 new Response(child.stdout).text(),
44 new Response(child.stderr).text(),
45 ])
46 return { stdout: stdout + stderr, code }
47}
48
49async function run(box: string, script: string): Promise<string> {
50 const result = await exec(box, script)
51 if (result.code !== 0) throw new Error(`${box} exited ${result.code}:\n${result.stdout}`)
52 return result.stdout
53}
54
55describe.skipIf(!canRun)('server:rename on a box (docker)', () => {
56 let workspace: string
57
58 beforeAll(async () => {
59 workspace = await mkdtemp(join(tmpdir(), 'ts-cloud-rename-e2e-'))
60 const dockerfile = join(workspace, 'Dockerfile')
61 await writeFile(
62 dockerfile,
63 ['FROM ubuntu:22.04', 'ENV DEBIAN_FRONTEND=noninteractive', 'RUN apt-get update && apt-get install -y hostname && apt-get clean'].join('\n'),
64 )
65 await Bun.$`docker build -q -f ${dockerfile} -t ${IMAGE} ${workspace}`.quiet()
66 await Bun.$`docker rm -f ${PLAIN}`.quiet().nothrow()
67 // No systemd, so `hostnamectl` is absent and the fallback has to carry it —
68 // the path a minimal or containerised box actually takes.
69 // CAP_SYS_ADMIN because setting a hostname needs it and containers drop it by
70 // default — a real box's root has it, so without this the container would be
71 // testing a restriction the target environment does not have.
72 await Bun.$`docker run -d --name ${PLAIN} --hostname bughq --cap-add SYS_ADMIN ${IMAGE} sleep infinity`.quiet()
73 }, 600_000)
74
75 afterAll(async () => {
76 await Bun.$`docker rm -f ${PLAIN}`.quiet().nothrow()
77 await rm(workspace, { recursive: true, force: true })
78 })
79
80 it('starts from the old name', async () => {
81 expect((await run(PLAIN, 'hostname')).trim()).toBe('bughq')
82 })
83
84 it('renames the box when hostnamectl is unavailable', async () => {
85 const out = await run(PLAIN, buildSetHostnameScript(NEW_NAME))
86 expect(out).toContain(`bughq -> ${NEW_NAME}`)
87 expect((await run(PLAIN, 'hostname')).trim()).toBe(NEW_NAME)
88 expect((await run(PLAIN, 'cat /etc/hostname')).trim()).toBe(NEW_NAME)
89 })
90
91 /**
92 * Two `127.0.1.1` lines would leave the box resolving its own name two ways,
93 * which is the failure mode this replaces-in-place rather than appends for.
94 */
95 it('leaves exactly one 127.0.1.1 entry, pointing at the new name', async () => {
96 const hosts = await run(PLAIN, 'cat /etc/hosts')
97 const lines = hosts.split('\n').filter(line => line.trim().startsWith('127.0.1.1'))
98 expect(lines).toHaveLength(1)
99 expect(lines[0]).toContain(NEW_NAME)
100 expect(lines[0]).not.toContain('bughq')
101 })
102
103 it('resolves its own new name', async () => {
104 expect((await exec(PLAIN, `getent hosts ${NEW_NAME}`)).code).toBe(0)
105 })
106
107 /** Re-running a finished rename is what a resumed operation does. */
108 it('is idempotent, and still leaves one entry', async () => {
109 await run(PLAIN, buildSetHostnameScript(NEW_NAME))
110 expect((await run(PLAIN, 'hostname')).trim()).toBe(NEW_NAME)
111 const lines = (await run(PLAIN, 'cat /etc/hosts')).split('\n').filter(l => l.trim().startsWith('127.0.1.1'))
112 expect(lines).toHaveLength(1)
113 })
114
115 it('adds the entry on a box that had none', async () => {
116 // In place, for the same reason the product script is: /etc/hosts is a bind
117 // mount in a container and cannot be renamed over.
118 await run(PLAIN, 'TS_CLOUD_H="$(grep -v "^127.0.1.1" /etc/hosts)"; printf \'%s\\n\' "$TS_CLOUD_H" > /etc/hosts')
119 expect((await run(PLAIN, 'cat /etc/hosts')).split('\n').filter(l => l.trim().startsWith('127.0.1.1'))).toHaveLength(0)
120
121 await run(PLAIN, buildSetHostnameScript('hq-second-name'))
122 const lines = (await run(PLAIN, 'cat /etc/hosts')).split('\n').filter(l => l.trim().startsWith('127.0.1.1'))
123 expect(lines).toHaveLength(1)
124 expect(lines[0]).toContain('hq-second-name')
125 })
126})
packages/ts-cloud/test/integration/site-move-database-e2e.test.tsadded+216-0
@@ -0,0 +1,216 @@
1/**
2 * End-to-end check that a moved site's on-box database really arrives.
3 *
4 * This is the part of `site:move` with the least margin for error. The tree and
5 * the units can be re-shipped by a deploy if something goes wrong; the rows
6 * cannot. And it is the part unit tests can say least about, because what has to
7 * be true is not "the right string was built" but "pg_dump wrote something psql
8 * could read back into a database that setup had just created".
9 *
10 * So this stands up Postgres on two machines and drives the ACTUAL builders the
11 * move composes — `buildBackupScript` for the dump, `buildDatabaseSetupScript`
12 * for the role and schema, `buildBackupRestoreScript` for the load — then reads
13 * the rows back out of the target.
14 *
15 * Local `trust` auth on the unix socket is not a fudge for the test: it is what
16 * the pantry Postgres the drivers provision actually configures, and it is why
17 * `pgAdminCommand` omits `-h` for a co-located engine.
18 *
19 * @see https://github.com/stacksjs/ts-cloud/issues/167
20 */
21
22import type { DatabaseConfig } from '@ts-cloud/core'
23import { afterAll, beforeAll, describe, expect, it } from 'bun:test'
24import { mkdtemp, rm, writeFile } from 'node:fs/promises'
25import { tmpdir } from 'node:os'
26import { join } from 'node:path'
27import { buildBackupScript } from '../../src/deploy/dashboard-database'
28import { buildBackupRestoreScript } from '../../src/drivers/shared/backups'
29import { buildDatabaseSetupScript } from '../../src/drivers/shared/db-provision'
30
31const SOURCE = 'ts-cloud-db-source'
32const TARGET = 'ts-cloud-db-target'
33const IMAGE = 'ts-cloud-db-box:22.04'
34const DB_NAME = 'bughq'
35const DUMP = '/tmp/ts-cloud-move-hq-bughq.sql.gz'
36
37const database: DatabaseConfig = {
38 engine: 'postgres',
39 name: DB_NAME,
40 username: 'bughq_app',
41 password: 'not-a-real-password',
42}
43
44async function available(command: string[]): Promise<boolean> {
45 const result = await Bun.$`${command}`.quiet().nothrow()
46 return result.exitCode === 0
47}
48
49const canRun = await available(['docker', 'info'])
50
51async function exec(box: string, script: string): Promise<{ stdout: string, code: number }> {
52 const child = Bun.spawn(['docker', 'exec', '-i', box, 'bash', '-s'], {
53 stdin: new Blob([script]),
54 stdout: 'pipe',
55 stderr: 'pipe',
56 })
57 const [code, stdout, stderr] = await Promise.all([
58 child.exited,
59 new Response(child.stdout).text(),
60 new Response(child.stderr).text(),
61 ])
62 return { stdout: stdout + stderr, code }
63}
64
65async function run(box: string, script: string): Promise<string> {
66 const result = await exec(box, script)
67 if (result.code !== 0) throw new Error(`${box} exited ${result.code}:\n${result.stdout}`)
68 return result.stdout
69}
70
71/** Query the box's Postgres the way an operator would, and return the raw value. */
72async function query(box: string, sql: string, db = DB_NAME): Promise<string> {
73 const out = await run(box, `psql -tA -U postgres -d ${db} -c ${JSON.stringify(sql)}`)
74 return out.trim()
75}
76
77describe.skipIf(!canRun)('a moved site\'s database (docker)', () => {
78 let workspace: string
79
80 async function boot(name: string): Promise<void> {
81 await Bun.$`docker rm -f ${name}`.quiet().nothrow()
82 await Bun.$`docker run -d --name ${name} ${IMAGE} sleep infinity`.quiet()
83 // `trust` on the local socket — what pantry's postgres configures, and what
84 // lets `psql -U postgres` connect passwordless as root.
85 await run(name, [
86 'set -eu',
87 'PGDIR=$(ls -d /etc/postgresql/*/main | head -1)',
88 // Replaced outright rather than patched: Ubuntu ships several `local`
89 // lines (postgres and all), and editing only one leaves peer auth in
90 // force for the superuser the admin commands connect as.
91 'cat > "$PGDIR/pg_hba.conf" <<EOF',
92 'local all all trust',
93 'host all all 127.0.0.1/32 trust',
94 'host all all ::1/128 trust',
95 'EOF',
96 'CLUSTER=$(ls /etc/postgresql | head -1)',
97 'pg_ctlcluster "$CLUSTER" main start || pg_ctlcluster "$CLUSTER" main reload',
98 'for i in $(seq 1 30); do pg_isready -q && break; sleep 1; done',
99 'pg_isready',
100 ].join('\n'))
101 }
102
103 beforeAll(async () => {
104 workspace = await mkdtemp(join(tmpdir(), 'ts-cloud-db-e2e-'))
105 const dockerfile = join(workspace, 'Dockerfile')
106 await writeFile(
107 dockerfile,
108 [
109 'FROM ubuntu:22.04',
110 'ENV DEBIAN_FRONTEND=noninteractive',
111 'RUN apt-get update && apt-get install -y postgresql gzip \\',
112 ' && apt-get clean && rm -rf /var/lib/apt/lists/*',
113 // The builders put the engine client on PATH via `pantry env`, which is a
114 // no-op here; the apt client already is.
115 'ENV PATH=/usr/lib/postgresql/14/bin:$PATH',
116 ].join('\n'),
117 )
118 await Bun.$`docker build -q -f ${dockerfile} -t ${IMAGE} ${workspace}`.quiet()
119 await boot(SOURCE)
120 await boot(TARGET)
121
122 // The source's live database, with rows worth losing.
123 await run(SOURCE, [
124 'set -eu',
125 ...buildDatabaseSetupScript(database, { postgres: true }),
126 ].join('\n'))
127 // Seeded AS THE APP ROLE, because that is who runs migrations in practice.
128 // Creating the tables as the superuser instead would leave the app unable to
129 // read its own data on the source too, and quietly turn the ownership check
130 // below into a test of nothing.
131 const asApp = `PGPASSWORD='${database.password}' psql -h 127.0.0.1 -U ${database.username} -d ${DB_NAME}`
132 await run(SOURCE, [
133 'set -eu',
134 `${asApp} -c "CREATE TABLE incidents (id serial primary key, title text not null);"`,
135 `${asApp} -c "INSERT INTO incidents (title) VALUES ('edge outage'), ('db failover'), ('cert expiry');"`,
136 ].join('\n'))
137 }, 900_000)
138
139 afterAll(async () => {
140 await Bun.$`docker rm -f ${SOURCE}`.quiet().nothrow()
141 await Bun.$`docker rm -f ${TARGET}`.quiet().nothrow()
142 await rm(workspace, { recursive: true, force: true })
143 })
144
145 it('creates the role and database from the same script provisioning runs', async () => {
146 expect(await query(SOURCE, 'SELECT count(*) FROM incidents')).toBe('3')
147 const role = await query(SOURCE, `SELECT count(*) FROM pg_roles WHERE rolname = '${database.username}'`, 'postgres')
148 expect(role).toBe('1')
149 })
150
151 /** Idempotent by design — a re-run must not fail or disturb the data. */
152 it('is idempotent, so a resumed move does not break the database', async () => {
153 await run(SOURCE, ['set -eu', ...buildDatabaseSetupScript(database, { postgres: true })].join('\n'))
154 expect(await query(SOURCE, 'SELECT count(*) FROM incidents')).toBe('3')
155 })
156
157 it('dumps the database to a file the move can carry', async () => {
158 await run(SOURCE, [
159 'set -euo pipefail',
160 ...buildBackupScript('postgres', DB_NAME, '/tmp', database),
161 `mv -f "$(ls -1t /tmp/${DB_NAME}-*.sql.gz | head -1)" ${DUMP}`,
162 ].join('\n'))
163 const size = await run(SOURCE, `stat -c %s ${DUMP}`)
164 expect(Number(size.trim())).toBeGreaterThan(0)
165 // A real dump, not an error page written to the file.
166 const head = await run(SOURCE, `gunzip -c ${DUMP} | head -40`)
167 expect(head).toContain('CREATE TABLE')
168 expect(head.toLowerCase()).toContain('incidents')
169 })
170
171 it('starts from a target that does not have the database at all', async () => {
172 const exists = await query(TARGET, `SELECT count(*) FROM pg_database WHERE datname = '${DB_NAME}'`, 'postgres')
173 expect(exists).toBe('0')
174 })
175
176 /**
177 * The whole point: rows written on one machine, read back on another, through
178 * the builders the move actually composes.
179 */
180 it('restores every row on the target', async () => {
181 const local = join(workspace, 'dump.sql.gz')
182 await Bun.$`docker cp ${`${SOURCE}:${DUMP}`} ${local}`.quiet()
183 await Bun.$`docker cp ${local} ${`${TARGET}:${DUMP}`}`.quiet()
184
185 await run(TARGET, [
186 ...buildDatabaseSetupScript(database, { postgres: true }),
187 ...buildBackupRestoreScript(database, { from: DUMP }),
188 ].join('\n'))
189
190 expect(await query(TARGET, 'SELECT count(*) FROM incidents')).toBe('3')
191 const titles = await query(TARGET, 'SELECT title FROM incidents ORDER BY id')
192 expect(titles.split('\n')).toEqual(['edge outage', 'db failover', 'cert expiry'])
193 })
194
195 /**
196 * The app connects as its own role, not as the superuser the restore ran as.
197 * `pg_dump` records ownership; if the restore lost it, the app would come up
198 * against a database full of tables it cannot read — which looks exactly like
199 * an empty database until someone reads the logs.
200 */
201 it('preserves table ownership, so the application role can still read', async () => {
202 const owner = await query(TARGET, "SELECT tableowner FROM pg_tables WHERE tablename = 'incidents'")
203 expect(owner).toBe(database.username!)
204
205 const out = await run(
206 TARGET,
207 `PGPASSWORD='${database.password}' psql -tA -h 127.0.0.1 -U ${database.username} -d ${DB_NAME} `
208 + `-c 'SELECT count(*) FROM incidents'`,
209 )
210 expect(out.trim()).toBe('3')
211 })
212
213 it('leaves the source untouched, so the move is still reversible', async () => {
214 expect(await query(SOURCE, 'SELECT count(*) FROM incidents')).toBe('3')
215 })
216})