ReviewOS
Reload

also looking at this

stacks/ts-cloud

test(fleet): run site:move against two real machines

#180
Merged glennmichael123 wants to merge test/site-move-e2e into main
1 file +327 -0

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

packages/ts-cloud/test/integration/site-move-e2e.test.tsadded+327-0
@@ -0,0 +1,327 @@
1/**
2 * End-to-end check of `site:move` against two real Linux machines.
3 *
4 * Every other test of this operation drives injected effects: they prove the
5 * PLAN is right — the ordering, the resume behaviour, what refuses to run — and
6 * assert the generated shell as strings. None of them can tell you the shell
7 * actually works. For an operation that archives a live site, carries its
8 * database and its private keys, and drains the box it came from, "the string
9 * looked right" is not the same claim as "it ran".
10 *
11 * So this boots two systemd containers, builds a realistic site on the first —
12 * release tree, `current` symlink, shared state symlinked into the release,
13 * systemd units that really run — and then executes the scripts the operation
14 * generates, in the order it generates them, against real `systemd`, real `tar`
15 * and real `curl`.
16 *
17 * Skipped when Docker is unavailable, so a machine without it reports a skip
18 * rather than a false pass.
19 *
20 * @see https://github.com/stacksjs/ts-cloud/issues/167
21 */
22
23import { afterAll, beforeAll, describe, expect, it } from 'bun:test'
24import { mkdtemp, rm, writeFile } from 'node:fs/promises'
25import { tmpdir } from 'node:os'
26import { join } from 'node:path'
27import {
28 buildCertificatePackScript,
29 buildCertificateStateScript,
30 buildCertificateUnpackScript,
31 buildDrainSourceScript,
32 buildHealthGateScript,
33 buildPauseWorkersScript,
34 buildRestoreScript,
35 buildSnapshotScript,
36 buildSourceStateScript,
37 buildTargetStateScript,
38 buildWorkersStateScript,
39 certificatesMatch,
40 parseCertificateState,
41 parseSourceDrained,
42 parseTargetReady,
43 parseWorkersPaused,
44 siteMoveArchivePath,
45 siteMoveCertArchivePath,
46} from '../../src/operations/site-move'
47
48const SOURCE = 'ts-cloud-move-source'
49const TARGET = 'ts-cloud-move-target'
50const IMAGE = 'ts-cloud-move-box:22.04'
51const SLUG = 'hq'
52const SITE = 'bughq'
53const APP_BASE = `/var/www/${SLUG}-${SITE}`
54const RELEASE = 'r1'
55const PORT = 3010
56const DOMAIN = 'bughq.example.com'
57const CERTS_DIR = '/etc/rpx/certs'
58/** Content written into shared state, to prove the move carries it byte-for-byte. */
59const SHARED_DB_CONTENT = 'the-production-rows'
60
61async function available(command: string[]): Promise<boolean> {
62 const result = await Bun.$`${command}`.quiet().nothrow()
63 return result.exitCode === 0
64}
65
66const canRun = await available(['docker', 'info'])
67
68/** Run a script inside a box, the way the operation's exec effects would. */
69async function exec(box: string, script: string): Promise<{ stdout: string, code: number }> {
70 // Fed over stdin rather than as an argument, the same way the real transport
71 // does it — so quoting behaves identically here and on a live box.
72 const child = Bun.spawn(['docker', 'exec', '-i', box, 'bash', '-s'], {
73 stdin: new Blob([script]),
74 stdout: 'pipe',
75 stderr: 'pipe',
76 })
77 const [code, stdout, stderr] = await Promise.all([
78 child.exited,
79 new Response(child.stdout).text(),
80 new Response(child.stderr).text(),
81 ])
82 return { stdout: stdout + stderr, code }
83}
84
85/** Run a script and require it to succeed, surfacing the box's own output when it does not. */
86async function run(box: string, script: string): Promise<string> {
87 const result = await exec(box, script)
88 if (result.code !== 0) throw new Error(`${box} exited ${result.code}:\n${result.stdout}`)
89 return result.stdout
90}
91
92describe.skipIf(!canRun)('site:move between two boxes (docker)', () => {
93 let workspace: string
94
95 async function boot(name: string): Promise<void> {
96 await Bun.$`docker rm -f ${name}`.quiet().nothrow()
97 await Bun.$`docker run -d --name ${name} --privileged --cgroupns=host \
98 -v /sys/fs/cgroup:/sys/fs/cgroup:rw ${IMAGE}`.quiet()
99 for (let attempt = 0; attempt < 60; attempt++) {
100 const state = await Bun.$`docker exec ${name} systemctl is-system-running`.quiet().nothrow()
101 const value = state.stdout.toString().trim()
102 if (value === 'running' || value === 'degraded') return
103 await Bun.sleep(1000)
104 }
105 throw new Error(`${name} never finished booting`)
106 }
107
108 /**
109 * Build the site the way a deploy leaves it: a release directory, shared state
110 * symlinked INTO that release, and `current` pointing at it. The symlinks are
111 * the point — a move that dereferences them silently turns shared state back
112 * into per-release state, which is the failure `sharedPaths` exists to prevent.
113 */
114 async function seedSite(): Promise<void> {
115 await run(SOURCE, [
116 'set -eu',
117 `mkdir -p ${APP_BASE}/releases/${RELEASE}/database ${APP_BASE}/shared/database`,
118 `printf '%s' '${SHARED_DB_CONTENT}' > ${APP_BASE}/shared/database/app.sqlite`,
119 // The release links out to shared state, exactly as buildLinkSharedPaths leaves it.
120 `ln -sfn ${APP_BASE}/shared/database/app.sqlite ${APP_BASE}/releases/${RELEASE}/database/app.sqlite`,
121 `printf 'ok' > ${APP_BASE}/releases/${RELEASE}/index.html`,
122 `ln -sfn ${APP_BASE}/releases/${RELEASE} ${APP_BASE}/current`,
123 // The app: something that really binds the port the health gate polls.
124 `cat > /etc/systemd/system/${SLUG}-${SITE}.service <<'EOF'`,
125 '[Unit]',
126 `Description=${SITE}`,
127 '[Service]',
128 `WorkingDirectory=${APP_BASE}/current`,
129 `ExecStart=/usr/bin/python3 -m http.server ${PORT} --bind 127.0.0.1`,
130 'Restart=always',
131 '[Install]',
132 'WantedBy=multi-user.target',
133 'EOF',
134 // Background work: the units the move must stop before it snapshots.
135 `cat > /etc/systemd/system/${SLUG}-${SITE}-scheduler.service <<'EOF'`,
136 '[Unit]',
137 'Description=scheduler',
138 '[Service]',
139 'ExecStart=/bin/sh -c "while true; do sleep 5; done"',
140 'Restart=always',
141 '[Install]',
142 'WantedBy=multi-user.target',
143 'EOF',
144 `sed 's/scheduler/queue-0/' /etc/systemd/system/${SLUG}-${SITE}-scheduler.service > /etc/systemd/system/${SLUG}-${SITE}-queue-0.service`,
145 'systemctl daemon-reload',
146 `systemctl enable --now ${SLUG}-${SITE}.service ${SLUG}-${SITE}-scheduler.service ${SLUG}-${SITE}-queue-0.service`,
147 // TLS material, as the gateway would hold it.
148 `mkdir -p ${CERTS_DIR}`,
149 `printf 'CERT-BODY' > ${CERTS_DIR}/${DOMAIN}.crt`,
150 `printf 'KEY-BODY' > ${CERTS_DIR}/${DOMAIN}.key`,
151 `chmod 600 ${CERTS_DIR}/${DOMAIN}.key`,
152 ].join('\n'))
153 }
154
155 beforeAll(async () => {
156 workspace = await mkdtemp(join(tmpdir(), 'ts-cloud-move-e2e-'))
157 const dockerfile = join(workspace, 'Dockerfile')
158 await writeFile(
159 dockerfile,
160 [
161 'FROM ubuntu:22.04',
162 'ENV DEBIAN_FRONTEND=noninteractive container=docker',
163 'RUN apt-get update && apt-get install -y systemd systemd-sysv curl python3 ca-certificates \\',
164 ' && apt-get clean && rm -rf /var/lib/apt/lists/* \\',
165 ' && rm -f /lib/systemd/system/multi-user.target.wants/* /etc/systemd/system/*.wants/* \\',
166 ' /lib/systemd/system/local-fs.target.wants/* /lib/systemd/system/sockets.target.wants/*udev*',
167 'STOPSIGNAL SIGRTMIN+3',
168 'CMD ["/lib/systemd/systemd"]',
169 ].join('\n'),
170 )
171 await Bun.$`docker build -q -f ${dockerfile} -t ${IMAGE} ${workspace}`.quiet()
172 await boot(SOURCE)
173 await boot(TARGET)
174 await seedSite()
175 }, 900_000)
176
177 afterAll(async () => {
178 await Bun.$`docker rm -f ${SOURCE}`.quiet().nothrow()
179 await Bun.$`docker rm -f ${TARGET}`.quiet().nothrow()
180 await rm(workspace, { recursive: true, force: true })
181 })
182
183 it('starts from a site that is genuinely serving', async () => {
184 const health = await exec(SOURCE, buildHealthGateScript(PORT, '/'))
185 expect(health.code).toBe(0)
186 expect(health.stdout).toContain('healthy')
187 })
188
189 /**
190 * The state check has to read real `systemctl` output. If the parser and the
191 * shell disagree the plan silently decides a step is already done.
192 */
193 it('sees the background units running, and the parser agrees', async () => {
194 const state = await run(SOURCE, buildWorkersStateScript(SLUG, SITE))
195 expect(parseWorkersPaused(state)).toBe(false)
196 expect(state).toContain(`active:${SLUG}-${SITE}-scheduler.service`)
197 expect(state).toContain(`active:${SLUG}-${SITE}-queue-0.service`)
198 })
199
200 it('pauses background work while leaving the site serving', async () => {
201 await run(SOURCE, buildPauseWorkersScript(SLUG, SITE))
202
203 expect(parseWorkersPaused(await run(SOURCE, buildWorkersStateScript(SLUG, SITE)))).toBe(true)
204 // The web service is deliberately untouched: the source is still live.
205 const app = await exec(SOURCE, `systemctl is-active ${SLUG}-${SITE}.service`)
206 expect(app.stdout.trim()).toBe('active')
207 expect((await exec(SOURCE, buildHealthGateScript(PORT, '/'))).code).toBe(0)
208 })
209
210 it('archives the tree and the unit files together', async () => {
211 await run(SOURCE, buildSnapshotScript(SLUG, SITE, APP_BASE))
212 const listing = await run(SOURCE, `tar tzf ${siteMoveArchivePath(SLUG, SITE)}`)
213 expect(listing).toContain(`${SLUG}-${SITE}/shared/database/app.sqlite`)
214 expect(listing).toContain(`${SLUG}-${SITE}/releases/${RELEASE}/index.html`)
215 expect(listing).toContain(`${SLUG}-${SITE}.service`)
216 expect(listing).toContain(`${SLUG}-${SITE}-scheduler.service`)
217 })
218
219 /**
220 * The claim this whole test exists for: `tar` must NOT dereference. A
221 * flattened `current` turns one release into a second copy, and a flattened
222 * shared path turns the database back into per-release state — silently, and
223 * only visible one deploy later.
224 */
225 it('keeps current and the shared links as symlinks inside the archive', async () => {
226 const verbose = await run(SOURCE, `tar tzvf ${siteMoveArchivePath(SLUG, SITE)}`)
227 const current = verbose.split('\n').find(line => line.includes(`${SLUG}-${SITE}/current`))
228 const shared = verbose.split('\n').find(line => line.includes(`releases/${RELEASE}/database/app.sqlite`))
229 expect(current?.startsWith('l')).toBe(true)
230 expect(shared?.startsWith('l')).toBe(true)
231 })
232
233 it('reports the target as empty before the move', async () => {
234 const state = await run(TARGET, buildTargetStateScript(SLUG, SITE, APP_BASE))
235 expect(parseTargetReady(state)).toBe(false)
236 expect(state).toContain('tree:absent')
237 })
238
239 it('unpacks on the target and brings the site up there', async () => {
240 const archive = siteMoveArchivePath(SLUG, SITE)
241 const local = join(workspace, 'tree.tar.gz')
242 await Bun.$`docker cp ${`${SOURCE}:${archive}`} ${local}`.quiet()
243 await Bun.$`docker cp ${local} ${`${TARGET}:${archive}`}`.quiet()
244
245 await run(TARGET, buildRestoreScript(SLUG, SITE, APP_BASE))
246
247 const state = await run(TARGET, buildTargetStateScript(SLUG, SITE, APP_BASE))
248 expect(parseTargetReady(state)).toBe(true)
249 })
250
251 it('carries the shared state across byte-for-byte, still as symlinks', async () => {
252 const content = await run(TARGET, `cat ${APP_BASE}/shared/database/app.sqlite`)
253 expect(content.trim()).toBe(SHARED_DB_CONTENT)
254
255 // The release still POINTS at shared state rather than holding a copy.
256 const linked = await run(TARGET, `readlink ${APP_BASE}/releases/${RELEASE}/database/app.sqlite`)
257 expect(linked.trim()).toBe(`${APP_BASE}/shared/database/app.sqlite`)
258 const current = await run(TARGET, `readlink ${APP_BASE}/current`)
259 expect(current.trim()).toBe(`${APP_BASE}/releases/${RELEASE}`)
260
261 // Writing through the release link reaches the shared file, as the app would.
262 await run(TARGET, `printf 'new-rows' > ${APP_BASE}/releases/${RELEASE}/database/app.sqlite`)
263 expect((await run(TARGET, `cat ${APP_BASE}/shared/database/app.sqlite`)).trim()).toBe('new-rows')
264 })
265
266 /** Two boxes running one scheduler against one dataset is the thing to avoid. */
267 it('starts the app on the target but leaves its background units stopped', async () => {
268 expect((await exec(TARGET, `systemctl is-active ${SLUG}-${SITE}.service`)).stdout.trim()).toBe('active')
269 expect(parseWorkersPaused(await run(TARGET, buildWorkersStateScript(SLUG, SITE)))).toBe(true)
270 })
271
272 it('passes the health gate on the target, against a real listener', async () => {
273 const health = await exec(TARGET, buildHealthGateScript(PORT, '/'))
274 expect(health.code).toBe(0)
275 expect(health.stdout).toContain('healthy')
276 })
277
278 it('fails the health gate when nothing is listening', async () => {
279 const health = await exec(TARGET, buildHealthGateScript(3999, '/', 1))
280 expect(health.code).not.toBe(0)
281 expect(health.stdout).toContain('no healthy response')
282 })
283
284 it('carries the TLS material, keys still unreadable to others', async () => {
285 const before = parseCertificateState(await run(SOURCE, buildCertificateStateScript(CERTS_DIR, [DOMAIN])))
286 const targetBefore = parseCertificateState(await run(TARGET, buildCertificateStateScript(CERTS_DIR, [DOMAIN])))
287 expect(certificatesMatch(before, targetBefore)).toBe(false)
288
289 const archive = siteMoveCertArchivePath(SLUG, SITE)
290 await run(SOURCE, buildCertificatePackScript(CERTS_DIR, [DOMAIN], archive))
291 const local = join(workspace, 'certs.tar.gz')
292 await Bun.$`docker cp ${`${SOURCE}:${archive}`} ${local}`.quiet()
293 await Bun.$`docker cp ${local} ${`${TARGET}:${archive}`}`.quiet()
294 await run(TARGET, buildCertificateUnpackScript(CERTS_DIR, archive))
295
296 const after = parseCertificateState(await run(TARGET, buildCertificateStateScript(CERTS_DIR, [DOMAIN])))
297 expect(certificatesMatch(before, after)).toBe(true)
298 expect((await run(TARGET, `cat ${CERTS_DIR}/${DOMAIN}.key`)).trim()).toBe('KEY-BODY')
299 expect((await run(TARGET, `stat -c %a ${CERTS_DIR}/${DOMAIN}.key`)).trim()).toBe('600')
300 })
301
302 /**
303 * The drain is what makes the whole operation reversible: it stops the site on
304 * the source and leaves every byte of it in place, so a bad cutover is undone
305 * by starting the units again.
306 */
307 it('drains the source completely', async () => {
308 await run(SOURCE, buildDrainSourceScript(SLUG, SITE))
309 const state = await run(SOURCE, buildSourceStateScript(SLUG, SITE))
310 expect(parseSourceDrained(state)).toBe(true)
311 expect((await exec(SOURCE, buildHealthGateScript(PORT, '/', 1))).code).not.toBe(0)
312 })
313
314 it('leaves the source files untouched, so the move can still be undone', async () => {
315 const content = await run(SOURCE, `cat ${APP_BASE}/shared/database/app.sqlite`)
316 expect(content.trim()).toBe(SHARED_DB_CONTENT)
317 expect((await exec(SOURCE, `test -L ${APP_BASE}/current`)).code).toBe(0)
318 expect((await exec(SOURCE, `test -f /etc/systemd/system/${SLUG}-${SITE}.service`)).code).toBe(0)
319 })
320
321 it('comes back on the source by restarting it, with its data intact', async () => {
322 await run(SOURCE, `systemctl start ${SLUG}-${SITE}.service`)
323 const health = await exec(SOURCE, buildHealthGateScript(PORT, '/'))
324 expect(health.code).toBe(0)
325 expect((await run(SOURCE, `cat ${APP_BASE}/shared/database/app.sqlite`)).trim()).toBe(SHARED_DB_CONTENT)
326 })
327})