ReviewOS
Reload

also looking at this

stacks/ts-cloud

feat(fleet): move a deployed site to another server (#167 operation 2)

#176
Merged glennmichael123 wants to merge feat/site-move into main
4 files +951 -0
packages/ts-cloud/src/operations/site-move.tsadded+361-0
@@ -0,0 +1,361 @@
1/**
2 * Move a deployed site from one box to another.
3 *
4 * The second of the consolidation operations: three boxes with one app each
5 * should be able to become one box with three sites, and today that is an
6 * afternoon of SSH. What makes it an afternoon is not the individual steps but
7 * the fact that a half-finished one leaves nothing to tell you what already
8 * happened — so this is expressed as a plan on the same scaffolding as
9 * `server:rename` (see `./plan`), where every step re-derives its own state and
10 * a run that dies is resumed by running it again.
11 *
12 * It moves the site's on-box footprint WHOLESALE — the whole
13 * `/var/www/<slug>-<site>` tree (every release, `shared/`, and the `current`
14 * symlink) plus the systemd units that run it. Copying those units is not a
15 * parallel mechanism: they are literally the units the deploy wrote, moved. The
16 * alternative, rebuilding from the repo on the target, would not be a move — it
17 * would be a fresh deploy that happens to be preceded by a data copy, and it
18 * would silently pick up whatever the repo says today rather than what is
19 * actually running.
20 *
21 * **Nothing here is irreversible.** The source's tree is never deleted, only
22 * drained: its units are stopped and disabled and its gateway route removed, so
23 * a bad cutover is undone by starting them again and pointing DNS back. That is
24 * the reversibility the operation is required to have — it lasts until the
25 * source SERVER is deleted, which is a separate, deliberately separate,
26 * destructive operation (`server:destroy`).
27 *
28 * Ordering is chosen for what a failure in the middle leaves behind. The source
29 * keeps serving until the target has passed a health gate, DNS is cut over only
30 * after that, and the source is drained last — so every prefix of this plan is a
31 * working system, either on the old box or the new one.
32 *
33 * @see https://github.com/stacksjs/ts-cloud/issues/167
34 */
35import type { OperationPlan, OperationStep } from './plan'
36
37/** Where a snapshot archive is staged on both boxes. */
38export function siteMoveArchivePath(slug: string, siteName: string): string {
39 return `/tmp/ts-cloud-move-${slug}-${siteName}.tar.gz`
40}
41
42/** Systemd unit-file glob covering a site's app, scheduler, queue and daemon units. */
43export function siteUnitGlob(slug: string, siteName: string): string {
44 return `${slug}-${siteName}*.service`
45}
46
47/** Single-quote a value for safe embedding in the generated shell. */
48function sh(value: string): string {
49 return `'${value.split('\'').join('\'\\\'\'')}'`
50}
51
52/**
53 * Stop the site's BACKGROUND units on the source — scheduler, queue workers,
54 * daemons — leaving the web service running.
55 *
56 * The snapshot has to be consistent, and background work is what writes to it: a
57 * queue worker committing to a SQLite database halfway through `tar` produces an
58 * archive with a torn page in it. The web service keeps serving because the
59 * source is still the live site at this point; read traffic during the copy is
60 * fine, and stopping it here would be an outage taken long before the target is
61 * ready to replace it.
62 */
63export function buildPauseWorkersScript(slug: string, siteName: string): string {
64 return [
65 'set -eu',
66 `for unit in $(ls /etc/systemd/system/ 2>/dev/null | grep -E ${sh(backgroundUnitPattern(slug, siteName))} || true); do`,
67 ' systemctl stop "$unit" 2>/dev/null || true',
68 'done',
69 ].join('\n')
70}
71
72/** Background units for a site: scheduler, queue workers, daemons. */
73function backgroundUnitPattern(slug: string, siteName: string): string {
74 return `^${slug}-${siteName}-((queue|daemon)-.*|scheduler)\\.service$`
75}
76
77/**
78 * Read-only companion to {@link buildPauseWorkersScript}. Separate because a
79 * plan's `satisfied()` must never mutate: if checking whether the workers are
80 * paused also paused them, a dry run would change the world.
81 */
82export function buildWorkersStateScript(slug: string, siteName: string): string {
83 return reportActiveUnits(backgroundUnitPattern(slug, siteName))
84}
85
86/** Emit one `active:<unit>` line per running unit matching `pattern`. */
87function reportActiveUnits(pattern: string): string {
88 return [
89 `ls /etc/systemd/system/ 2>/dev/null | grep -E ${sh(pattern)} | while read -r unit; do`,
90 ' systemctl is-active "$unit" >/dev/null 2>&1 && echo "active:$unit" || true',
91 'done',
92 ].join('\n')
93}
94
95/** Are the site's background units all stopped on the source? */
96export function parseWorkersPaused(output: string): boolean {
97 return !output.split('\n').some(line => line.trim().startsWith('active:'))
98}
99
100/**
101 * Archive the site's whole footprint on the source: its install tree and its
102 * unit files, in one tarball.
103 *
104 * `-h` is deliberately NOT passed: `current` and the shared-path symlinks must
105 * stay symlinks. Dereferencing them would flatten `current` into a second copy
106 * of the live release and turn every shared path into a per-release file again —
107 * which is the exact failure `sharedPaths` exists to prevent.
108 */
109export function buildSnapshotScript(slug: string, siteName: string, appBase: string): string {
110 const archive = siteMoveArchivePath(slug, siteName)
111 return [
112 'set -eu',
113 `test -d ${sh(appBase)} || { echo "no site tree at ${appBase}" >&2; exit 1; }`,
114 `rm -f ${sh(archive)}`,
115 // Two roots in one archive, each stored relative to its own parent so the
116 // restore can place them without a path-rewriting step.
117 `tar czf ${sh(archive)}`
118 + ` -C "$(dirname ${sh(appBase)})" "$(basename ${sh(appBase)})"`
119 + ` -C /etc/systemd/system $(cd /etc/systemd/system && ls ${siteUnitGlob(slug, siteName)} 2>/dev/null | tr '\\n' ' ')`,
120 `sha256sum ${sh(archive)} | cut -d' ' -f1`,
121 ].join('\n')
122}
123
124/**
125 * Unpack the archive on the target and bring the app up.
126 *
127 * The app unit is started but the gateway is not yet pointed at it: the site
128 * has to answer on loopback before anything is routed to it, and DNS is not
129 * touched until it has. Background units are enabled but NOT started — they stay
130 * paused until the source has been drained, so the two boxes can never both be
131 * running the same scheduler against the same data.
132 */
133export function buildRestoreScript(slug: string, siteName: string, appBase: string): string {
134 const archive = siteMoveArchivePath(slug, siteName)
135 return [
136 'set -eu',
137 `test -f ${sh(archive)} || { echo "archive not staged at ${archive}" >&2; exit 1; }`,
138 `mkdir -p "$(dirname ${sh(appBase)})"`,
139 // Extract the tree first, then the units, from the same archive.
140 `tar xzf ${sh(archive)} -C "$(dirname ${sh(appBase)})" "$(basename ${sh(appBase)})"`,
141 `tar xzf ${sh(archive)} -C /etc/systemd/system --wildcards ${sh(siteUnitGlob(slug, siteName))} 2>/dev/null || true`,
142 'systemctl daemon-reload',
143 `systemctl enable ${slug}-${siteName}.service 2>/dev/null || true`,
144 `systemctl restart ${slug}-${siteName}.service`,
145 `rm -f ${sh(archive)}`,
146 ].join('\n')
147}
148
149/** Is the site's tree already on the target, with its app unit running? */
150export function buildTargetStateScript(slug: string, siteName: string, appBase: string): string {
151 return [
152 `test -d ${sh(appBase)} && echo tree:present || echo tree:absent`,
153 `systemctl is-active ${slug}-${siteName}.service >/dev/null 2>&1 && echo unit:active || echo unit:inactive`,
154 ].join('\n')
155}
156
157/** Does the target already hold the site's tree AND run its app unit? */
158export function parseTargetReady(output: string): boolean {
159 return output.includes('tree:present') && output.includes('unit:active')
160}
161
162/**
163 * Health gate on the target, before any traffic is sent to it.
164 *
165 * Polls loopback rather than the public name on purpose: the public name still
166 * points at the SOURCE at this point in the plan, so asking it would cheerfully
167 * report the old box as healthy and wave a broken target through.
168 */
169export function buildHealthGateScript(port: number, path = '/', attempts = 10): string {
170 const url = `http://127.0.0.1:${port}${path.startsWith('/') ? path : `/${path}`}`
171 return [
172 'set -eu',
173 `for i in $(seq 1 ${attempts}); do`,
174 ` if curl -fsS -o /dev/null --max-time 5 ${sh(url)}; then echo healthy; exit 0; fi`,
175 ' sleep 3',
176 'done',
177 `echo "no healthy response from ${url}" >&2`,
178 'exit 1',
179 ].join('\n')
180}
181
182/**
183 * Drain the site on the source: stop and disable every unit, and drop its
184 * gateway fragment so the old box stops answering for it.
185 *
186 * The tree is left exactly where it is. That is what keeps the whole operation
187 * reversible: undoing a bad cutover is `systemctl start` plus pointing DNS back,
188 * with the data still sitting on the source. It stops being reversible when the
189 * source SERVER is deleted, which this operation deliberately does not do.
190 */
191export function buildDrainSourceScript(slug: string, siteName: string): string {
192 return [
193 'set -eu',
194 `for unit in $(ls /etc/systemd/system/ 2>/dev/null | grep -E ${sh(siteUnitPattern(slug, siteName))} || true); do`,
195 ' systemctl disable --now "$unit" 2>/dev/null || true',
196 'done',
197 `rm -f /etc/rpx/sites.d/${slug}.json 2>/dev/null || true`,
198 'systemctl reload rpx-gateway.service 2>/dev/null || systemctl restart rpx-gateway.service 2>/dev/null || true',
199 ].join('\n')
200}
201
202/** Every unit belonging to a site: the app, its template, and its background units. */
203function siteUnitPattern(slug: string, siteName: string): string {
204 return `^${slug}-${siteName}[-@.]`
205}
206
207/**
208 * Read-only companion to {@link buildDrainSourceScript} — see
209 * {@link buildWorkersStateScript} for why the two are not one script.
210 */
211export function buildSourceStateScript(slug: string, siteName: string): string {
212 return reportActiveUnits(siteUnitPattern(slug, siteName))
213}
214
215/** Is the site fully stopped on the source? */
216export function parseSourceDrained(output: string): boolean {
217 return !output.split('\n').some(line => line.trim().startsWith('active:'))
218}
219
220/**
221 * The side effects a move needs, injected so the operation is testable without
222 * two live boxes, a provider, or a DNS account.
223 */
224export interface SiteMoveEffects {
225 /** Run a script on the source box; resolves with stdout, rejects on failure. */
226 runOnSource: (script: string) => Promise<string>
227 /** Run a script on the target box. */
228 runOnTarget: (script: string) => Promise<string>
229 /**
230 * Carry the staged archive from the source to the target. Separate from the
231 * two exec effects because how bytes get between two boxes is a deployment
232 * decision — through the operator, over a direct SSH hop, via object storage —
233 * and none of it belongs in this plan.
234 */
235 transferArchive: () => Promise<void>
236 /** Is the archive already staged on the target? Lets the transfer resume. */
237 archiveStaged: () => Promise<boolean>
238 /** Point the site's hostname at the target. Resolves with any provider warnings. */
239 cutoverDns: () => Promise<string[]>
240 /** Address the site's hostname currently publishes, for the cutover check. */
241 publishedAddress: () => Promise<string | undefined>
242 /** Refresh the target's gateway so it routes the site. */
243 refreshTargetGateway: () => Promise<void>
244 /** Does the target's gateway already route this site? */
245 targetRoutesSite: () => Promise<boolean>
246}
247
248export interface SiteMoveOptions {
249 slug: string
250 siteName: string
251 /** Install base on both boxes — `siteInstallBase(slug, siteName)`. */
252 appBase: string
253 /** Source and target box names, for the plan's own prose. */
254 from: string
255 to: string
256 /** Target's public address, which DNS is cut over to. */
257 targetAddress: string
258 /** Loopback port the health gate polls. Omitted for a portless site. */
259 port?: number
260 /** Health-check path, defaulting to `/`. */
261 healthCheckPath?: string
262}
263
264/**
265 * Build the plan that moves `siteName` from one box to another.
266 *
267 * Async because the preconditions — the site exists on the source, the target is
268 * not already serving a different tree there — are checked here rather than
269 * becoming steps. A precondition is not a unit of work.
270 */
271export async function planSiteMove(options: SiteMoveOptions, effects: SiteMoveEffects): Promise<OperationPlan> {
272 const { slug, siteName, appBase, from, to, targetAddress } = options
273
274 if (from === to) throw new Error(`'${siteName}' is already on ${to}.`)
275
276 const steps: OperationStep[] = [
277 {
278 id: 'pause-workers',
279 title: `Stop background work on ${from} so the snapshot is consistent`,
280 satisfied: async () => parseWorkersPaused(await effects.runOnSource(buildWorkersStateScript(slug, siteName))),
281 apply: async () => {
282 await effects.runOnSource(buildPauseWorkersScript(slug, siteName))
283 },
284 },
285 {
286 id: 'snapshot',
287 title: `Archive the site's tree and units on ${from}`,
288 change: { from: `${from}:${appBase}`, to: siteMoveArchivePath(slug, siteName) },
289 // Always re-taken: an archive from an earlier attempt predates whatever
290 // the source served since, and a move that silently shipped stale data
291 // would be worse than one that copies twice.
292 satisfied: async () => false,
293 apply: async () => {
294 await effects.runOnSource(buildSnapshotScript(slug, siteName, appBase))
295 },
296 },
297 {
298 id: 'transfer',
299 title: `Carry the archive to ${to}`,
300 change: { from, to },
301 satisfied: () => effects.archiveStaged(),
302 apply: () => effects.transferArchive(),
303 },
304 {
305 id: 'restore',
306 title: `Unpack the site on ${to} and start it`,
307 change: { from: siteMoveArchivePath(slug, siteName), to: `${to}:${appBase}` },
308 satisfied: async () => parseTargetReady(await effects.runOnTarget(buildTargetStateScript(slug, siteName, appBase))),
309 apply: async () => {
310 await effects.runOnTarget(buildRestoreScript(slug, siteName, appBase))
311 },
312 },
313 ]
314
315 // A portless site — a worker or a scheduler — has nothing to poll, and gating
316 // on a port it never binds would fail every move of one.
317 if (options.port != null) {
318 const { port } = options
319 steps.push({
320 id: 'health',
321 title: `Wait for the site to answer on ${to} (127.0.0.1:${port})`,
322 // Re-run every time: this is the gate that decides whether traffic may be
323 // moved, and a gate that remembers a previous pass is not a gate.
324 satisfied: async () => false,
325 apply: async () => {
326 await effects.runOnTarget(buildHealthGateScript(port, options.healthCheckPath ?? '/'))
327 },
328 })
329 }
330
331 steps.push(
332 {
333 id: 'gateway',
334 title: `Route the site on ${to}`,
335 satisfied: () => effects.targetRoutesSite(),
336 apply: () => effects.refreshTargetGateway(),
337 },
338 {
339 id: 'dns',
340 title: `Point DNS at ${to}`,
341 change: { from: `${from}`, to: targetAddress },
342 satisfied: async () => (await effects.publishedAddress()) === targetAddress,
343 apply: async () => {
344 const warnings = await effects.cutoverDns()
345 // A cutover that reported success while editing nothing is the failure
346 // this whole ordering exists to avoid; refuse to continue to the drain.
347 if (warnings.length > 0) throw new Error(warnings.join('; '))
348 },
349 },
350 {
351 id: 'drain-source',
352 title: `Stop the site on ${from}, leaving its files in place`,
353 satisfied: async () => parseSourceDrained(await effects.runOnSource(buildSourceStateScript(slug, siteName))),
354 apply: async () => {
355 await effects.runOnSource(buildDrainSourceScript(slug, siteName))
356 },
357 },
358 )
359
360 return { operation: 'site:move', target: siteName, steps }
361}