feat(fleet): move a deployed site to another server (#167 operation 2)

packages/ts-cloud/src/operations/site-move.test.tsadded+287-0

Changes to packages/ts-cloud/src/operations/site-move.test.ts

		@@ -0,0 +1,287 @@
	1	+import type { SiteMoveEffects } from './site-move'
	2	+import { describe, expect, it } from 'bun:test'
	3	+import { applyPlan, formatPlan, resolvePlan } from './plan'
	4	+import {
	5	+ buildDrainSourceScript,
	6	+ buildHealthGateScript,
	7	+ buildPauseWorkersScript,
	8	+ buildRestoreScript,
	9	+ buildSnapshotScript,
	10	+ buildSourceStateScript,
	11	+ buildWorkersStateScript,
	12	+ parseSourceDrained,
	13	+ parseTargetReady,
	14	+ parseWorkersPaused,
	15	+ planSiteMove,
	16	+ siteMoveArchivePath,
	17	+
	18	+} from './site-move'
	19	+
	20	+const options = {
	21	+ slug: 'hq',
	22	+ siteName: 'bughq',
	23	+ appBase: '/var/www/hq-bughq',
	24	+ from: 'bughq-box',
	25	+ to: 'statushq-box',
	26	+ targetAddress: '203.0.113.9',
	27	+ port: 3010,
	28	+}
	29	+
	30	+/** A world where the move has not started: source serving, target empty. */
	31	+function world() {
	32	+ const state = {
	33	+ workersRunning: true,
	34	+ onTarget: false,
	35	+ targetRunning: false,
	36	+ staged: false,
	37	+ published: '203.0.113.1',
	38	+ routed: false,
	39	+ sourceRunning: true,
	40	+ ran: [] as string[],
	41	+ }
	42	+ const effects: SiteMoveEffects = {
	43	+ runOnSource: async (script) => {
	44	+ state.ran.push(`source:${script.slice(0, 24)}`)
	45	+ if (script === buildWorkersStateScript('hq', 'bughq'))
	46	+ return state.workersRunning ? 'active:hq-bughq-scheduler.service' : ''
	47	+ if (script === buildSourceStateScript('hq', 'bughq'))
	48	+ return state.sourceRunning ? 'active:hq-bughq.service' : ''
	49	+ if (script.startsWith('set -eu\nfor unit') && script.includes('systemctl stop')) state.workersRunning = false
	50	+ if (script.includes('disable --now')) { state.sourceRunning = false; state.workersRunning = false }
	51	+ if (script.includes('tar czf')) state.staged = false
	52	+ return ''
	53	+ },
	54	+ runOnTarget: async (script) => {
	55	+ state.ran.push(`target:${script.slice(0, 24)}`)
	56	+ if (script.startsWith('test -d'))
	57	+ return `${state.onTarget ? 'tree:present' : 'tree:absent'}\n${state.targetRunning ? 'unit:active' : 'unit:inactive'}`
	58	+ if (script.includes('tar xzf')) { state.onTarget = true; state.targetRunning = true }
	59	+ if (script.includes('curl')) {
	60	+ if (!state.targetRunning) throw new Error('no healthy response')
	61	+ return 'healthy'
	62	+ }
	63	+ return ''
	64	+ },
	65	+ transferArchive: async () => { state.staged = true },
	66	+ archiveStaged: async () => state.staged,
	67	+ cutoverDns: async () => { state.published = options.targetAddress; return [] },
	68	+ publishedAddress: async () => state.published,
	69	+ refreshTargetGateway: async () => { state.routed = true },
	70	+ targetRoutesSite: async () => state.routed,
	71	+ }
	72	+ return { state, effects }
	73	+}
	74	+
	75	+describe('planSiteMove ordering', () => {
	76	+ /**
	77	+ * Every prefix of the plan has to be a working system — on the old box or the
	78	+ * new one. The source keeps serving until the target passes a health gate,
	79	+ * DNS moves only after that, and the source is drained last.
	80	+ */
	81	+ it('gates health before routing, routes before DNS, and drains last', async () => {
	82	+ const p = await planSiteMove(options, world().effects)
	83	+ expect(p.steps.map(step => step.id)).toEqual([
	84	+ 'pause-workers',
	85	+ 'snapshot',
	86	+ 'transfer',
	87	+ 'restore',
	88	+ 'health',
	89	+ 'gateway',
	90	+ 'dns',
	91	+ 'drain-source',
	92	+ ])
	93	+ })
	94	+
	95	+ /** A worker or scheduler binds no port; gating on one would fail every move. */
	96	+ it('omits the health gate for a portless site', async () => {
	97	+ const p = await planSiteMove({ ...options, port: undefined }, world().effects)
	98	+ expect(p.steps.map(step => step.id)).not.toContain('health')
	99	+ })
	100	+
	101	+ it('refuses a move to the box the site is already on', async () => {
	102	+ await expect(planSiteMove({ ...options, to: options.from }, world().effects)).rejects.toThrow('already on')
	103	+ })
	104	+
	105	+ /**
	106	+ * Nothing here deletes: the source tree survives, so a bad cutover is undone
	107	+ * by starting the units again. Irreversibility begins at `server:destroy`.
	108	+ */
	109	+ it('declares no destructive step, because the source is drained and not deleted', async () => {
	110	+ const p = await planSiteMove(options, world().effects)
	111	+ expect(p.steps.some(step => step.destructive)).toBe(false)
	112	+ })
	113	+})
	114	+
	115	+describe('planSiteMove execution', () => {
	116	+ it('moves the site end to end', async () => {
	117	+ const { state, effects } = world()
	118	+ const p = await planSiteMove(options, effects)
	119	+ const outcome = await applyPlan(p, await resolvePlan(p))
	120	+ expect(outcome.success).toBe(true)
	121	+ expect(state.onTarget).toBe(true)
	122	+ expect(state.targetRunning).toBe(true)
	123	+ expect(state.routed).toBe(true)
	124	+ expect(state.published).toBe('203.0.113.9')
	125	+ expect(state.sourceRunning).toBe(false)
	126	+ })
	127	+
	128	+ /**
	129	+ * A cutover that reports success while editing nothing is the failure the
	130	+ * whole ordering exists to avoid — it must not reach the drain.
	131	+ */
	132	+ it('refuses to drain the source when DNS reported a warning', async () => {
	133	+ const { state, effects } = world()
	134	+ const p = await planSiteMove(options, {
	135	+ ...effects,
	136	+ cutoverDns: async () => ['bughq.example.com → 203.0.113.9 failed: zone not found'],
	137	+ })
	138	+ const outcome = await applyPlan(p, await resolvePlan(p))
	139	+ expect(outcome.success).toBe(false)
	140	+ expect(outcome.steps.at(-1)?.id).toBe('dns')
	141	+ expect(state.sourceRunning).toBe(true)
	142	+ })
	143	+
	144	+ it('leaves the source serving when the target never becomes healthy', async () => {
	145	+ const { state, effects } = world()
	146	+ const p = await planSiteMove(options, {
	147	+ ...effects,
	148	+ runOnTarget: async (script) => {
	149	+ if (script.startsWith('test -d')) return 'tree:absent\nunit:inactive'
	150	+ if (script.includes('curl')) throw new Error('no healthy response from http://127.0.0.1:3010/')
	151	+ return ''
	152	+ },
	153	+ })
	154	+ const outcome = await applyPlan(p, await resolvePlan(p))
	155	+ expect(outcome.success).toBe(false)
	156	+ expect(outcome.steps.at(-1)?.id).toBe('health')
	157	+ expect(state.published).toBe('203.0.113.1')
	158	+ expect(state.sourceRunning).toBe(true)
	159	+ })
	160	+
	161	+ /** The point of the scaffolding: a run that died is resumed, not restarted. */
	162	+ it('resumes after a failed cutover without re-transferring', async () => {
	163	+ const { state, effects } = world()
	164	+ let transfers = 0
	165	+ const counted = { ...effects, transferArchive: async () => { transfers++; await effects.transferArchive() } }
	166	+ let failDns = true
	167	+ const withFlakyDns = {
	168	+ ...counted,
	169	+ cutoverDns: async () => (failDns ? ['provider timed out'] : counted.cutoverDns()),
	170	+ }
	171	+
	172	+ const first = await planSiteMove(options, withFlakyDns)
	173	+ expect((await applyPlan(first, await resolvePlan(first))).success).toBe(false)
	174	+ expect(transfers).toBe(1)
	175	+
	176	+ failDns = false
	177	+ const second = await planSiteMove(options, withFlakyDns)
	178	+ const resolved = await resolvePlan(second)
	179	+ // Everything up to the cutover is already done and skips itself.
	180	+ expect(resolved.find(item => item.step.id === 'transfer')?.state).toBe('satisfied')
	181	+ expect(resolved.find(item => item.step.id === 'restore')?.state).toBe('satisfied')
	182	+ expect((await applyPlan(second, resolved)).success).toBe(true)
	183	+ expect(transfers).toBe(1)
	184	+ expect(state.sourceRunning).toBe(false)
	185	+ })
	186	+
	187	+ /**
	188	+ * The health gate decides whether traffic may move. A gate that remembers a
	189	+ * previous pass is not a gate.
	190	+ */
	191	+ it('re-runs the health gate on every attempt', async () => {
	192	+ const p = await planSiteMove(options, world().effects)
	193	+ const resolved = await resolvePlan(p)
	194	+ expect(resolved.find(item => item.step.id === 'health')?.state).toBe('pending')
	195	+ expect(resolved.find(item => item.step.id === 'snapshot')?.state).toBe('pending')
	196	+ })
	197	+
	198	+ it('prints a plan that names both boxes before touching anything', async () => {
	199	+ const { state, effects } = world()
	200	+ const p = await planSiteMove(options, effects)
	201	+ const lines = formatPlan(p, await resolvePlan(p)).join('\n')
	202	+ expect(lines).toContain('site:move bughq')
	203	+ expect(lines).toContain('bughq-box:/var/www/hq-bughq → /tmp/ts-cloud-move-hq-bughq.tar.gz')
	204	+ expect(lines).toContain('bughq-box → 203.0.113.9')
	205	+ // A plan changes nothing.
	206	+ expect(state.onTarget).toBe(false)
	207	+ expect(state.sourceRunning).toBe(true)
	208	+ expect(state.workersRunning).toBe(true)
	209	+ })
	210	+})
	211	+
	212	+describe('remote scripts', () => {
	213	+ /**
	214	+ * Dereferencing symlinks would flatten `current` into a second copy of the
	215	+ * live release and turn every shared path back into a per-release file — the
	216	+ * exact failure `sharedPaths` exists to prevent.
	217	+ */
	218	+ it('archives the tree without dereferencing symlinks', () => {
	219	+ const script = buildSnapshotScript('hq', 'bughq', '/var/www/hq-bughq')
	220	+ expect(script).toContain('tar czf')
	221	+ expect(script).not.toContain('tar czhf')
	222	+ expect(script).not.toContain(' -h ')
	223	+ expect(script).toContain('hq-bughq*.service')
	224	+ })
	225	+
	226	+ it('carries the unit files alongside the tree', () => {
	227	+ expect(buildRestoreScript('hq', 'bughq', '/var/www/hq-bughq')).toContain('-C /etc/systemd/system')
	228	+ expect(buildSnapshotScript('hq', 'bughq', '/var/www/hq-bughq')).toContain('-C /etc/systemd/system')
	229	+ })
	230	+
	231	+ /** Two boxes running one scheduler against one dataset is the thing to avoid. */
	232	+ it('starts the app on the target but not its background units', () => {
	233	+ const script = buildRestoreScript('hq', 'bughq', '/var/www/hq-bughq')
	234	+ expect(script).toContain('systemctl restart hq-bughq.service')
	235	+ expect(script).not.toContain('scheduler')
	236	+ expect(script).not.toContain('queue')
	237	+ })
	238	+
	239	+ /** The public name still points at the SOURCE at gate time. */
	240	+ it('polls loopback, never the public name', () => {
	241	+ const script = buildHealthGateScript(3010, '/health')
	242	+ expect(script).toContain('http://127.0.0.1:3010/health')
	243	+ expect(script).not.toContain('bughq')
	244	+ })
	245	+
	246	+ it('normalizes a health path without a leading slash', () => {
	247	+ expect(buildHealthGateScript(3010, 'health')).toContain('http://127.0.0.1:3010/health')
	248	+ })
	249	+
	250	+ it('pauses only background units, leaving the site serving', () => {
	251	+ const script = buildPauseWorkersScript('hq', 'bughq')
	252	+ expect(script).toContain('queue|daemon')
	253	+ expect(script).toContain('scheduler')
	254	+ expect(script).toContain('systemctl stop')
	255	+ expect(script).not.toContain('disable')
	256	+ })
	257	+
	258	+ it('drains every unit on the source but deletes no files', () => {
	259	+ const script = buildDrainSourceScript('hq', 'bughq')
	260	+ expect(script).toContain('systemctl disable --now')
	261	+ expect(script).toContain('rm -f /etc/rpx/sites.d/hq.json')
	262	+ expect(script).not.toContain('/var/www')
	263	+ expect(script).not.toContain('rm -rf')
	264	+ })
	265	+
	266	+ /** A dry run must not change the world — the checks only report. */
	267	+ it('checks state without mutating it', () => {
	268	+ for (const script of [buildWorkersStateScript('hq', 'bughq'), buildSourceStateScript('hq', 'bughq')]) {
	269	+ expect(script).not.toContain('systemctl stop')
	270	+ expect(script).not.toContain('systemctl disable')
	271	+ expect(script).not.toContain('rm ')
	272	+ }
	273	+ })
	274	+
	275	+ it('parses unit and tree state', () => {
	276	+ expect(parseWorkersPaused('')).toBe(true)
	277	+ expect(parseWorkersPaused('active:hq-bughq-scheduler.service')).toBe(false)
	278	+ expect(parseSourceDrained('active:hq-bughq.service')).toBe(false)
	279	+ expect(parseTargetReady('tree:present\nunit:active')).toBe(true)
	280	+ expect(parseTargetReady('tree:present\nunit:inactive')).toBe(false)
	281	+ expect(parseTargetReady('tree:absent\nunit:active')).toBe(false)
	282	+ })
	283	+
	284	+ it('stages the archive under a slug- and site-scoped name', () => {
	285	+ expect(siteMoveArchivePath('hq', 'bughq')).toBe('/tmp/ts-cloud-move-hq-bughq.tar.gz')
	286	+ })
	287	+})