feat(deploy): make host cleanup a property of the host, not of deploying

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

docs/features/releases.mdmodified+44-0

Changes to docs/features/releases.md

		⋯95 linesShow the 95 lines above this hunk@@ -96,6 +96,50 @@ cloud release:pin <release-id> --remove
96	96
97	97	Retention candidates never include active, activating, awaiting-approval, pinned, or currently referenced rollback releases. The store keeps a configurable minimum per resource and exposes candidates rather than deleting provider artifacts implicitly.
98	98
	99	+## Host disk retention
	100	+
	101	+Release history is one thing a box accumulates; deploy machinery is another. Compute hosts also carry a content-addressed upload cache, staging tarballs, package caches, container images, and journals. These are reclaimed by a cleanup pass that runs **on a systemd timer as well as after each deploy** — a box that stops deploying keeps cleaning itself, and a deploy that fails no longer skips the pass that would have collected its leftovers.
	102	+
	103	+The pass reads root-filesystem usage before it does anything, and behaves differently by band:
	104	+
	105	+| Root usage | Behavior |
	106	+| --- | --- |
	107	+| Below `relaxedBelowPercent` (default 50%) | Reclaims deploy leftovers only. Caches that cost network to rebuild — the Bun download cache, container images, apt — are left alone. |
	108	+| Between the two thresholds | Full pass at the configured windows. |
	109	+| At or above `escalateAtPercent` (default 85%) | Full pass with every configurable window divided by `escalationFactor` (default 4). |
	110	+
	111	+Bounds that exist for correctness rather than retention never move with pressure: an in-flight upload belonging to a concurrent deploy on a shared box survives its hour however full the disk is.
	112	+
	113	+Defaults suit a typical box. A host packing many sites onto one disk is the case worth tuning:
	114	+
	115	+```ts
	116	+infrastructure: {
	117	+ compute: {
	118	+ cleanup: {
	119	+ // The upload cache mostly pays for itself within a single deploy — one
	120	+ // upload, N sites copying it locally seconds later. Keep it short.
	121	+ artifactMaxAgeMinutes: 720,
	122	+ escalateAtPercent: 80,
	123	+ onCalendar: 'daily',
	124	+ },
	125	+ },
	126	+}
	127	+```
	128	+
	129	+Components that write timestamped output to paths ts-cloud does not own register their own rules rather than accumulating unattended:
	130	+
	131	+```ts
	132	+cleanup: {
	133	+ rules: [
	134	+ { path: '/root/rpx-backups', pattern: 'cert-backup-*', maxAgeMinutes: 4320, directories: true },
	135	+ ],
	136	+}
	137	+```
	138	+
	139	+Each run prints root usage before and after, so the deploy log carries the trend. Set `timer: false` to keep the deploy-time pass and schedule the host's cleanup yourself.
	140	+
	141	+Rollback depth is separate and unchanged: `keepReleases` (default 4) governs how many past releases stay on the box, and cleanup never touches active or rollback releases.
	142	+
99	143	## Automation API
100	144
101	145	The versioned OpenAPI document and TypeScript client expose:

packages/core/src/types.tsmodified+77-0

Changes to packages/core/src/types.ts

		⋯2,691 linesShow the 2,691 lines above this hunk@@ -2692,6 +2692,75 @@ export interface ContainerItemConfig {
2692	2692	}
2693	2693	}
2694	2694
	2695	+/**
	2696	+ * A directory whose aged files a host may reclaim.
	2697	+ *
	2698	+ * Components that write timestamped output outside the paths ts-cloud owns —
	2699	+ * backup dirs, rollback snapshots, exported certs — register a rule here rather
	2700	+ * than leaving it to accumulate unattended.
	2701	+ */
	2702	+export interface HostCleanupRuleConfig {
	2703	+ /** Absolute directory to prune. A missing directory is skipped silently. */
	2704	+ path: string
	2705	+ /** `find -name` pattern. @default '*' */
	2706	+ pattern?: string
	2707	+ /** Delete entries last modified more than this many minutes ago. */
	2708	+ maxAgeMinutes: number
	2709	+ /** Descend below the top level. @default false */
	2710	+ recursive?: boolean
	2711	+ /** Delete matching directories rather than files. @default false */
	2712	+ directories?: boolean
	2713	+ /** Remove directories left empty afterwards. @default false */
	2714	+ pruneEmptyDirs?: boolean
	2715	+ /** Exempt this window from disk-pressure shortening. @default false */
	2716	+ fixedWindow?: boolean
	2717	+ /** Skip while usage is below {@link HostCleanupConfig.relaxedBelowPercent}. @default false */
	2718	+ onlyUnderPressure?: boolean
	2719	+}
	2720	+
	2721	+/**
	2722	+ * Host disk retention.
	2723	+ *
	2724	+ * Cleanup runs on a timer as well as after each deploy, and reads root-filesystem
	2725	+ * usage first: below `relaxedBelowPercent` it leaves caches that cost network to
	2726	+ * rebuild, and at or above `escalateAtPercent` it divides every window here by
	2727	+ * `escalationFactor`. Bounds that exist for correctness rather than retention —
	2728	+ * an in-flight upload belonging to a concurrent deploy — never move.
	2729	+ */
	2730	+export interface HostCleanupConfig {
	2731	+ /**
	2732	+ * Content-addressed upload cache window. The dominant payoff is intra-deploy
	2733	+ * reuse — one upload, N sites copying it locally seconds later — so the
	2734	+ * default is deliberately short. @default 2880 (2 days)
	2735	+ */
	2736	+ artifactMaxAgeMinutes?: number
	2737	+ /** In-flight upload/staging bound. @default 60 */
	2738	+ stagingMaxAgeMinutes?: number
	2739	+ /** Bun download cache window. @default 10080 (7 days) */
	2740	+ bunCacheMaxAgeMinutes?: number
	2741	+ /** `journalctl --vacuum-time`. @default '14d' */
	2742	+ journalMaxAge?: string
	2743	+ /** `journalctl --vacuum-size`. @default '512M' */
	2744	+ journalMaxSize?: string
	2745	+ /** Container image prune cutoff. @default '168h' */
	2746	+ imageMaxAge?: string
	2747	+ /** Extra retention rules for paths ts-cloud does not own. */
	2748	+ rules?: HostCleanupRuleConfig[]
	2749	+ /** Below this root usage %, keep caches that cost network to rebuild. @default 50 */
	2750	+ relaxedBelowPercent?: number
	2751	+ /** At or above this root usage %, shorten every configurable window. @default 85 */
	2752	+ escalateAtPercent?: number
	2753	+ /** Divisor applied to configurable windows when escalating. @default 4 */
	2754	+ escalationFactor?: number
	2755	+ /**
	2756	+ * Install a systemd timer so the host keeps cleaning itself once it stops
	2757	+ * deploying. @default true
	2758	+ */
	2759	+ timer?: boolean
	2760	+ /** Timer cadence, as a systemd `OnCalendar=` expression. @default 'daily' */
	2761	+ onCalendar?: string
	2762	+}
	2763	+
2695	2764	export interface ComputeConfig {
2696	2765	/**
2697	2766	* Compute mode: 'server' for EC2, 'serverless' for Fargate/Lambda
		⋯371 linesShow the 371 lines above this hunk@@ -3069,6 +3138,14 @@ export interface ComputeConfig {
3069	3138	*/
3070	3139	nginxTemplates?: Record<string, string[]>
3071	3140
	3141	+ /**
	3142	+ * Host disk retention. ts-cloud reclaims stale deploy artifacts and caches on
	3143	+ * a timer and after every deploy; this tunes what it keeps. Omit for defaults
	3144	+ * sized to a typical box — a host packing many sites onto one disk is the
	3145	+ * case worth tuning.
	3146	+ */
	3147	+ cleanup?: HostCleanupConfig
	3148	+
3072	3149	/**
3073	3150	* On-box managed services to install (Forge's single-server model): the
3074	3151	* database engine, cache, and search. Each may be `true` for defaults or an

packages/ts-cloud/src/drivers/hetzner/driver.tsmodified+4-1

Changes to packages/ts-cloud/src/drivers/hetzner/driver.ts

		⋯1,111 linesShow the 1,111 lines above this hunk@@ -1112,6 +1112,10 @@ export class HetznerDriver implements CloudDriver {
1112	1112	// publish it, then copy it to this deployment's unique staging path.
1113	1113	}
1114	1114
	1115	+ // Retention for this directory — both the published `*.tar.gz` and the
	1116	+ // `.tmp` files a died-mid-SCP upload strands here — lives in
	1117	+ // buildHostCleanupScript, which runs unconditionally and on a timer.
	1118	+ // Pruning here instead meant it only ever ran on a cache MISS.
1115	1119	const uploadPath = `${artifactDir}/.${digest}-${randomUUID()}.tmp`
1116	1120	this.scpToHost(target.publicIp, options.localPath, uploadPath)
1117	1121	this.sshExec(
		⋯3 linesShow the 3 lines above this hunk@@ -1121,7 +1125,6 @@ export class HetznerDriver implements CloudDriver {
1121	1125	`chmod 600 ${shellQuote(uploadPath)}`,
1122	1126	`mv -f -- ${shellQuote(uploadPath)} ${shellQuote(cachedPath)}`,
1123	1127	`cp -- ${shellQuote(cachedPath)} ${shellQuote(remotePath)}`,
1124		- `find ${shellQuote(artifactDir)} -type f -name '*.tar.gz' -mtime +7 -delete 2>/dev/null || true`,
1125	1128	].join('\n'),
1126	1129	)
1127	1130	}

packages/ts-cloud/src/drivers/shared/compute-deploy.tsmodified+1-1

Changes to packages/ts-cloud/src/drivers/shared/compute-deploy.ts

		⋯321 linesShow the 321 lines above this hunk@@ -322,7 +322,7 @@ export async function deploySiteRelease(
322	322	...staticVhost,
323	323	...staticSsl,
324	324	...servicesScript,
325		- ...buildHostCleanupScript(),
	325	+ ...buildHostCleanupScript(compute?.cleanup),
326	326	]
327	327
328	328	logger.step(`Deploying to ${targets.length} target(s)...`)

packages/ts-cloud/src/drivers/shared/deploy-script.tsmodified+154-19

Changes to packages/ts-cloud/src/drivers/shared/deploy-script.ts

		⋯9 linesShow the 9 lines above this hunk@@ -10,7 +10,7 @@
10	10	* release (no window where the code is half-replaced). Old releases are kept for
11	11	* instant rollback. See {@link import('./releases')}.
12	12	*/
13		-import type { SharedPathEntry } from '@ts-cloud/core'
	13	+import type { HostCleanupConfig, HostCleanupRuleConfig, SharedPathEntry } from '@ts-cloud/core'
14	14	import { formatEnvFile } from './env-file'
15	15	import { buildActivateRelease, buildDeployLock, buildEnsureReleaseLayout, buildLinkSharedPaths, buildPromoteStagedRelease, buildPruneReleases, buildResetReleaseDir, buildStrandedReleaseTrap, dedupeSharedPaths, DEFAULT_KEEP_RELEASES, releasePaths } from './releases'
16	16	import { sqliteSharedPaths } from './sqlite-shared-path'
		⋯734 linesShow the 734 lines above this hunk@@ -751,28 +751,163 @@ export function buildLocalArtifactFetch(localPath: string, destPath: string): st
751	751	]
752	752	}
753	753
	754	+/** Root-filesystem usage (%) at or above which retention windows are shortened. */
	755	+const DEFAULT_ESCALATE_AT_PERCENT = 85
	756	+/** Root-filesystem usage (%) below which caches worth re-downloading are left alone. */
	757	+const DEFAULT_RELAXED_BELOW_PERCENT = 50
	758	+/** Divisor applied to configurable windows once {@link DEFAULT_ESCALATE_AT_PERCENT} is reached. */
	759	+const DEFAULT_ESCALATION_FACTOR = 4
	760	+
	761	+const CLEANUP_SCRIPT_PATH = '/usr/local/sbin/ts-cloud-cleanup'
	762	+const CLEANUP_UNIT = 'ts-cloud-cleanup'
	763	+
	764	+function resolveCleanupRules(options: HostCleanupConfig): HostCleanupRuleConfig[] {
	765	+ const staging = Math.max(1, options.stagingMaxAgeMinutes ?? 60)
	766	+ return [
	767	+ { path: '/var/ts-cloud/staging', maxAgeMinutes: staging, fixedWindow: true },
	768	+ { path: '/tmp', pattern: '*-release.tar.gz', maxAgeMinutes: staging, fixedWindow: true },
	769	+ // Content-addressed upload cache. Retention used to be inlined in the
	770	+ // Hetzner driver's upload path, where it only ran on a cache MISS — so the
	771	+ // box reusing the cache most pruned it least — and its `*.tar.gz` filter
	772	+ // could never match the `.<digest>-<uuid>.tmp` files a died-mid-SCP upload
	773	+ // leaves behind, which therefore leaked permanently. Both belong here.
	774	+ { path: '/var/ts-cloud/artifacts', pattern: '*.tar.gz', maxAgeMinutes: Math.max(1, options.artifactMaxAgeMinutes ?? 2880) },
	775	+ // Same case the staging rule above already handles, one directory over.
	776	+ { path: '/var/ts-cloud/artifacts', pattern: '.*.tmp', maxAgeMinutes: staging, fixedWindow: true },
	777	+ // Retained releases contain installed dependencies; these are only download
	778	+ // caches. Left alone on a host with room, since re-fetching costs network.
	779	+ {
	780	+ path: '/root/.bun/install/cache',
	781	+ maxAgeMinutes: Math.max(1, options.bunCacheMaxAgeMinutes ?? 10080),
	782	+ recursive: true,
	783	+ pruneEmptyDirs: true,
	784	+ onlyUnderPressure: true,
	785	+ },
	786	+ ...(options.rules ?? []),
	787	+ ]
	788	+}
	789	+
	790	+function renderCleanupRule(rule: HostCleanupRuleConfig): string[] {
	791	+ const depth = rule.recursive ? '' : ' -maxdepth 1'
	792	+ const name = rule.pattern ? ` -name ${JSON.stringify(rule.pattern)}` : ''
	793	+ const type = rule.directories ? ' -type d' : ' -type f'
	794	+ // Fixed windows are literals; the rest divide by the pressure factor the
	795	+ // preamble resolved, so one script covers both the normal and escalated pass.
	796	+ const age = rule.fixedWindow ? `${rule.maxAgeMinutes}` : `$((${rule.maxAgeMinutes} / TS_CLOUD_DIV))`
	797	+ const find = `find ${rule.path} -xdev${depth}${rule.directories ? ' -depth' : ''}${type}${name} -mmin +${age} -delete 2>/dev/null || true`
	798	+ const lines = [find]
	799	+ if (rule.pruneEmptyDirs)
	800	+ lines.push(`find ${rule.path} -xdev -depth -type d -empty -delete 2>/dev/null || true`)
	801	+ return rule.onlyUnderPressure ? lines.map((line) => ` ${line}`) : lines
	802	+}
	803	+
754	804	/**
755		- * Bounded, multi-tenant-safe host maintenance run after a successful deploy.
756		- * Current and rollback releases are deliberately out of scope: their retention
757		- * is managed by buildPruneReleases. This only removes abandoned staging/temp
758		- * archives, aged package caches, unused container images, and old journals.
	805	+ * The cleanup pass itself, as a POSIX `sh` program.
	806	+ *
	807	+ * Bounded and multi-tenant-safe. Current and rollback releases are deliberately
	808	+ * out of scope: their retention is `buildPruneReleases`' job. This only removes
	809	+ * abandoned staging/temp archives, aged upload and package caches, unused
	810	+ * container images, and old journals.
759	811	*/
760		-export function buildHostCleanupScript(): string[] {
	812	+function buildCleanupProgram(options: HostCleanupConfig): string[] {
	813	+ const relaxedBelow = options.relaxedBelowPercent ?? DEFAULT_RELAXED_BELOW_PERCENT
	814	+ const escalateAt = options.escalateAtPercent ?? DEFAULT_ESCALATE_AT_PERCENT
	815	+ const factor = Math.max(1, options.escalationFactor ?? DEFAULT_ESCALATION_FACTOR)
	816	+ const rules = resolveCleanupRules(options)
	817	+ const [pressured, always] = [
	818	+ rules.filter((rule) => rule.onlyUnderPressure),
	819	+ rules.filter((rule) => !rule.onlyUnderPressure),
	820	+ ]
	821	+
761	822	return [
762	823	'echo "[ts-cloud] host cleanup (disk before): $(df -h / | tail -1)"',
763		- // One hour protects concurrent deploys on a shared box while bounding
764		- // uploads stranded by failed or cancelled deploys.
765		- 'find /var/ts-cloud/staging -xdev -maxdepth 1 -type f -mmin +60 -delete 2>/dev/null || true',
766		- 'find /tmp -xdev -maxdepth 1 -type f -name "*-release.tar.gz" -mmin +60 -delete 2>/dev/null || true',
767		- // Retained releases contain installed dependencies; these are only
768		- // download caches. Keep a week to avoid unnecessary network churn.
769		- 'find /root/.bun/install/cache -xdev -type f -mtime +7 -delete 2>/dev/null || true',
770		- 'find /root/.bun/install/cache -xdev -depth -type d -empty -delete 2>/dev/null || true',
771		- 'journalctl --vacuum-time=14d --vacuum-size=512M >/dev/null 2>&1 || true',
772		- // Only unused, week-old images qualify. Daemon-less hosts skip this.
773		- 'if command -v docker >/dev/null 2>&1; then docker image prune --all --force --filter "until=168h" >/dev/null 2>&1 || true; fi',
774		- 'if command -v podman >/dev/null 2>&1; then podman image prune --all --force --filter "until=168h" >/dev/null 2>&1 || true; fi',
775		- 'if command -v apt-get >/dev/null 2>&1; then apt-get clean >/dev/null 2>&1 || true; fi',
	824	+ // Root usage drives the whole pass. Anything unparseable reads as 0, which
	825	+ // is the conservative answer: cheap rules only, no shortened windows.
	826	+ // `|| true` matters: this pass is also embedded inline at the tail of a
	827	+ // deploy script running under `set -euo pipefail`, where a bare assignment
	828	+ // from a failing pipeline would abort a deploy whose release is already live.
	829	+ 'TS_CLOUD_PCT=$(df -P / 2>/dev/null | awk \'NR==2 { gsub(/%/, "", $5); print $5 }\') || true',
	830	+ 'case "${TS_CLOUD_PCT:-}" in \'\'|*[!0-9]*) TS_CLOUD_PCT=0 ;; esac',
	831	+ 'TS_CLOUD_DIV=1',
	832	+ `if [ "$TS_CLOUD_PCT" -ge ${escalateAt} ]; then`,
	833	+ ` TS_CLOUD_DIV=${factor}`,
	834	+ ` echo "[ts-cloud] host cleanup: root at \${TS_CLOUD_PCT}% (>= ${escalateAt}%) — shortening retention windows ${factor}x"`,
	835	+ 'fi',
	836	+ ...always.flatMap(renderCleanupRule),
	837	+ // Caches that cost network to rebuild, and journals. A host with room keeps
	838	+ // them; a filling one does not.
	839	+ `if [ "$TS_CLOUD_PCT" -ge ${relaxedBelow} ]; then`,
	840	+ ...pressured.flatMap(renderCleanupRule),
	841	+ ` journalctl --vacuum-time=${options.journalMaxAge ?? '14d'} --vacuum-size=${options.journalMaxSize ?? '512M'} >/dev/null 2>&1 || true`,
	842	+ // Only unused, aged images qualify. Daemon-less hosts skip this.
	843	+ ` if command -v docker >/dev/null 2>&1; then docker image prune --all --force --filter "until=${options.imageMaxAge ?? '168h'}" >/dev/null 2>&1 || true; fi`,
	844	+ ` if command -v podman >/dev/null 2>&1; then podman image prune --all --force --filter "until=${options.imageMaxAge ?? '168h'}" >/dev/null 2>&1 || true; fi`,
	845	+ ' if command -v apt-get >/dev/null 2>&1; then apt-get clean >/dev/null 2>&1 || true; fi',
	846	+ 'else',
	847	+ ` echo "[ts-cloud] host cleanup: root at \${TS_CLOUD_PCT}% (< ${relaxedBelow}%) — keeping rebuildable caches"`,
	848	+ 'fi',
776	849	'echo "[ts-cloud] host cleanup (disk after): $(df -h / | tail -1)"',
777	850	]
778	851	}
	852	+
	853	+/**
	854	+ * Install the host cleanup pass and run it once.
	855	+ *
	856	+ * The pass is installed as a script plus a systemd timer rather than only being
	857	+ * executed inline, because cleanup that runs solely as a side effect of
	858	+ * deploying stops entirely on a box that stops deploying — and a deploy that
	859	+ * fails, the run most likely to have stranded something, skips it too. The
	860	+ * timer makes retention a property of the host; the inline run keeps a deploy's
	861	+ * own leftovers from waiting for it.
	862	+ */
	863	+export function buildHostCleanupScript(options: HostCleanupConfig = {}): string[] {
	864	+ const program = buildCleanupProgram(options)
	865	+ if (options.timer === false)
	866	+ return program
	867	+
	868	+ return [
	869	+ // Every step here is guarded. This runs at the very end of a deploy, after
	870	+ // the release is already live, so a host that cannot take the timer — no
	871	+ // systemd, read-only /etc — must degrade to the inline pass, never fail a
	872	+ // deploy that otherwise succeeded.
	873	+ 'if command -v systemctl >/dev/null 2>&1; then',
	874	+ `cat > ${CLEANUP_SCRIPT_PATH} <<'TS_CLOUD_CLEANUP_EOF'`,
	875	+ '#!/bin/sh',
	876	+ // Deliberately not `set -e`: every rule is individually guarded, and a host
	877	+ // missing one of these directories must not skip the rest of the pass.
	878	+ 'set -u',
	879	+ ...program,
	880	+ 'TS_CLOUD_CLEANUP_EOF',
	881	+ `chmod +x ${CLEANUP_SCRIPT_PATH}`,
	882	+ `cat > /etc/systemd/system/${CLEANUP_UNIT}.service <<'TS_CLOUD_UNIT_EOF'`,
	883	+ '[Unit]',
	884	+ 'Description=Reclaim stale deploy artifacts and caches (managed by ts-cloud)',
	885	+ '',
	886	+ '[Service]',
	887	+ 'Type=oneshot',
	888	+ `ExecStart=${CLEANUP_SCRIPT_PATH}`,
	889	+ 'TS_CLOUD_UNIT_EOF',
	890	+ `cat > /etc/systemd/system/${CLEANUP_UNIT}.timer <<'TS_CLOUD_UNIT_EOF'`,
	891	+ '[Unit]',
	892	+ `Description=Run ts-cloud host cleanup (${options.onCalendar ?? 'daily'})`,
	893	+ '',
	894	+ '[Timer]',
	895	+ `OnCalendar=${options.onCalendar ?? 'daily'}`,
	896	+ // A box that was off at the scheduled time still cleans when it comes back.
	897	+ 'Persistent=true',
	898	+ 'RandomizedDelaySec=30m',
	899	+ '',
	900	+ '[Install]',
	901	+ 'WantedBy=timers.target',
	902	+ 'TS_CLOUD_UNIT_EOF',
	903	+ 'systemctl daemon-reload >/dev/null 2>&1 || true',
	904	+ `systemctl enable --now ${CLEANUP_UNIT}.timer >/dev/null 2>&1 || true`,
	905	+ 'fi',
	906	+ // The timer's own first run may be up to a day out; this deploy's leftovers
	907	+ // should not wait for it. Falls back to the inline pass wherever the
	908	+ // install above was skipped or failed.
	909	+ `if [ -x ${CLEANUP_SCRIPT_PATH} ]; then ${CLEANUP_SCRIPT_PATH} || true; else`,
	910	+ ...program,
	911	+ 'fi',
	912	+ ]
	913	+}

packages/ts-cloud/test/drivers/deploy-script.test.tsmodified+94-0

Changes to packages/ts-cloud/test/drivers/deploy-script.test.ts

		⋯454 linesShow the 454 lines above this hunk@@ -455,6 +455,100 @@ describe('buildHostCleanupScript', () => {
455	455	expect(joined).not.toContain('/var/www')
456	456	expect(joined).not.toContain('releases/')
457	457	})
	458	+
	459	+ /**
	460	+ * The artifact cache used to be pruned inline in the Hetzner driver's upload
	461	+ * path, which put retention on the cache-MISS branch: the box reusing the
	462	+ * cache most pruned it least. One production host reached 23 GB there.
	463	+ */
	464	+ it('prunes the artifact cache, including the .tmp files a failed upload strands', () => {
	465	+ const joined = buildHostCleanupScript().join('\n')
	466	+ expect(joined).toContain('find /var/ts-cloud/artifacts -xdev -maxdepth 1 -type f -name "*.tar.gz"')
	467	+ expect(joined).toContain('find /var/ts-cloud/artifacts -xdev -maxdepth 1 -type f -name ".*.tmp"')
	468	+ })
	469	+
	470	+ it('installs a timer so a host that stops deploying keeps cleaning itself', () => {
	471	+ const joined = buildHostCleanupScript().join('\n')
	472	+ expect(joined).toContain('/etc/systemd/system/ts-cloud-cleanup.timer')
	473	+ expect(joined).toContain('OnCalendar=daily')
	474	+ expect(joined).toContain('Persistent=true')
	475	+ expect(joined).toContain('systemctl enable --now ts-cloud-cleanup.timer')
	476	+ // The timer's first run can be a day out; this deploy's own leftovers
	477	+ // should not wait for it, and it must never fail a live release.
	478	+ expect(joined).toContain('/usr/local/sbin/ts-cloud-cleanup || true')
	479	+ })
	480	+
	481	+ it('emits the bare pass with timer: false, for callers that schedule it themselves', () => {
	482	+ const script = buildHostCleanupScript({ timer: false })
	483	+ expect(script.join('\n')).not.toContain('systemctl')
	484	+ // A bare pass, safe to embed: no shebang and no shell options of its own.
	485	+ expect(script[0]).toContain('host cleanup (disk before)')
	486	+ expect(script.join('\n')).not.toContain('#!/bin/sh')
	487	+ })
	488	+
	489	+ it('honours configured retention windows', () => {
	490	+ const joined = buildHostCleanupScript({
	491	+ artifactMaxAgeMinutes: 120,
	492	+ stagingMaxAgeMinutes: 15,
	493	+ bunCacheMaxAgeMinutes: 1440,
	494	+ journalMaxAge: '3d',
	495	+ journalMaxSize: '128M',
	496	+ imageMaxAge: '24h',
	497	+ onCalendar: 'hourly',
	498	+ }).join('\n')
	499	+ expect(joined).toContain('-name "*.tar.gz" -mmin +$((120 / TS_CLOUD_DIV))')
	500	+ expect(joined).toContain('find /var/ts-cloud/staging -xdev -maxdepth 1 -type f -mmin +15')
	501	+ expect(joined).toContain('journalctl --vacuum-time=3d --vacuum-size=128M')
	502	+ expect(joined).toContain('until=24h')
	503	+ expect(joined).toContain('OnCalendar=hourly')
	504	+ })
	505	+
	506	+ /**
	507	+ * Windows that exist for correctness rather than retention must not move with
	508	+ * disk pressure: an in-flight upload belonging to a concurrent deploy has to
	509	+ * survive its hour however full the box is.
	510	+ */
	511	+ it('shortens retention windows under pressure but leaves in-flight bounds fixed', () => {
	512	+ const joined = buildHostCleanupScript({ escalateAtPercent: 90, escalationFactor: 6 }).join('\n')
	513	+ expect(joined).toContain('if [ "$TS_CLOUD_PCT" -ge 90 ]; then')
	514	+ expect(joined).toContain('TS_CLOUD_DIV=6')
	515	+ expect(joined).toContain('find /var/ts-cloud/staging -xdev -maxdepth 1 -type f -mmin +60 -delete')
	516	+ expect(joined).toContain('-name ".*.tmp" -mmin +60 -delete')
	517	+ })
	518	+
	519	+ it('leaves rebuildable caches alone while the host has room', () => {
	520	+ const joined = buildHostCleanupScript({ relaxedBelowPercent: 40 }).join('\n')
	521	+ expect(joined).toContain('if [ "$TS_CLOUD_PCT" -ge 40 ]; then')
	522	+ const gate = joined.indexOf('if [ "$TS_CLOUD_PCT" -ge 40 ]; then')
	523	+ // The bun cache and image prunes cost network to rebuild, so they sit
	524	+ // behind the gate; the artifact rules do not and must run unconditionally.
	525	+ expect(joined.indexOf('/root/.bun/install/cache')).toBeGreaterThan(gate)
	526	+ expect(joined.indexOf('/var/ts-cloud/artifacts')).toBeLessThan(gate)
	527	+ })
	528	+
	529	+ /**
	530	+ * The pass is appended to a deploy script that has `set -euo pipefail` in
	531	+ * effect, and it runs after the release is already live. A bare assignment
	532	+ * from a failing pipeline (`df` on an odd host) would abort the deploy there.
	533	+ */
	534	+ it('cannot fail a deploy whose release is already live', () => {
	535	+ const script = buildHostCleanupScript()
	536	+ const usage = script.find((line) => line.startsWith('TS_CLOUD_PCT='))
	537	+ expect(usage).toBeDefined()
	538	+ expect(usage).toEndWith('|| true')
	539	+ for (const line of script) {
	540	+ if (line.startsWith('find ') || line.trimStart().startsWith('find '))
	541	+ expect(line).toEndWith('|| true')
	542	+ }
	543	+ expect(script.join('\n')).toContain('systemctl daemon-reload >/dev/null 2>&1 || true')
	544	+ })
	545	+
	546	+ it('accepts retention rules from components that own their own paths', () => {
	547	+ const joined = buildHostCleanupScript({
	548	+ rules: [{ path: '/root/rpx-backups', pattern: 'cert-backup-*', maxAgeMinutes: 4320, directories: true }],
	549	+ }).join('\n')
	550	+ expect(joined).toContain('find /root/rpx-backups -xdev -maxdepth 1 -depth -type d -name "cert-backup-*" -mmin +$((4320 / TS_CLOUD_DIV)) -delete')
	551	+ })
458	552	})
459	553
460	554	describe('releaseTarballTmpPath', () => {