also looking at this
feat(fleet): refuse to destroy a server still holding a moved site's rollback
#183
4 files
+284
-2
| @@ -0,0 +1,110 @@ | ||
| 1 | /** | |
| 2 | * Detect sites whose files are still on a box that no longer serves them. | |
| 3 | * | |
| 4 | * `site:move` deliberately never deletes anything on the source: it stops the | |
| 5 | * units, drops the gateway route, and leaves the whole tree in place. That | |
| 6 | * leftover tree IS the rollback — the one thing that makes a bad cutover | |
| 7 | * recoverable, and the reason the operation can promise reversibility "until the | |
| 8 | * source server is destroyed". | |
| 9 | * | |
| 10 | * Which makes destroying that server the moment the promise expires, and nothing | |
| 11 | * about the box says so. It has no running units for the site, so it looks idle; | |
| 12 | * the config has moved on; the operator is tidying up. Terminating it is exactly | |
| 13 | * the right thing to do once the move is verified, and exactly the wrong thing | |
| 14 | * to do before — and those two look identical from outside. | |
| 15 | * | |
| 16 | * So: before a teardown, ask the box whether it is holding anyone's rollback. | |
| 17 | * | |
| 18 | * @see https://github.com/stacksjs/ts-cloud/issues/167 | |
| 19 | */ | |
| 20 | ||
| 21 | /** Escape a value for safe use inside a POSIX ERE. */ | |
| 22 | function reEscape(value: string): string { | |
| 23 | return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') | |
| 24 | } | |
| 25 | ||
| 26 | /** Single-quote a value for safe embedding in the generated shell. */ | |
| 27 | function sh(value: string): string { | |
| 28 | return `'${value.split('\'').join('\'\\\'\'')}'` | |
| 29 | } | |
| 30 | ||
| 31 | /** | |
| 32 | * Report every one of this project's site trees on the box and whether anything | |
| 33 | * is still running for it. | |
| 34 | * | |
| 35 | * Read-only by construction — it lists and queries, and that is all. A scan that | |
| 36 | * could change the box would be a poor thing to run immediately before deciding | |
| 37 | * whether to keep it. | |
| 38 | * | |
| 39 | * A directory is only counted when it holds `releases/`, so a stray folder under | |
| 40 | * `/var/www` is not mistaken for a deployed site and does not block a teardown | |
| 41 | * the operator genuinely wants. | |
| 42 | */ | |
| 43 | export function buildDrainedSiteScanScript(slug: string, wwwRoot = '/var/www'): string[] { | |
| 44 | const prefix = `${wwwRoot}/${slug}-` | |
| 45 | return [ | |
| 46 | 'set -u', | |
| 47 | `for TS_CLOUD_DIR in ${prefix}*; do`, | |
| 48 | ' [ -d "$TS_CLOUD_DIR" ] || continue', | |
| 49 | // Only a real release tree counts as a deployed site. | |
| 50 | ' [ -d "$TS_CLOUD_DIR/releases" ] || continue', | |
| 51 | ` TS_CLOUD_SITE="\${TS_CLOUD_DIR#${prefix}}"`, | |
| 52 | ' TS_CLOUD_ACTIVE=no', | |
| 53 | ` for TS_CLOUD_UNIT in $(ls /etc/systemd/system/ 2>/dev/null | grep -E "^${reEscape(slug)}-$TS_CLOUD_SITE[-@.]" || true); do`, | |
| 54 | ' systemctl is-active "$TS_CLOUD_UNIT" >/dev/null 2>&1 && TS_CLOUD_ACTIVE=yes', | |
| 55 | ' done', | |
| 56 | // Size is reported so the refusal can say how much is at stake. | |
| 57 | ' TS_CLOUD_SIZE="$(du -sh "$TS_CLOUD_DIR" 2>/dev/null | cut -f1)"', | |
| 58 | ' echo "site:$TS_CLOUD_SITE:$TS_CLOUD_ACTIVE:${TS_CLOUD_SIZE:-?}"', | |
| 59 | 'done', | |
| 60 | 'exit 0', | |
| 61 | ] | |
| 62 | } | |
| 63 | ||
| 64 | export interface DrainedSite { | |
| 65 | name: string | |
| 66 | /** Human-readable size of the tree left behind, e.g. `1.2G`. */ | |
| 67 | size: string | |
| 68 | } | |
| 69 | ||
| 70 | /** | |
| 71 | * Sites the box still holds but no longer runs. | |
| 72 | * | |
| 73 | * A site with something active is NOT drained — it is simply a site on a box | |
| 74 | * being torn down, which is ordinary and already covered by the teardown's own | |
| 75 | * confirmation. The interesting case is files with nothing running: either a | |
| 76 | * completed `site:move` whose rollback this is, or a site that has been stopped | |
| 77 | * and forgotten. Both are worth a sentence before the disk goes away. | |
| 78 | */ | |
| 79 | export function parseDrainedSites(output: string | undefined): DrainedSite[] { | |
| 80 | if (!output) return [] | |
| 81 | const drained: DrainedSite[] = [] | |
| 82 | for (const line of output.split('\n')) { | |
| 83 | const trimmed = line.trim() | |
| 84 | if (!trimmed.startsWith('site:')) continue | |
| 85 | const [, name, active, size] = trimmed.split(':') | |
| 86 | if (!name || active !== 'no') continue | |
| 87 | drained.push({ name, size: size || '?' }) | |
| 88 | } | |
| 89 | return drained | |
| 90 | } | |
| 91 | ||
| 92 | /** | |
| 93 | * What the operator sees instead of a teardown. | |
| 94 | * | |
| 95 | * Names the sites, says what the files are FOR, and gives the two ways forward. | |
| 96 | * The flag is deliberately its own thing rather than `--force`: `--force` exists | |
| 97 | * so a teardown can run unattended, and a CI job that skips a prompt must not | |
| 98 | * also silently discard the only copy of a rollback. | |
| 99 | */ | |
| 100 | export function formatDrainedSiteRefusal(sites: readonly DrainedSite[], slug: string, flag: string): string { | |
| 101 | const list = sites.map(site => ` ${slug}-${site.name} (${site.size})`).join('\n') | |
| 102 | const them = sites.length === 1 ? 'it' : 'them' | |
| 103 | return ( | |
| 104 | `This server still holds ${sites.length} site tree${sites.length === 1 ? '' : 's'} with nothing running for ` | |
| 105 | + `${them}:\n${list}\n` | |
| 106 | + 'A site moved off this box keeps its files here on purpose — they are the rollback, and they are what makes ' | |
| 107 | + 'the move reversible until this server is destroyed. Destroying it now discards that.\n' | |
| 108 | + `Verify the site is serving from its new home first, then re-run with ${flag} to destroy it anyway.` | |
| 109 | ) | |
| 110 | } | |