| @@ -10,7 +10,7 @@ |
| 10 | 10 | * release (no window where the code is half-replaced). Old releases are kept for |
| 11 | 11 | * instant rollback. See {@link import('./releases')}. |
| 12 | 12 | */ |
| 13 | | -import type { SharedPathEntry } from '@ts-cloud/core' |
| 13 | +import type { HostCleanupConfig, HostCleanupRuleConfig, SharedPathEntry } from '@ts-cloud/core' |
| 14 | 14 | import { formatEnvFile } from './env-file' |
| 15 | 15 | import { buildActivateRelease, buildDeployLock, buildEnsureReleaseLayout, buildLinkSharedPaths, buildPromoteStagedRelease, buildPruneReleases, buildResetReleaseDir, buildStrandedReleaseTrap, dedupeSharedPaths, DEFAULT_KEEP_RELEASES, releasePaths } from './releases' |
| 16 | 16 | import { sqliteSharedPaths } from './sqlite-shared-path' |
| @@ -751,28 +751,163 @@ export function buildLocalArtifactFetch(localPath: string, destPath: string): st |
| 751 | 751 | ] |
| 752 | 752 | } |
| 753 | 753 | |
| 754 | + |
| 755 | +const DEFAULT_ESCALATE_AT_PERCENT = 85 |
| 756 | + |
| 757 | +const DEFAULT_RELAXED_BELOW_PERCENT = 50 |
| 758 | + |
| 759 | +const DEFAULT_ESCALATION_FACTOR = 4 |
| 760 | + |
| 761 | +const CLEANUP_SCRIPT_PATH = '/usr/local/sbin/ts-cloud-cleanup' |
| 762 | +const CLEANUP_UNIT = 'ts-cloud-cleanup' |
| 763 | + |
| 764 | +function resolveCleanupRules(options: HostCleanupConfig): HostCleanupRuleConfig[] { |
| 765 | + const staging = Math.max(1, options.stagingMaxAgeMinutes ?? 60) |
| 766 | + return [ |
| 767 | + { path: '/var/ts-cloud/staging', maxAgeMinutes: staging, fixedWindow: true }, |
| 768 | + { path: '/tmp', pattern: '*-release.tar.gz', maxAgeMinutes: staging, fixedWindow: true }, |
| 769 | + |
| 770 | + |
| 771 | + |
| 772 | + |
| 773 | + |
| 774 | + { path: '/var/ts-cloud/artifacts', pattern: '*.tar.gz', maxAgeMinutes: Math.max(1, options.artifactMaxAgeMinutes ?? 2880) }, |
| 775 | + |
| 776 | + { path: '/var/ts-cloud/artifacts', pattern: '.*.tmp', maxAgeMinutes: staging, fixedWindow: true }, |
| 777 | + |
| 778 | + |
| 779 | + { |
| 780 | + path: '/root/.bun/install/cache', |
| 781 | + maxAgeMinutes: Math.max(1, options.bunCacheMaxAgeMinutes ?? 10080), |
| 782 | + recursive: true, |
| 783 | + pruneEmptyDirs: true, |
| 784 | + onlyUnderPressure: true, |
| 785 | + }, |
| 786 | + ...(options.rules ?? []), |
| 787 | + ] |
| 788 | +} |
| 789 | + |
| 790 | +function renderCleanupRule(rule: HostCleanupRuleConfig): string[] { |
| 791 | + const depth = rule.recursive ? '' : ' -maxdepth 1' |
| 792 | + const name = rule.pattern ? ` -name ${JSON.stringify(rule.pattern)}` : '' |
| 793 | + const type = rule.directories ? ' -type d' : ' -type f' |
| 794 | + |
| 795 | + |
| 796 | + const age = rule.fixedWindow ? `${rule.maxAgeMinutes}` : `$((${rule.maxAgeMinutes} / TS_CLOUD_DIV))` |
| 797 | + const find = `find ${rule.path} -xdev${depth}${rule.directories ? ' -depth' : ''}${type}${name} -mmin +${age} -delete 2>/dev/null || true` |
| 798 | + const lines = [find] |
| 799 | + if (rule.pruneEmptyDirs) |
| 800 | + lines.push(`find ${rule.path} -xdev -depth -type d -empty -delete 2>/dev/null || true`) |
| 801 | + return rule.onlyUnderPressure ? lines.map((line) => ` ${line}`) : lines |
| 802 | +} |
| 803 | + |
| 754 | 804 | |
| 755 | | - |
| 756 | | - |
| 757 | | - |
| 758 | | - |
| 805 | + |
| 806 | + |
| 807 | + |
| 808 | + |
| 809 | + |
| 810 | + |
| 759 | 811 | |
| 760 | | -export function buildHostCleanupScript(): string[] { |
| 812 | +function buildCleanupProgram(options: HostCleanupConfig): string[] { |
| 813 | + const relaxedBelow = options.relaxedBelowPercent ?? DEFAULT_RELAXED_BELOW_PERCENT |
| 814 | + const escalateAt = options.escalateAtPercent ?? DEFAULT_ESCALATE_AT_PERCENT |
| 815 | + const factor = Math.max(1, options.escalationFactor ?? DEFAULT_ESCALATION_FACTOR) |
| 816 | + const rules = resolveCleanupRules(options) |
| 817 | + const [pressured, always] = [ |
| 818 | + rules.filter((rule) => rule.onlyUnderPressure), |
| 819 | + rules.filter((rule) => !rule.onlyUnderPressure), |
| 820 | + ] |
| 821 | + |
| 761 | 822 | return [ |
| 762 | 823 | 'echo "[ts-cloud] host cleanup (disk before): $(df -h / | tail -1)"', |
| 763 | | - |
| 764 | | - |
| 765 | | - 'find /var/ts-cloud/staging -xdev -maxdepth 1 -type f -mmin +60 -delete 2>/dev/null || true', |
| 766 | | - 'find /tmp -xdev -maxdepth 1 -type f -name "*-release.tar.gz" -mmin +60 -delete 2>/dev/null || true', |
| 767 | | - |
| 768 | | - |
| 769 | | - 'find /root/.bun/install/cache -xdev -type f -mtime +7 -delete 2>/dev/null || true', |
| 770 | | - 'find /root/.bun/install/cache -xdev -depth -type d -empty -delete 2>/dev/null || true', |
| 771 | | - 'journalctl --vacuum-time=14d --vacuum-size=512M >/dev/null 2>&1 || true', |
| 772 | | - |
| 773 | | - 'if command -v docker >/dev/null 2>&1; then docker image prune --all --force --filter "until=168h" >/dev/null 2>&1 || true; fi', |
| 774 | | - 'if command -v podman >/dev/null 2>&1; then podman image prune --all --force --filter "until=168h" >/dev/null 2>&1 || true; fi', |
| 775 | | - 'if command -v apt-get >/dev/null 2>&1; then apt-get clean >/dev/null 2>&1 || true; fi', |
| 824 | + |
| 825 | + |
| 826 | + |
| 827 | + |
| 828 | + |
| 829 | + 'TS_CLOUD_PCT=$(df -P / 2>/dev/null | awk \'NR==2 { gsub(/%/, "", $5); print $5 }\') || true', |
| 830 | + 'case "${TS_CLOUD_PCT:-}" in \'\'|*[!0-9]*) TS_CLOUD_PCT=0 ;; esac', |
| 831 | + 'TS_CLOUD_DIV=1', |
| 832 | + `if [ "$TS_CLOUD_PCT" -ge ${escalateAt} ]; then`, |
| 833 | + ` TS_CLOUD_DIV=${factor}`, |
| 834 | + ` echo "[ts-cloud] host cleanup: root at \${TS_CLOUD_PCT}% (>= ${escalateAt}%) — shortening retention windows ${factor}x"`, |
| 835 | + 'fi', |
| 836 | + ...always.flatMap(renderCleanupRule), |
| 837 | + |
| 838 | + |
| 839 | + `if [ "$TS_CLOUD_PCT" -ge ${relaxedBelow} ]; then`, |
| 840 | + ...pressured.flatMap(renderCleanupRule), |
| 841 | + ` journalctl --vacuum-time=${options.journalMaxAge ?? '14d'} --vacuum-size=${options.journalMaxSize ?? '512M'} >/dev/null 2>&1 || true`, |
| 842 | + |
| 843 | + ` if command -v docker >/dev/null 2>&1; then docker image prune --all --force --filter "until=${options.imageMaxAge ?? '168h'}" >/dev/null 2>&1 || true; fi`, |
| 844 | + ` if command -v podman >/dev/null 2>&1; then podman image prune --all --force --filter "until=${options.imageMaxAge ?? '168h'}" >/dev/null 2>&1 || true; fi`, |
| 845 | + ' if command -v apt-get >/dev/null 2>&1; then apt-get clean >/dev/null 2>&1 || true; fi', |
| 846 | + 'else', |
| 847 | + ` echo "[ts-cloud] host cleanup: root at \${TS_CLOUD_PCT}% (< ${relaxedBelow}%) — keeping rebuildable caches"`, |
| 848 | + 'fi', |
| 776 | 849 | 'echo "[ts-cloud] host cleanup (disk after): $(df -h / | tail -1)"', |
| 777 | 850 | ] |
| 778 | 851 | } |
| 852 | + |
| 853 | + |
| 854 | + |
| 855 | + |
| 856 | + |
| 857 | + |
| 858 | + |
| 859 | + |
| 860 | + |
| 861 | + |
| 862 | + |
| 863 | +export function buildHostCleanupScript(options: HostCleanupConfig = {}): string[] { |
| 864 | + const program = buildCleanupProgram(options) |
| 865 | + if (options.timer === false) |
| 866 | + return program |
| 867 | + |
| 868 | + return [ |
| 869 | + |
| 870 | + |
| 871 | + |
| 872 | + |
| 873 | + 'if command -v systemctl >/dev/null 2>&1; then', |
| 874 | + `cat > ${CLEANUP_SCRIPT_PATH} <<'TS_CLOUD_CLEANUP_EOF'`, |
| 875 | + '#!/bin/sh', |
| 876 | + |
| 877 | + |
| 878 | + 'set -u', |
| 879 | + ...program, |
| 880 | + 'TS_CLOUD_CLEANUP_EOF', |
| 881 | + `chmod +x ${CLEANUP_SCRIPT_PATH}`, |
| 882 | + `cat > /etc/systemd/system/${CLEANUP_UNIT}.service <<'TS_CLOUD_UNIT_EOF'`, |
| 883 | + '[Unit]', |
| 884 | + 'Description=Reclaim stale deploy artifacts and caches (managed by ts-cloud)', |
| 885 | + '', |
| 886 | + '[Service]', |
| 887 | + 'Type=oneshot', |
| 888 | + `ExecStart=${CLEANUP_SCRIPT_PATH}`, |
| 889 | + 'TS_CLOUD_UNIT_EOF', |
| 890 | + `cat > /etc/systemd/system/${CLEANUP_UNIT}.timer <<'TS_CLOUD_UNIT_EOF'`, |
| 891 | + '[Unit]', |
| 892 | + `Description=Run ts-cloud host cleanup (${options.onCalendar ?? 'daily'})`, |
| 893 | + '', |
| 894 | + '[Timer]', |
| 895 | + `OnCalendar=${options.onCalendar ?? 'daily'}`, |
| 896 | + |
| 897 | + 'Persistent=true', |
| 898 | + 'RandomizedDelaySec=30m', |
| 899 | + '', |
| 900 | + '[Install]', |
| 901 | + 'WantedBy=timers.target', |
| 902 | + 'TS_CLOUD_UNIT_EOF', |
| 903 | + 'systemctl daemon-reload >/dev/null 2>&1 || true', |
| 904 | + `systemctl enable --now ${CLEANUP_UNIT}.timer >/dev/null 2>&1 || true`, |
| 905 | + 'fi', |
| 906 | + |
| 907 | + |
| 908 | + |
| 909 | + `if [ -x ${CLEANUP_SCRIPT_PATH} ]; then ${CLEANUP_SCRIPT_PATH} || true; else`, |
| 910 | + ...program, |
| 911 | + 'fi', |
| 912 | + ] |
| 913 | +} |