feat(operations): fleet inventory, attach preflight, and the exports that were missing

packages/ts-cloud/src/operations/inventory.tsadded+430-0

Changes to packages/ts-cloud/src/operations/inventory.ts

		@@ -0,0 +1,430 @@
	1	+/**
	2	+ * What is hosted on which box, across every project sharing it.
	3	+ *
	4	+ * Consolidating servers starts with a question nothing could answer: what is
	5	+ * actually running on each one? A project's own config cannot answer it. It
	6	+ * describes ONE project's sites, and the boxes are multi-tenant - other
	7	+ * projects deploy onto them with `cloud.attachTo`, from their own repositories,
	8	+ * and appear in no file the first project owns. Reading config and calling it
	9	+ * an inventory reports a shared box as if one project were alone on it, which
	10	+ * is precisely the wrong answer to consolidate against.
	11	+ *
	12	+ * So the answer comes from the box. Each project's deploy writes an rpx
	13	+ * registry fragment into `HOST_SITES_DIR`, and those files together are the
	14	+ * only complete record of what the host serves and for whom. `site-ports`
	15	+ * already reads them to allocate ports around co-tenants; this module reads
	16	+ * the same files for the routes rather than the ports, so the two can never
	17	+ * disagree about what is on a box.
	18	+ *
	19	+ * Everything here is pure except {@link probeHostRoutes}, which takes its exec
	20	+ * as an argument - the same shape `site-move` uses, so an inventory can be
	21	+ * tested without a box, a provider, or a credential.
	22	+ *
	23	+ * @see https://github.com/stacksjs/ts-cloud/issues/167
	24	+ * @see https://github.com/stacksjs/stacks/issues/2342
	25	+ */
	26	+
	27	+import type { HostSiteFragment } from '../deploy/site-ports'
	28	+import { buildHostSitePortsScript, HOST_SITES_DIR, parseHostSiteFragments } from '../deploy/site-ports'
	29	+
	30	+/** The ts-cloud labels every provisioned box carries. */
	31	+export const PROJECT_LABEL = 'ts-cloud/project'
	32	+export const ENVIRONMENT_LABEL = 'ts-cloud/environment'
	33	+export const ROLE_LABEL = 'ts-cloud/role'
	34	+
	35	+/**
	36	+ * A server in the inventory, reduced to what deciding a consolidation needs.
	37	+ *
	38	+ * Structural and provider-agnostic on purpose: a Hetzner server satisfies it
	39	+ * after {@link toInventoryServer}, and another driver can satisfy it without
	40	+ * importing anything from here.
	41	+ */
	42	+export interface InventoryServer {
	43	+ id: string
	44	+ name: string
	45	+ status: string
	46	+ ipv4?: string
	47	+ ipv6?: string
	48	+ type?: string
	49	+ location?: string
	50	+ labels: Record<string, string>
	51	+ /** `ts-cloud/project`: the project that provisioned and owns this box. */
	52	+ project?: string
	53	+ /** `ts-cloud/environment`: production, staging, ... */
	54	+ environment?: string
	55	+ /** `ts-cloud/role`: app, services, lb. */
	56	+ role?: string
	57	+}
	58	+
	59	+/** One route a box serves, and the project whose fragment declared it. */
	60	+export interface HostedRoute {
	61	+ slug: string
	62	+ host: string
	63	+ path: string
	64	+ /** Where the route goes: an upstream, a served directory, or a redirect target. */
	65	+ target: string
	66	+ kind: 'app' | 'static' | 'redirect' | 'unknown'
	67	+}
	68	+
	69	+/** What a box answered when asked what it serves. */
	70	+export interface HostProbe {
	71	+ server: string
	72	+ ip?: string
	73	+ routes: HostedRoute[]
	74	+ /** Why the probe produced nothing, when it produced nothing. */
	75	+ unavailable?: string
	76	+}
	77	+
	78	+function text(value: unknown): string | undefined {
	79	+ return typeof value === 'string' && value.trim() ? value.trim() : undefined
	80	+}
	81	+
	82	+/**
	83	+ * Shape one provider server record into {@link InventoryServer}.
	84	+ *
	85	+ * Written against the Hetzner payload, but touches only fields any provider
	86	+ * listing carries, so a driver with a different shape maps onto the same type
	87	+ * rather than forcing a second one.
	88	+ *
	89	+ * An unlabelled box is kept rather than dropped. A server provisioned by hand,
	90	+ * or by a ts-cloud old enough not to have labelled it, is exactly the kind a
	91	+ * consolidation needs to see; requiring the labels would hide it.
	92	+ */
	93	+export function toInventoryServer(raw: any): InventoryServer {
	94	+ const labels: Record<string, string> = {}
	95	+ for (const [key, value] of Object.entries(raw?.labels ?? {})) {
	96	+ if (typeof value === 'string') labels[key] = value
	97	+ }
	98	+
	99	+ return {
	100	+ id: String(raw?.id ?? ''),
	101	+ name: text(raw?.name) ?? '(unnamed)',
	102	+ status: text(raw?.status) ?? 'unknown',
	103	+ ipv4: text(raw?.public_net?.ipv4?.ip) ?? text(raw?.ipv4),
	104	+ ipv6: text(raw?.public_net?.ipv6?.ip) ?? text(raw?.ipv6),
	105	+ type: text(raw?.server_type?.name) ?? text(raw?.type),
	106	+ location: text(raw?.datacenter?.location?.name) ?? text(raw?.datacenter?.name) ?? text(raw?.location),
	107	+ labels,
	108	+ project: text(labels[PROJECT_LABEL]),
	109	+ environment: text(labels[ENVIRONMENT_LABEL]),
	110	+ role: text(labels[ROLE_LABEL]),
	111	+ }
	112	+}
	113	+
	114	+/** The slug a fragment belongs to, matching the writer's `'app'` default. */
	115	+function fragmentSlug(fragment: HostSiteFragment): string {
	116	+ return fragment.slug?.trim() || 'app'
	117	+}
	118	+
	119	+/**
	120	+ * Flatten registry fragments into one route list, sorted so two runs against an
	121	+ * unchanged box produce an identical listing.
	122	+ */
	123	+export function routesFromFragments(fragments: readonly HostSiteFragment[]): HostedRoute[] {
	124	+ const routes: HostedRoute[] = []
	125	+
	126	+ for (const fragment of fragments) {
	127	+ const slug = fragmentSlug(fragment)
	128	+
	129	+ for (const proxy of fragment.proxies ?? []) {
	130	+ const host = text(proxy?.to)
	131	+ if (!host) continue
	132	+
	133	+ routes.push({ slug, host, path: text(proxy?.path) ?? '/', ...describeRouteTarget(proxy) })
	134	+ }
	135	+ }
	136	+
	137	+ return routes.sort((a, b) =>
	138	+ a.slug.localeCompare(b.slug) || a.host.localeCompare(b.host) || a.path.localeCompare(b.path))
	139	+}
	140	+
	141	+function describeRouteTarget(proxy: NonNullable<HostSiteFragment['proxies']>[number]): Pick<HostedRoute, 'target' | 'kind'> {
	142	+ const from = proxy?.from
	143	+ if (typeof from === 'string' && from.trim()) return { target: from.trim(), kind: 'app' }
	144	+ if (Array.isArray(from) && from.length > 0) {
	145	+ return { target: from.filter(upstream => typeof upstream === 'string').join(', '), kind: 'app' }
	146	+ }
	147	+
	148	+ const served = proxy?.static
	149	+ if (typeof served === 'string' && served.trim()) return { target: served.trim(), kind: 'static' }
	150	+ if (served && typeof served === 'object' && text(served.dir)) return { target: text(served.dir)!, kind: 'static' }
	151	+
	152	+ const redirect = typeof proxy?.redirect === 'string' ? text(proxy.redirect) : text(proxy?.redirect?.to)
	153	+ if (redirect) return { target: redirect, kind: 'redirect' }
	154	+
	155	+ return { target: '(no upstream)', kind: 'unknown' }
	156	+}
	157	+
	158	+/** Group routes by the project that owns them, biggest tenant first. */
	159	+export function tenantsOf(routes: readonly HostedRoute[]): Array<{ slug: string, routes: HostedRoute[] }> {
	160	+ const bySlug = new Map<string, HostedRoute[]>()
	161	+ for (const route of routes) {
	162	+ const bucket = bySlug.get(route.slug)
	163	+ if (bucket) bucket.push(route)
	164	+ else bySlug.set(route.slug, [route])
	165	+ }
	166	+
	167	+ return [...bySlug.entries()]
	168	+ .map(([slug, grouped]) => ({ slug, routes: grouped }))
	169	+ .sort((a, b) => b.routes.length - a.routes.length || a.slug.localeCompare(b.slug))
	170	+}
	171	+
	172	+/** Run a script on a host, resolving with its exit code and output. */
	173	+export type InventoryExec = (host: string, command: string) => Promise<{ code: number, stdout: string, stderr: string }>
	174	+
	175	+/**
	176	+ * Ask one box what it serves.
	177	+ *
	178	+ * Never throws. A box that is off, unreachable, or refuses the key comes back
	179	+ * with an `unavailable` reason instead, so one bad server does not cost the
	180	+ * listing of every other one - and so a caller can tell "serves nothing" apart
	181	+ * from "was never asked", which are the two answers most worth not confusing.
	182	+ */
	183	+export async function probeHostRoutes(
	184	+ server: InventoryServer,
	185	+ exec: InventoryExec,
	186	+ sitesDir: string = HOST_SITES_DIR,
	187	+): Promise<HostProbe> {
	188	+ if (!server.ipv4) {
	189	+ return { server: server.name, routes: [], unavailable: 'no public IPv4 address to reach it on' }
	190	+ }
	191	+
	192	+ if (server.status !== 'running') {
	193	+ return { server: server.name, ip: server.ipv4, routes: [], unavailable: `server is ${server.status}` }
	194	+ }
	195	+
	196	+ try {
	197	+ const result = await exec(server.ipv4, buildHostSitePortsScript(sitesDir))
	198	+ if (result.code !== 0) {
	199	+ const reason = result.stderr.trim().split('\n')[0] || `remote command exited ${result.code}`
	200	+ return { server: server.name, ip: server.ipv4, routes: [], unavailable: reason }
	201	+ }
	202	+
	203	+ return { server: server.name, ip: server.ipv4, routes: routesFromFragments(parseHostSiteFragments(result.stdout)) }
	204	+ }
	205	+ catch (error) {
	206	+ const reason = error instanceof Error ? error.message.split('\n')[0] : String(error)
	207	+ return { server: server.name, ip: server.ipv4, routes: [], unavailable: reason }
	208	+ }
	209	+}
	210	+
	211	+/** One site a project declares, in the terms the box reports. */
	212	+export interface DeclaredSite {
	213	+ name: string
	214	+ kind?: string
	215	+ domain?: string
	216	+ path: string
	217	+ port?: number
	218	+ /** `siteInstallBase(slug, name)`, when the caller resolved it. */
	219	+ installBase?: string
	220	+ /**
	221	+ * No `domain`, so the gateway never routes it and it cannot appear in a
	222	+ * registry fragment. That is a deliberate configuration - a loopback-only
	223	+ * service reached through another site's proxy - not a missing deploy.
	224	+ */
	225	+ loopbackOnly: boolean
	226	+}
	227	+
	228	+/** Declared sites lined up against what one box actually serves. */
	229	+export interface Reconciliation {
	230	+ /** Declared and present on the box. */
	231	+ present: DeclaredSite[]
	232	+ /** Declared, routable, and absent from the box's registry. */
	233	+ absent: DeclaredSite[]
	234	+ /** Declared with no domain: no gateway route by design. */
	235	+ loopback: DeclaredSite[]
	236	+ /** Routes on the box owned by some other project. */
	237	+ foreign: HostedRoute[]
	238	+}
	239	+
	240	+function routeKey(host: string, path: string): string {
	241	+ const normalized = path === '/' ? '/' : path.replace(/\/+$/, '')
	242	+ return `${host.toLowerCase()}${normalized || '/'}`
	243	+}
	244	+
	245	+/**
	246	+ * Line a project's declared sites up against what a box serves.
	247	+ *
	248	+ * Matched on host and path rather than on the site key, because the site key is
	249	+ * local to a repository and the box has no idea what it is. Two projects both
	250	+ * calling a site `main` is ordinary; two projects serving the same host and
	251	+ * path is the collision worth seeing.
	252	+ */
	253	+export function reconcile(
	254	+ declared: readonly DeclaredSite[],
	255	+ routes: readonly HostedRoute[],
	256	+ slug: string,
	257	+): Reconciliation {
	258	+ const ours = new Set(routes.filter(route => route.slug === slug).map(route => routeKey(route.host, route.path)))
	259	+
	260	+ const present: DeclaredSite[] = []
	261	+ const absent: DeclaredSite[] = []
	262	+ const loopback: DeclaredSite[] = []
	263	+
	264	+ for (const site of declared) {
	265	+ if (site.loopbackOnly) loopback.push(site)
	266	+ else if (site.domain && ours.has(routeKey(site.domain, site.path))) present.push(site)
	267	+ else absent.push(site)
	268	+ }
	269	+
	270	+ return { present, absent, loopback, foreign: routes.filter(route => route.slug !== slug) }
	271	+}
	272	+
	273	+/**
	274	+ * Sites no probed box accounts for.
	275	+ *
	276	+ * Callers must pass only the probes that ANSWERED. A box that could not be read
	277	+ * proves nothing about what it holds, and counting it would report every site
	278	+ * on it as missing - a true statement about the listing and a false one about
	279	+ * the deployment.
	280	+ */
	281	+export function unaccountedSites(
	282	+ declared: readonly DeclaredSite[],
	283	+ answered: readonly HostProbe[],
	284	+ slug: string,
	285	+): DeclaredSite[] {
	286	+ const seen = new Set<string>()
	287	+ for (const probe of answered) {
	288	+ for (const route of probe.routes) {
	289	+ if (route.slug === slug) seen.add(routeKey(route.host, route.path))
	290	+ }
	291	+ }
	292	+
	293	+ return declared.filter(site => !site.loopbackOnly && site.domain !== undefined && !seen.has(routeKey(site.domain, site.path)))
	294	+}
	295	+
	296	+export interface Inventory {
	297	+ slug: string
	298	+ servers: InventoryServer[]
	299	+ probes: HostProbe[]
	300	+ declared: DeclaredSite[]
	301	+}
	302	+
	303	+/**
	304	+ * The inventory as lines, ready to print.
	305	+ *
	306	+ * Lines rather than printed output, matching {@link import('./plan').formatPlan}:
	307	+ * the caller owns the stream, and the format stays testable. Every count is
	308	+ * stated so a partial answer reads as partial - a box that could not be read
	309	+ * says so on its own line, and the summary separates "not deployed" from "not
	310	+ * visible from here".
	311	+ */
	312	+export function formatInventory(inventory: Inventory): string[] {
	313	+ const { slug, servers, probes, declared } = inventory
	314	+ const lines: string[] = []
	315	+
	316	+ lines.push(servers.length === 0
	317	+ ? 'No servers found.'
	318	+ : `${servers.length} server${servers.length === 1 ? '' : 's'}:`)
	319	+ if (servers.length > 0) lines.push('')
	320	+
	321	+ const probesByServer = new Map(probes.map(probe => [probe.server, probe]))
	322	+
	323	+ for (const server of servers) {
	324	+ const facts = [server.ipv4, server.type, server.location, server.status].filter(Boolean)
	325	+ lines.push(` ${server.name} ${facts.join(' ')}`)
	326	+ lines.push(` ${describeOwnership(server)}`)
	327	+
	328	+ const probe = probesByServer.get(server.name)
	329	+ if (!probe) {
	330	+ lines.push(' not probed, so co-tenants on this box are not listed')
	331	+ }
	332	+ else if (probe.unavailable) {
	333	+ lines.push(` could not read ${HOST_SITES_DIR}: ${probe.unavailable}`)
	334	+ }
	335	+ else {
	336	+ lines.push(...describeTenants(probe, slug))
	337	+ }
	338	+
	339	+ lines.push('')
	340	+ }
	341	+
	342	+ lines.push(...describeDeclared(inventory))
	343	+
	344	+ return lines
	345	+}
	346	+
	347	+function describeOwnership(server: InventoryServer): string {
	348	+ if (!server.project) {
	349	+ return 'no ts-cloud labels: provisioned outside ts-cloud, or by a version that did not label boxes'
	350	+ }
	351	+
	352	+ const detail = [server.environment, server.role && `role ${server.role}`].filter(Boolean).join(', ')
	353	+ return `owned by '${server.project}'${detail ? ` (${detail})` : ''}`
	354	+}
	355	+
	356	+function describeTenants(probe: HostProbe, slug: string): string[] {
	357	+ const tenants = tenantsOf(probe.routes)
	358	+ if (tenants.length === 0) return [` serves nothing: ${HOST_SITES_DIR} is empty or absent`]
	359	+
	360	+ const lines = [
	361	+ ` serves ${probe.routes.length} route${probe.routes.length === 1 ? '' : 's'} `
	362	+ + `for ${tenants.length} project${tenants.length === 1 ? '' : 's'}:`,
	363	+ ]
	364	+
	365	+ for (const tenant of tenants) {
	366	+ lines.push(` ${tenant.slug}${tenant.slug === slug ? ' (this project)' : ''}`)
	367	+ for (const route of tenant.routes) {
	368	+ lines.push(` ${route.host}${route.path === '/' ? '/' : route.path} -> ${describeTarget(route)}`)
	369	+ }
	370	+ }
	371	+
	372	+ return lines
	373	+}
	374	+
	375	+function describeTarget(route: HostedRoute): string {
	376	+ if (route.kind === 'redirect') return `redirect to ${route.target}`
	377	+ if (route.kind === 'static') return `static ${route.target}`
	378	+ return route.target
	379	+}
	380	+
	381	+function describeDeclared(inventory: Inventory): string[] {
	382	+ const { slug, declared, probes, servers } = inventory
	383	+ if (declared.length === 0) return [`This project ('${slug}') declares no sites.`]
	384	+
	385	+ const lines = [
	386	+ `This project ('${slug}') declares ${declared.length} site${declared.length === 1 ? '' : 's'}: `
	387	+ + declared.map(site => site.name).join(', '),
	388	+ ]
	389	+
	390	+ const loopback = declared.filter(site => site.loopbackOnly)
	391	+ if (loopback.length > 0) {
	392	+ lines.push(
	393	+ ` ${loopback.length} with no domain, so the gateway never routes ${loopback.length === 1 ? 'it' : 'them'} `
	394	+ + `(reached through another site's proxy): ${loopback.map(site => site.name).join(', ')}`,
	395	+ )
	396	+ }
	397	+
	398	+ // Reconciliation needs at least one box that actually answered. Without one,
	399	+ // every routable site is "not found", which is the shape of wrong answer this
	400	+ // module exists to stop producing.
	401	+ const answered = probes.filter(probe => !probe.unavailable)
	402	+ if (answered.length === 0) {
	403	+ lines.push(' Nothing to reconcile them against: no box reported what it serves.')
	404	+ return lines
	405	+ }
	406	+
	407	+ const unaccounted = unaccountedSites(declared, answered, slug)
	408	+ lines.push(` ${declared.length - loopback.length - unaccounted.length} routed by a box above`)
	409	+
	410	+ if (unaccounted.length > 0) {
	411	+ // Two very different causes, and this listing cannot tell them apart, so it
	412	+ // must not pick one: an undeployed site and a site on a box that was not
	413	+ // read look identical from here.
	414	+ lines.push(` ${unaccounted.length} not routed by any box above: ${unaccounted.map(site => site.name).join(', ')}`)
	415	+
	416	+ const unread = probes.length - answered.length
	417	+ const unprobed = servers.length - probes.length
	418	+ if (unread > 0) {
	419	+ lines.push(` Either they were never deployed, or they are on one of the ${unread} server${unread === 1 ? '' : 's'} that could not be read.`)
	420	+ }
	421	+ else if (unprobed > 0) {
	422	+ lines.push(` Either they were never deployed, or they are on one of the ${unprobed} server${unprobed === 1 ? '' : 's'} this run did not probe.`)
	423	+ }
	424	+ else {
	425	+ lines.push(' Either they were never deployed, or they are on a server this listing did not cover.')
	426	+ }
	427	+ }
	428	+
	429	+ return lines
	430	+}