ReviewOS
Reload

also looking at this

stacks/ts-cloud

feat(operations): fleet inventory, attach preflight, and the exports that were missing

#192
Merged glennmichael123 wants to merge feat/fleet-inventory-and-attach-preflight into main
11 files +1313 -5
packages/ts-cloud/src/operations/inventory.test.tsadded+280-0
@@ -0,0 +1,280 @@
1import { describe, expect, it } from 'bun:test'
2import { buildHostSitePortsScript, parseHostSiteFragments } from '../deploy/site-ports'
3import {
4 formatInventory,
5 probeHostRoutes,
6 reconcile,
7 routesFromFragments,
8 tenantsOf,
9 toInventoryServer,
10 unaccountedSites,
11} from './inventory'
12
13/**
14 * The trap this module exists to avoid: a project's own config describes ONE
15 * project, the boxes are shared, and so reading config alone reports a
16 * multi-tenant server as if that project were alone on it. Every test that
17 * matters below is about a co-tenant being visible, or about a partial answer
18 * being labelled partial rather than passed off as a complete one.
19 */
20
21const STACKS_FRAGMENT = {
22 slug: 'stacks',
23 proxies: [
24 { to: 'stacksjs.com', from: '127.0.0.1:3000' },
25 { to: 'stacksjs.com', path: '/docs', static: { dir: '/var/www/stacks-docs' } },
26 { to: 'stacksjs.com', path: '/discord', redirect: { to: 'https://discord.gg/example' } },
27 ],
28}
29
30const RAPPID_FRAGMENT = {
31 slug: 'rappid',
32 proxies: [{ to: 'rappid.hq.training', from: '127.0.0.1:3024' }],
33}
34
35function site(name: string, overrides: Record<string, any> = {}) {
36 return { name, path: '/', loopbackOnly: overrides.domain === undefined, ...overrides }
37}
38
39describe('shaping a provider server', () => {
40 it('resolves the ts-cloud identity a box was labelled with', () => {
41 expect(toInventoryServer({
42 id: 12345,
43 name: 'stacks-production-app',
44 status: 'running',
45 public_net: { ipv4: { ip: '5.161.0.1' } },
46 server_type: { name: 'cpx41' },
47 datacenter: { location: { name: 'fsn1' } },
48 labels: { 'ts-cloud/project': 'stacks', 'ts-cloud/environment': 'production', 'ts-cloud/role': 'app' },
49 })).toMatchObject({
50 id: '12345',
51 name: 'stacks-production-app',
52 ipv4: '5.161.0.1',
53 type: 'cpx41',
54 location: 'fsn1',
55 project: 'stacks',
56 environment: 'production',
57 role: 'app',
58 })
59 })
60
61 it('accepts a flatter record from a driver that is not Hetzner', () => {
62 expect(toInventoryServer({ id: 2, name: 'box', status: 'running', ipv4: '1.2.3.4', type: 'medium', location: 'nbg1', labels: {} }))
63 .toMatchObject({ ipv4: '1.2.3.4', type: 'medium', location: 'nbg1' })
64 })
65
66 it('keeps an unlabelled box rather than dropping it', () => {
67 // A box provisioned by hand, or by a ts-cloud old enough not to label, is
68 // exactly the kind a consolidation needs to see.
69 expect(toInventoryServer({ id: 7, name: 'legacy-box', status: 'running', labels: {} }))
70 .toMatchObject({ name: 'legacy-box', project: undefined })
71 })
72})
73
74describe('reading the box registry', () => {
75 it('reads every project on the box, not just one', () => {
76 expect(routesFromFragments([STACKS_FRAGMENT, RAPPID_FRAGMENT]).map(r => `${r.slug} ${r.host}${r.path}`)).toEqual([
77 'rappid rappid.hq.training/',
78 'stacks stacksjs.com/',
79 'stacks stacksjs.com/discord',
80 'stacks stacksjs.com/docs',
81 ])
82 })
83
84 it('describes each route by where it actually goes', () => {
85 const byPath = Object.fromEntries(routesFromFragments([STACKS_FRAGMENT]).map(r => [r.path, r]))
86
87 expect(byPath['/']).toMatchObject({ kind: 'app', target: '127.0.0.1:3000' })
88 expect(byPath['/docs']).toMatchObject({ kind: 'static', target: '/var/www/stacks-docs' })
89 expect(byPath['/discord']).toMatchObject({ kind: 'redirect', target: 'https://discord.gg/example' })
90 })
91
92 it('reads a load-balanced route as its whole upstream pool', () => {
93 expect(routesFromFragments([{ slug: 'x', proxies: [{ to: 'x.com', from: ['10.0.0.1:3000', '10.0.0.2:3000'] }] }])[0])
94 .toMatchObject({ kind: 'app', target: '10.0.0.1:3000, 10.0.0.2:3000' })
95 })
96
97 it('defaults a fragment with no slug the way the writer does', () => {
98 // An older ts-cloud wrote fragments without `slug`; reading them as an
99 // unnamed tenant would split one project in two.
100 expect(routesFromFragments([{ proxies: [{ to: 'old.example', from: '127.0.0.1:3000' }] }])[0]?.slug).toBe('app')
101 })
102
103 it('groups tenants biggest first so the box owner reads at the top', () => {
104 expect(tenantsOf(routesFromFragments([STACKS_FRAGMENT, RAPPID_FRAGMENT])).map(t => t.slug)).toEqual(['stacks', 'rappid'])
105 })
106
107 it('reads the same files site-ports does, through the same script', () => {
108 // The port allocator and this inventory must never disagree about what is
109 // on a box, which is why neither owns its own copy of the read.
110 const stdout = [STACKS_FRAGMENT, RAPPID_FRAGMENT]
111 .map(fragment => Buffer.from(JSON.stringify(fragment)).toString('base64'))
112 .join('\n')
113
114 expect(routesFromFragments(parseHostSiteFragments(stdout))).toHaveLength(4)
115 expect(buildHostSitePortsScript('/etc/rpx/sites.d')).toContain('/etc/rpx/sites.d')
116 })
117})
118
119describe('probing a box', () => {
120 const server = toInventoryServer({ id: 1, name: 'box', status: 'running', public_net: { ipv4: { ip: '5.5.5.5' } }, labels: {} })
121
122 it('returns the routes a reachable box reports', async () => {
123 const probe = await probeHostRoutes(server, async () => ({
124 code: 0,
125 stdout: `${Buffer.from(JSON.stringify(STACKS_FRAGMENT)).toString('base64')}\n`,
126 stderr: '',
127 }))
128
129 expect(probe.unavailable).toBeUndefined()
130 expect(probe.routes).toHaveLength(3)
131 })
132
133 it('reports an unreachable box instead of failing the whole listing', async () => {
134 const probe = await probeHostRoutes(server, async () => {
135 throw new Error('Permission denied (publickey).\nssh gave up')
136 })
137
138 expect(probe).toMatchObject({ routes: [], unavailable: 'Permission denied (publickey).' })
139 })
140
141 it('does not reach for a box that is powered off', async () => {
142 let attempted = false
143 const probe = await probeHostRoutes({ ...server, status: 'off' }, async () => {
144 attempted = true
145 return { code: 0, stdout: '', stderr: '' }
146 })
147
148 expect(attempted).toBe(false)
149 expect(probe.unavailable).toBe('server is off')
150 })
151
152 it('surfaces the remote stderr when the command itself fails', async () => {
153 const probe = await probeHostRoutes(server, async () => ({ code: 1, stdout: '', stderr: 'find: permission denied\n' }))
154
155 expect(probe.unavailable).toBe('find: permission denied')
156 })
157})
158
159describe('reconciling declared sites against a box', () => {
160 const declared = [
161 site('main', { domain: 'stacksjs.com', path: '/' }),
162 site('docs', { domain: 'stacksjs.com', path: '/docs' }),
163 site('blog', { domain: 'stacksjs.com', path: '/blog' }),
164 site('api', { port: 3008 }),
165 ]
166 const routes = routesFromFragments([STACKS_FRAGMENT, RAPPID_FRAGMENT])
167
168 it('separates present, absent, loopback and somebody else entirely', () => {
169 const result = reconcile(declared, routes, 'stacks')
170
171 expect(result.present.map(s => s.name).sort()).toEqual(['docs', 'main'])
172 expect(result.absent.map(s => s.name)).toEqual(['blog'])
173 expect(result.loopback.map(s => s.name)).toEqual(['api'])
174 expect(result.foreign.map(r => r.slug)).toEqual(['rappid'])
175 })
176
177 it('matches on host and path, not on the site key', () => {
178 // The box has no idea what a repository calls its sites, and two projects
179 // both naming one `main` is ordinary.
180 expect(reconcile([site('frontend', { domain: 'stacksjs.com', path: '/' })], routes, 'stacks').present.map(s => s.name))
181 .toEqual(['frontend'])
182 })
183
184 it('does not credit our site to another project serving the same host', () => {
185 expect(reconcile([site('main', { domain: 'rappid.hq.training', path: '/' })], routes, 'stacks').absent.map(s => s.name))
186 .toEqual(['main'])
187 })
188
189 it('ignores a trailing slash on a path prefix', () => {
190 expect(reconcile([site('docs', { domain: 'stacksjs.com', path: '/docs/' })], routes, 'stacks').present).toHaveLength(1)
191 })
192
193 it('only counts a site missing when no answering box serves it', () => {
194 const probes = [
195 { server: 'a', routes: routesFromFragments([STACKS_FRAGMENT]) },
196 { server: 'b', routes: routesFromFragments([RAPPID_FRAGMENT]) },
197 ]
198
199 expect(unaccountedSites(declared, probes, 'stacks').map(s => s.name)).toEqual(['blog'])
200 })
201})
202
203describe('the listing an operator reads', () => {
204 const servers = [toInventoryServer({
205 id: 1,
206 name: 'stacks-production-app',
207 status: 'running',
208 public_net: { ipv4: { ip: '5.161.0.1' } },
209 server_type: { name: 'cpx41' },
210 labels: { 'ts-cloud/project': 'stacks', 'ts-cloud/environment': 'production', 'ts-cloud/role': 'app' },
211 })]
212
213 const declared = [
214 site('main', { domain: 'stacksjs.com', path: '/' }),
215 site('blog', { domain: 'blog.example', path: '/' }),
216 site('api', { port: 3008 }),
217 ]
218
219 it('names the co-tenant sharing the box', () => {
220 const output = formatInventory({
221 slug: 'stacks',
222 servers,
223 probes: [{ server: 'stacks-production-app', routes: routesFromFragments([STACKS_FRAGMENT, RAPPID_FRAGMENT]) }],
224 declared,
225 }).join('\n')
226
227 expect(output).toContain('serves 4 routes for 2 projects')
228 expect(output).toContain('stacks (this project)')
229 expect(output).toContain('rappid.hq.training/ -> 127.0.0.1:3024')
230 })
231
232 it('says a box was not probed rather than implying it hosts nothing', () => {
233 const output = formatInventory({ slug: 'stacks', servers, probes: [], declared }).join('\n')
234
235 expect(output).toContain('not probed')
236 expect(output).toContain('Nothing to reconcile them against')
237 })
238
239 it('refuses to reconcile against a box that could not be read', () => {
240 // "Every site is missing" is a true statement about the listing and a false
241 // one about the deployment.
242 const output = formatInventory({
243 slug: 'stacks',
244 servers,
245 probes: [{ server: 'stacks-production-app', routes: [], unavailable: 'Permission denied (publickey).' }],
246 declared,
247 }).join('\n')
248
249 expect(output).toContain('could not read /etc/rpx/sites.d: Permission denied (publickey).')
250 expect(output).toContain('Nothing to reconcile them against')
251 expect(output).not.toContain('not routed by any box above')
252 })
253
254 it('blames an unreadable box before it blames the deploy', () => {
255 const output = formatInventory({
256 slug: 'stacks',
257 servers: [...servers, toInventoryServer({ id: 2, name: 'other', status: 'running', labels: {} })],
258 probes: [
259 { server: 'stacks-production-app', routes: routesFromFragments([STACKS_FRAGMENT]) },
260 { server: 'other', routes: [], unavailable: 'Permission denied (publickey).' },
261 ],
262 declared,
263 }).join('\n')
264
265 expect(output).toContain('1 not routed by any box above: blog')
266 expect(output).toContain('one of the 1 server that could not be read')
267 })
268
269 it('explains a domainless site instead of listing it as missing', () => {
270 const output = formatInventory({
271 slug: 'stacks',
272 servers,
273 probes: [{ server: 'stacks-production-app', routes: routesFromFragments([STACKS_FRAGMENT]) }],
274 declared,
275 }).join('\n')
276
277 expect(output).toContain('1 with no domain')
278 expect(output).toContain('1 not routed by any box above: blog')
279 })
280})