ReviewOS

also looking at this

stacks/postline

chore(deps): upgrade vendored Stacks framework to 0.70.163

#29
Merged glennmichael123 wants to merge chore/upgrade-0.70.163 into main
224 files +3516 -874
storage/framework/core/ai/src/context.tsadded+300-0
Changes to storage/framework/core/ai/src/context.ts
@@ -0,0 +1,300 @@
1import { existsSync, lstatSync, readFileSync, readdirSync } from 'node:fs'
2import { basename, relative, resolve, sep } from 'node:path'
3import { estimateTokens } from './utils/tokens'
4
5export const STACKS_PROJECT_CONTEXT_SCHEMA = 'https://stacksjs.org/schemas/ai-project-context/v1'
6
7const DEFAULT_MAX_CHARS = 4000
8const DEFAULT_MAX_FILES = 30
9
10const excludedSegments = new Set([
11 '.git',
12 '.bun',
13 '.cache',
14 '.next',
15 '.output',
16 '.turbo',
17 '.vite',
18 'build',
19 'coverage',
20 'dist',
21 'node_modules',
22 'tmp',
23 'vendor',
24])
25
26const sensitiveNames = /^(?:\.env(?:\..*)?|\.npmrc|\.pypirc|credentials(?:\..*)?|id_(?:rsa|ed25519)(?:\.pub)?|secrets?(?:\..*)?)$/i
27const lockNames = /(?:^|\.)(?:bun|package-lock|pnpm-lock|yarn|pantry)\.?(?:lock|yaml)?$/i
28
29export interface ProjectContextOptions {
30 maxChars?: number
31 maxFiles?: number
32 model?: string
33}
34
35export interface ProjectContextSurface {
36 id: string
37 purpose: string
38 paths: string[]
39}
40
41export interface StacksProjectContext {
42 schema: typeof STACKS_PROJECT_CONTEXT_SCHEMA
43 schemaVersion: '1.0.0'
44 framework: 'stacks'
45 architecture: {
46 pattern: 'Model-View-Action'
47 overrideRule: string
48 authoringOrder: string[]
49 principles: string[]
50 }
51 project: {
52 name: string | null
53 version: string | null
54 scripts: string[]
55 dependencies: string[]
56 }
57 instructionFiles: string[]
58 surfaces: ProjectContextSurface[]
59 representativeFiles: string[]
60 exclusions: string[]
61}
62
63export interface ProjectContextMetrics {
64 candidateFiles: number
65 includedFiles: number
66 maxCharacters: number
67 outputCharacters: number
68 estimatedTokens: number
69 model: string
70 truncated: boolean
71 baselineMethod: 'sorted-first-50-paths+readme-2000+package-json'
72 baselineCharacters: number
73 baselineEstimatedTokens: number
74 estimatedTokenReductionPercent: number | null
75 tokenEstimateIsHeuristic: true
76}
77
78export interface ProjectContextResult {
79 context: StacksProjectContext
80 text: string
81 metrics: ProjectContextMetrics
82}
83
84interface SurfaceDefinition {
85 id: string
86 purpose: string
87 prefixes: string[]
88}
89
90const surfaceDefinitions: SurfaceDefinition[] = [
91 { id: 'models', purpose: 'Domain schema, validation, relationships, factories, and traits.', prefixes: ['app/Models/', 'storage/framework/defaults/app/Models/'] },
92 { id: 'actions', purpose: 'Transport-independent application behavior.', prefixes: ['app/Actions/', 'storage/framework/defaults/app/Actions/'] },
93 { id: 'routes', purpose: 'HTTP and API route registration.', prefixes: ['routes/', 'app/Routes.ts'] },
94 { id: 'jobs', purpose: 'Background and inline work.', prefixes: ['app/Jobs/', 'storage/framework/defaults/app/Jobs/'] },
95 { id: 'configuration', purpose: 'Typed application and provider configuration.', prefixes: ['config/'] },
96 { id: 'views', purpose: 'STX views, layouts, components, and client functions.', prefixes: ['resources/'] },
97 { id: 'database', purpose: 'Reviewable migrations and seed data.', prefixes: ['database/'] },
98 { id: 'tests', purpose: 'Unit, feature, contract, and end-to-end evidence.', prefixes: ['tests/', 'storage/framework/core/'] },
99]
100
101function normalizePath(path: string): string {
102 return path.split(sep).join('/')
103}
104
105function isExcluded(path: string): boolean {
106 const segments = path.split('/')
107 const name = basename(path)
108 return segments.some(segment => excludedSegments.has(segment))
109 || sensitiveNames.test(name)
110 || lockNames.test(name)
111 || name.endsWith('.lock')
112}
113
114function collectFiles(root: string): string[] {
115 const files: string[] = []
116
117 function visit(directory: string): void {
118 const entries = readdirSync(directory, { withFileTypes: true })
119 .sort((a, b) => a.name.localeCompare(b.name))
120
121 for (const entry of entries) {
122 const fullPath = resolve(directory, entry.name)
123 const projectPath = normalizePath(relative(root, fullPath))
124 if (isExcluded(projectPath)) continue
125 if (entry.isDirectory()) visit(fullPath)
126 else if (entry.isFile() && !lstatSync(fullPath).isSymbolicLink()) files.push(projectPath)
127 }
128 }
129
130 visit(root)
131 return files.sort((a, b) => a.localeCompare(b))
132}
133
134function readPackageSummary(root: string): StacksProjectContext['project'] {
135 const packagePath = resolve(root, 'package.json')
136 if (!existsSync(packagePath)) return { name: null, version: null, scripts: [], dependencies: [] }
137
138 try {
139 const manifest = JSON.parse(readFileSync(packagePath, 'utf8')) as Record<string, unknown>
140 const dependencies = [
141 ...Object.keys((manifest.dependencies || {}) as Record<string, unknown>),
142 ...Object.keys((manifest.devDependencies || {}) as Record<string, unknown>),
143 ...Object.keys((manifest.peerDependencies || {}) as Record<string, unknown>),
144 ]
145 return {
146 name: typeof manifest.name === 'string' ? manifest.name : null,
147 version: typeof manifest.version === 'string' ? manifest.version : null,
148 scripts: Object.keys((manifest.scripts || {}) as Record<string, unknown>).sort().slice(0, 25),
149 dependencies: [...new Set(dependencies)].sort().slice(0, 30),
150 }
151 }
152 catch {
153 return { name: null, version: null, scripts: [], dependencies: [] }
154 }
155}
156
157function filePriority(path: string): number {
158 if (/^(?:AGENTS|CLAUDE)\.md$|^\.github\/copilot-instructions\.md$/.test(path)) return 0
159 if (path === 'package.json' || path === 'app/Routes.ts' || path === 'app/Commands.ts') return 1
160 if (/^(?:app|routes|config)\//.test(path)) return 2
161 if (/^(?:resources|database|tests)\//.test(path)) return 3
162 if (/^docs\//.test(path)) return 4
163 if (/^storage\/framework\/defaults\//.test(path)) return 5
164 return 6
165}
166
167function representativeFiles(files: string[], maxFiles: number): string[] {
168 return [...files]
169 .sort((a, b) => filePriority(a) - filePriority(b) || a.localeCompare(b))
170 .slice(0, maxFiles)
171 .sort((a, b) => a.localeCompare(b))
172}
173
174function surfacePaths(files: string[], prefixes: string[]): string[] {
175 return files
176 .filter(file => prefixes.some(prefix => prefix.endsWith('/') ? file.startsWith(prefix) : file === prefix))
177 .slice(0, 4)
178}
179
180function renderContext(context: StacksProjectContext): string {
181 const lines = [
182 'Stacks AI Project Context v1',
183 '',
184 `Project: ${context.project.name || 'unnamed'}${context.project.version ? `@${context.project.version}` : ''}`,
185 'Architecture: Model-View-Action',
186 `Override rule: ${context.architecture.overrideRule}`,
187 `Authoring order: ${context.architecture.authoringOrder.join(' -> ')}`,
188 '',
189 'Authoring principles:',
190 ...context.architecture.principles.map(principle => `- ${principle}`),
191 '',
192 'Instruction files:',
193 ...(context.instructionFiles.length ? context.instructionFiles.map(file => `- ${file}`) : ['- none detected']),
194 '',
195 'Available surfaces:',
196 ...context.surfaces.flatMap(surface => [
197 `- ${surface.id}: ${surface.purpose}`,
198 ...surface.paths.map(path => ` - ${path}`),
199 ]),
200 '',
201 `Scripts: ${context.project.scripts.join(', ') || 'none detected'}`,
202 `Dependencies: ${context.project.dependencies.join(', ') || 'none detected'}`,
203 '',
204 'Representative files:',
205 ...context.representativeFiles.map(file => `- ${file}`),
206 ]
207 return lines.join('\n')
208}
209
210function fitToBudget(text: string, maxChars: number): { text: string, truncated: boolean } {
211 if (text.length <= maxChars) return { text, truncated: false }
212 const marker = '\n[context truncated by character budget]'
213 if (maxChars <= marker.length) return { text: marker.slice(0, maxChars), truncated: true }
214 const available = maxChars - marker.length
215 const prefix = text.slice(0, available)
216 const lastLine = prefix.lastIndexOf('\n')
217 return {
218 text: `${prefix.slice(0, lastLine > 0 ? lastLine : available)}${marker}`,
219 truncated: true,
220 }
221}
222
223function unstructuredBaseline(root: string, files: string[]): string {
224 const readmePath = resolve(root, 'README.md')
225 const packagePath = resolve(root, 'package.json')
226 const readme = existsSync(readmePath) ? readFileSync(readmePath, 'utf8').slice(0, 2000) : ''
227 const packageJson = existsSync(packagePath) ? readFileSync(packagePath, 'utf8') : ''
228 return [
229 'Repository Structure:',
230 files.slice(0, 50).join('\n'),
231 readme ? `README.md (excerpt):\n${readme}` : '',
232 packageJson ? `package.json:\n${packageJson}` : '',
233 ].filter(Boolean).join('\n\n')
234}
235
236export function buildProjectContext(repoPath: string, options: ProjectContextOptions = {}): ProjectContextResult {
237 const root = resolve(repoPath)
238 const maxChars = options.maxChars ?? DEFAULT_MAX_CHARS
239 const maxFiles = options.maxFiles ?? DEFAULT_MAX_FILES
240 const model = options.model || 'gpt-4o'
241 if (!Number.isInteger(maxChars) || maxChars < 256) throw new Error('maxChars must be an integer of at least 256')
242 if (!Number.isInteger(maxFiles) || maxFiles < 1) throw new Error('maxFiles must be a positive integer')
243
244 const files = collectFiles(root)
245 const included = representativeFiles(files, maxFiles)
246 const context: StacksProjectContext = {
247 schema: STACKS_PROJECT_CONTEXT_SCHEMA,
248 schemaVersion: '1.0.0',
249 framework: 'stacks',
250 architecture: {
251 pattern: 'Model-View-Action',
252 overrideRule: 'Application files under app/ override matching framework defaults; do not edit defaults for application customization.',
253 authoringOrder: ['model', 'migration', 'action', 'route', 'test'],
254 principles: [
255 'Prefer conventional paths and framework generators over bespoke glue code.',
256 'Declare domain shape once and derive validation, migrations, API surfaces, and types where supported.',
257 'Keep Actions transport-independent and make unsupported drivers fail loudly.',
258 'Read repository instruction files before changing source.',
259 ],
260 },
261 project: readPackageSummary(root),
262 instructionFiles: files.filter(file => /(?:^|\/)(?:AGENTS|CLAUDE)\.md$|^\.github\/copilot-instructions\.md$/.test(file)),
263 surfaces: surfaceDefinitions
264 .map(surface => ({ id: surface.id, purpose: surface.purpose, paths: surfacePaths(files, surface.prefixes) }))
265 .filter(surface => surface.paths.length > 0),
266 representativeFiles: included,
267 exclusions: [
268 'dependency, build, cache, coverage, temporary, and vendor directories',
269 'lockfiles',
270 'environment, credential, private-key, and secret files',
271 ],
272 }
273
274 const fitted = fitToBudget(renderContext(context), maxChars)
275 const baseline = unstructuredBaseline(root, files)
276 const outputTokens = estimateTokens(fitted.text, model)
277 const baselineTokens = estimateTokens(baseline, model)
278 const reduction = baselineTokens > 0
279 ? Math.round((1 - outputTokens / baselineTokens) * 1000) / 10
280 : null
281
282 return {
283 context,
284 text: fitted.text,
285 metrics: {
286 candidateFiles: files.length,
287 includedFiles: included.length,
288 maxCharacters: maxChars,
289 outputCharacters: fitted.text.length,
290 estimatedTokens: outputTokens,
291 model,
292 truncated: fitted.truncated,
293 baselineMethod: 'sorted-first-50-paths+readme-2000+package-json',
294 baselineCharacters: baseline.length,
295 baselineEstimatedTokens: baselineTokens,
296 estimatedTokenReductionPercent: reduction,
297 tokenEstimateIsHeuristic: true,
298 },
299 }
300}