ReviewOS
Reload

also looking at this

stacks/bunpress

chore: wip generate llm-files

#24
Open cab-mikee wants to merge chore/generate-agent-config-file into main
2 files +741 -296
bin/cli.tsmodified+618-44
@@ -218,51 +218,8 @@ async function generateIndexHtml(outdir: string, markdownFiles: string[]) {
218218 await Bun.write(join(outdir, 'index.html'), indexHtml)
219219}
220220
221cli
222 .command('build', 'Build the documentation site')
223 .option('--outdir <outdir>', 'Output directory', { default: defaultOptions.outdir })
224 .option('--config <config>', 'Path to config file')
225 .option('--verbose', 'Enable verbose logging', { default: defaultOptions.verbose })
226 .action(async (options: CliOption) => {
227 const success = await buildDocs(options)
228 if (!success)
229 process.exit(1)
230
231 console.log('Documentation built successfully!')
232 })
233
234cli
235 .command('dev', 'Build and serve documentation using BunPress server')
236 .option('--port <port>', 'Port to listen on', { default: defaultOptions.port })
237 .option('--dir <dir>', 'Documentation directory', { default: './docs' })
238 .option('--watch', 'Watch for changes', { default: defaultOptions.watch })
239 .option('--verbose', 'Enable verbose logging', { default: defaultOptions.verbose })
240 .action(async (options: CliOption) => {
241 const port = options.port || defaultOptions.port
242 const root = options.dir || './docs'
243 const watch = options.watch ?? defaultOptions.watch
244 const verbose = options.verbose ?? defaultOptions.verbose
245
246 if (verbose) {
247 console.log('Starting BunPress dev server with options:', {
248 port,
249 root,
250 watch,
251 verbose,
252 })
253 }
254
255 // Start the server using the serve.ts implementation
256 await serveCLI({
257 port,
258 root,
259 watch,
260 config: config as any,
261 })
262 })
263
264221/**
265 * Generate LLM-friendly markdown file from all documentation
222 * Generate LLM markdown file from all documentation
266223 */
267224async function generateLlmMarkdown(options: CliOption = {}): Promise<boolean> {
268225 const docsDir = options.dir || './docs'
@@ -363,6 +320,613 @@ async function generateLlmMarkdown(options: CliOption = {}): Promise<boolean> {
363320 return true
364321}
365322
323cli
324 .command('build', 'Build the documentation site')
325 .option('--outdir <outdir>', 'Output directory', { default: defaultOptions.outdir })
326 .option('--config <config>', 'Path to config file')
327 .option('--verbose', 'Enable verbose logging', { default: defaultOptions.verbose })
328 .action(async (options: CliOption) => {
329 const success = await buildDocs(options)
330 if (!success)
331 process.exit(1)
332
333 console.log('Documentation built successfully!')
334 })
335
336cli
337 .command('dev', 'Build and serve documentation using BunPress server')
338 .option('--port <port>', 'Port to listen on', { default: defaultOptions.port })
339 .option('--dir <dir>', 'Documentation directory', { default: './docs' })
340 .option('--watch', 'Watch for changes', { default: defaultOptions.watch })
341 .option('--verbose', 'Enable verbose logging', { default: defaultOptions.verbose })
342 .action(async (options: CliOption) => {
343 const port = options.port || defaultOptions.port
344 const root = options.dir || './docs'
345 const watch = options.watch ?? defaultOptions.watch
346 const verbose = options.verbose ?? defaultOptions.verbose
347
348 if (verbose) {
349 console.log('Starting BunPress dev server with options:', {
350 port,
351 root,
352 watch,
353 verbose,
354 })
355 }
356
357 // Start the server using the serve.ts implementation
358 await serveCLI({
359 port,
360 root,
361 watch,
362 config: config as any,
363 })
364 })
365
366/**
367 * Step 1: File discovery and reading from docs/
368 * Reads all markdown files from the docs directory and extracts their content
369 */
370async function readDocsContent(docsDir: string, verbose: boolean = false): Promise<{
371 files: Array<{ path: string, content: string, frontmatter: any }>
372 totalSize: number
373}> {
374 const markdownFiles = await findMarkdownFiles(docsDir)
375
376 if (verbose) {
377 console.log(`Found ${markdownFiles.length} markdown files in ${docsDir}`)
378 }
379
380 const files = []
381 let totalSize = 0
382
383 for (const filePath of markdownFiles) {
384 const content = await Bun.file(filePath).text()
385 totalSize += content.length
386
387 // Extract frontmatter if present
388 const frontmatterRegex = /^---\n([\s\S]*?)\n---\n?/
389 const match = content.match(frontmatterRegex)
390 let frontmatter = {}
391 let mainContent = content
392
393 if (match) {
394 try {
395 // Simple YAML parsing for frontmatter
396 const yamlContent = match[1]
397 frontmatter = yamlContent.split('\n').reduce((acc, line) => {
398 const [key, ...valueParts] = line.split(':')
399 if (key && valueParts.length > 0) {
400 acc[key.trim()] = valueParts.join(':').trim()
401 }
402 return acc
403 }, {} as any)
404 mainContent = content.slice(match[0].length)
405 } catch (error) {
406 if (verbose) {
407 console.warn(`Failed to parse frontmatter in ${filePath}`)
408 }
409 }
410 }
411
412 files.push({
413 path: filePath.replace(`${docsDir}/`, ''),
414 content: mainContent,
415 frontmatter,
416 })
417 }
418
419 return { files, totalSize }
420}
421
422/**
423 * Step 2: Text summarization and transformation logic
424 * Extracts key information and creates summaries from documentation
425 */
426function extractProjectInfo(files: Array<{ path: string, content: string, frontmatter: any }>) {
427 // Extract project name and description from README or index
428 const readme = files.find(f => f.path.toLowerCase().includes('readme') || f.path === 'index.md')
429 let projectName = 'BunPress'
430 let projectDescription = 'A modern documentation engine powered by Bun'
431 let features: string[] = []
432 let quickStart = ''
433
434 if (readme) {
435 // Extract title (first h1)
436 const titleMatch = readme.content.match(/^#\s+(.+)$/m)
437 if (titleMatch) {
438 projectName = titleMatch[1].trim()
439 }
440
441 // Extract description (first paragraph after title)
442 const descMatch = readme.content.match(/^#\s+.+$\n\n(.+?)(?:\n\n|$)/m)
443 if (descMatch) {
444 projectDescription = descMatch[1].trim()
445 }
446
447 // Extract features (look for ## Features section)
448 const featuresMatch = readme.content.match(/##\s+Features\s*\n([\s\S]*?)(?=\n##|$)/i)
449 if (featuresMatch) {
450 features = featuresMatch[1]
451 .split('\n')
452 .filter(line => line.trim().startsWith('-') || line.trim().startsWith('*'))
453 .map(line => line.replace(/^[-*]\s*/, '').trim())
454 .filter(Boolean)
455 }
456
457 // Extract quick start section
458 const quickStartMatch = readme.content.match(/##\s+(?:Quick Start|Getting Started)\s*\n([\s\S]*?)(?=\n##|$)/i)
459 if (quickStartMatch) {
460 quickStart = quickStartMatch[1].trim()
461 }
462 }
463
464 // Extract all headings for structure
465 const allHeadings = files.flatMap(file => {
466 const headings: Array<{ level: number, text: string, file: string }> = []
467 const headingRegex = /^(#{1,6})\s+(.+)$/gm
468 let match
469
470 while ((match = headingRegex.exec(file.content)) !== null) {
471 headings.push({
472 level: match[1].length,
473 text: match[2].trim(),
474 file: file.path,
475 })
476 }
477
478 return headings
479 })
480
481 return {
482 projectName,
483 projectDescription,
484 features,
485 quickStart,
486 allHeadings,
487 fileCount: files.length,
488 }
489}
490
491/**
492 * Generate short summary for llm.txt (under 500 tokens)
493 */
494function generateShortSummary(projectInfo: ReturnType<typeof extractProjectInfo>): string {
495 const { projectName, projectDescription, features } = projectInfo
496
497 let summary = `# ${projectName}\n\n${projectDescription}\n\n`
498
499 if (features.length > 0) {
500 summary += `## Key Features\n`
501 // Limit to top 5 features for brevity
502 features.slice(0, 5).forEach(feature => {
503 summary += `- ${feature}\n`
504 })
505 }
506
507 summary += `\n## Documentation Structure\n`
508 summary += `This project contains ${projectInfo.fileCount} documentation files.\n`
509
510 return summary
511}
512
513/**
514 * Generate detailed context for llm-full.txt
515 */
516function generateFullContext(
517 projectInfo: ReturnType<typeof extractProjectInfo>,
518 files: Array<{ path: string, content: string, frontmatter: any }>
519): string {
520 const { projectName, projectDescription, features, quickStart, allHeadings } = projectInfo
521
522 let context = `# ${projectName} - Complete Documentation Context\n\n`
523 context += `${projectDescription}\n\n`
524
525 if (features.length > 0) {
526 context += `## Features\n`
527 features.forEach(feature => {
528 context += `- ${feature}\n`
529 })
530 context += '\n'
531 }
532
533 if (quickStart) {
534 context += `## Quick Start\n${quickStart}\n\n`
535 }
536
537 context += `## Documentation Structure\n\n`
538 context += `Total files: ${files.length}\n\n`
539
540 // Group headings by file
541 const fileStructure = files.map(file => {
542 const fileHeadings = allHeadings.filter(h => h.file === file.path)
543 if (fileHeadings.length === 0) return null
544
545 let structure = `### ${file.path}\n`
546 fileHeadings.forEach(heading => {
547 const indent = ' '.repeat(heading.level - 1)
548 structure += `${indent}- ${heading.text}\n`
549 })
550 return structure
551 }).filter(Boolean)
552
553 context += fileStructure.join('\n')
554
555 context += `\n## Full Content\n\n`
556
557 // Include full content of all files
558 files.forEach(file => {
559 context += `---\n\n### File: ${file.path}\n\n`
560 if (Object.keys(file.frontmatter).length > 0) {
561 context += `**Frontmatter:**\n\`\`\`yaml\n`
562 Object.entries(file.frontmatter).forEach(([key, value]) => {
563 context += `${key}: ${value}\n`
564 })
565 context += `\`\`\`\n\n`
566 }
567 context += file.content
568 context += '\n\n'
569 })
570
571 return context
572}
573
574/**
575 * Generate CLAUDE.md with instructions for Claude Sonnet 4.5
576 */
577function generateClaudeInstructions(projectInfo: ReturnType<typeof extractProjectInfo>): string {
578 const { projectName, projectDescription } = projectInfo
579
580 return `# Claude Instructions for ${projectName}
581
582## Project Context
583${projectDescription}
584
585## Your Role
586You are an expert AI assistant helping developers work on ${projectName}. You have deep knowledge of:
587- Bun runtime and its ecosystem
588- TypeScript and modern JavaScript
589- Documentation engines and static site generators
590- Markdown processing and syntax highlighting
591- CLI development with Node.js/Bun
592
593## Behavior Guidelines
594
595### Tone & Style
596- **Professional but friendly**: Clear technical communication without jargon overload
597- **Solution-oriented**: Focus on actionable advice and working code
598- **Educational**: Explain concepts when relevant, don't just provide answers
599- **Efficient**: Respect the developer's time with concise responses
600
601### Capabilities
602✅ **You CAN:**
603- Explain how BunPress works internally
604- Help debug issues with markdown processing
605- Suggest improvements to the codebase
606- Write new features or fix bugs
607- Optimize performance
608- Improve documentation
609- Add new CLI commands
610- Extend markdown syntax support
611
612❌ **You SHOULD NOT:**
613- Suggest migrating away from Bun to Node.js
614- Recommend complex frameworks when simple solutions exist
615- Break backward compatibility without strong justification
616- Add features that conflict with VitePress compatibility
617- Over-engineer solutions
618
619### Decision-Making Framework
620When helping with ${projectName}:
621
6221. **Understand the goal**: What is the developer trying to achieve?
6232. **Check existing patterns**: How does the codebase handle similar cases?
6243. **Consider alternatives**: What are the trade-offs?
6254. **Prioritize simplicity**: Simple, maintainable code > clever code
6265. **Think about users**: How does this affect documentation authors?
627
628### Code Quality Standards
629- Type safety: Use TypeScript types, avoid \`any\`
630- Error handling: Always handle edge cases and errors gracefully
631- Performance: Consider build time and runtime performance
632- Readability: Code should be self-explanatory
633- Testing: Suggest tests for complex logic
634
635### Common Scenarios
636
637#### Adding a New Markdown Feature
6381. Check if VitePress supports it (maintain compatibility)
6392. Implement in \`/src/serve.ts\` markdown processing
6403. Add CSS styles to \`/src/config.ts\`
6414. Update TypeScript types in \`/src/types.ts\`
6425. Document the feature
643
644#### Debugging Build Issues
6451. Check the CLI command in \`/bin/cli.ts\`
6462. Verify markdown file processing in \`/src/serve.ts\`
6473. Inspect template rendering in \`/src/template-loader.ts\`
6484. Review configuration in \`bunpress.config.ts\`
649
650#### Performance Optimization
6511. Profile the slow operation
6522. Check if Bun has native APIs for the task
6533. Consider caching strategies
6544. Optimize regex patterns and string operations
6555. Minimize file I/O operations
656
657## Knowledge Boundaries
658
659### What You Know Well
660- Bun runtime features and APIs
661- TypeScript best practices
662- Markdown processing and extensions
663- Static site generation patterns
664- CLI development
665- Template systems
666
667### What You Should Research
668- Specific VitePress features not yet implemented
669- New Bun APIs in recent versions
670- Third-party plugin compatibility
671- Specific user's custom configuration needs
672
673## Interaction Guidelines
674
675### When Answering Questions
6761. Acknowledge the question clearly
6772. Provide context if needed
6783. Give a direct answer with code examples
6794. Explain trade-offs or alternatives
6805. Suggest next steps or related improvements
681
682### When Writing Code
6831. Follow existing code style and patterns
6842. Add comments for complex logic
6853. Include error handling
6864. Consider edge cases
6875. Make it easy to test
688
689### When Suggesting Changes
6901. Explain the problem being solved
6912. Show the proposed solution
6923. Discuss potential impacts
6934. Provide migration path if breaking
6945. Consider documentation updates
695
696## Success Metrics
697You're doing well when:
698- Developers can quickly understand and implement your suggestions
699- Code changes integrate smoothly with existing patterns
700- Solutions are performant and maintainable
701- Documentation authors have a better experience
702- The project moves forward without breaking changes
703`
704}
705
706/**
707 * Generate AGENT.md with general AI agent instructions
708 */
709function generateAgentInstructions(projectInfo: ReturnType<typeof extractProjectInfo>): string {
710 const { projectName, projectDescription } = projectInfo
711
712 return `# AI Agent Instructions for ${projectName}
713
714## Project Overview
715${projectDescription}
716
717This is a documentation engine built with Bun, inspired by VitePress. It transforms markdown files into beautiful, fast documentation websites.
718
719## Domain Knowledge
720
721### Core Technologies
722- **Bun**: Modern JavaScript runtime (faster alternative to Node.js)
723- **TypeScript**: Strongly-typed JavaScript for better DX
724- **Markdown**: Content format with extended syntax support
725- **Static Site Generation**: Build-time rendering for performance
726
727### Architecture Components
7281. **CLI** (\`/bin/cli.ts\`): Command-line interface with build, dev, and llm commands
7292. **Config** (\`/src/config.ts\`): Default configuration and settings
7303. **Serve** (\`/src/serve.ts\`): Development server with hot reload
7314. **Templates** (\`/src/templates/\`): STX template files for layouts
7325. **TOC** (\`/src/toc.ts\`): Table of contents generation
7336. **Highlighter** (\`/src/highlighter.ts\`): Syntax highlighting for code blocks
734
735## Agent Behavior
736
737### Tone & Style
738- **Professional but friendly**: Clear technical communication without jargon overload
739- **Solution-oriented**: Focus on actionable advice and working code
740- **Educational**: Explain concepts when relevant, don't just provide answers
741- **Efficient**: Respect the developer's time with concise responses
742
743### Capabilities
744✅ **You CAN:**
745- Explain how BunPress works internally
746- Help debug issues with markdown processing
747- Suggest improvements to the codebase
748- Write new features or fix bugs
749- Optimize performance
750- Improve documentation
751- Add new CLI commands
752- Extend markdown syntax support
753
754❌ **You SHOULD NOT:**
755- Suggest migrating away from Bun to Node.js
756- Recommend complex frameworks when simple solutions exist
757- Break backward compatibility without strong justification
758- Add features that conflict with VitePress compatibility
759- Over-engineer solutions
760
761### Decision-Making Framework
762When helping with ${projectName}:
763
7641. **Understand the goal**: What is the developer trying to achieve?
7652. **Check existing patterns**: How does the codebase handle similar cases?
7663. **Consider alternatives**: What are the trade-offs?
7674. **Prioritize simplicity**: Simple, maintainable code > clever code
7685. **Think about users**: How does this affect documentation authors?
769
770### Code Quality Standards
771- Type safety: Use TypeScript types, avoid \`any\`
772- Error handling: Always handle edge cases and errors gracefully
773- Performance: Consider build time and runtime performance
774- Readability: Code should be self-explanatory
775- Testing: Suggest tests for complex logic
776
777### Common Scenarios
778
779#### Adding a New Markdown Feature
7801. Check if VitePress supports it (maintain compatibility)
7812. Implement in \`/src/serve.ts\` markdown processing
7823. Add CSS styles to \`/src/config.ts\`
7834. Update TypeScript types in \`/src/types.ts\`
7845. Document the feature
785
786#### Debugging Build Issues
7871. Check the CLI command in \`/bin/cli.ts\`
7882. Verify markdown file processing in \`/src/serve.ts\`
7893. Inspect template rendering in \`/src/template-loader.ts\`
7904. Review configuration in \`bunpress.config.ts\`
791
792#### Performance Optimization
7931. Profile the slow operation
7942. Check if Bun has native APIs for the task
7953. Consider caching strategies
7964. Optimize regex patterns and string operations
7975. Minimize file I/O operations
798
799## Knowledge Boundaries
800
801### What You Know Well
802- Bun runtime features and APIs
803- TypeScript best practices
804- Markdown processing and extensions
805- Static site generation patterns
806- CLI development
807- Template systems
808
809### What You Should Research
810- Specific VitePress features not yet implemented
811- New Bun APIs in recent versions
812- Third-party plugin compatibility
813- Specific user's custom configuration needs
814
815## Interaction Guidelines
816
817### When Answering Questions
8181. Acknowledge the question clearly
8192. Provide context if needed
8203. Give a direct answer with code examples
8214. Explain trade-offs or alternatives
8225. Suggest next steps or related improvements
823
824### When Writing Code
8251. Follow existing code style and patterns
8262. Add comments for complex logic
8273. Include error handling
8284. Consider edge cases
8295. Make it easy to test
830
831### When Suggesting Changes
8321. Explain the problem being solved
8332. Show the proposed solution
8343. Discuss potential impacts
8354. Provide migration path if breaking
8365. Consider documentation updates
837
838## Success Metrics
839You're doing well when:
840- Developers can quickly understand and implement your suggestions
841- Code changes integrate smoothly with existing patterns
842- Solutions are performant and maintainable
843- Documentation authors have a better experience
844- The project moves forward without breaking changes
845`
846}
847
848/**
849 * Step 3: File generation and output
850 * Writes the generated content to files in the project root
851 */
852async function generateLlmFiles(options: CliOption = {}): Promise<boolean> {
853 const docsDir = options.dir || './docs'
854 const verbose = options.verbose ?? defaultOptions.verbose
855
856 if (verbose) {
857 console.log('Starting LLM files generation...')
858 console.log(`Reading documentation from: ${docsDir}`)
859 }
860
861 try {
862 // Step 1: Read all documentation files
863 const { files, totalSize } = await readDocsContent(docsDir, verbose)
864
865 if (files.length === 0) {
866 console.log('No markdown files found in docs directory')
867 return false
868 }
869
870 if (verbose) {
871 console.log(`Processed ${files.length} files (${(totalSize / 1024).toFixed(2)} KB total)`)
872 }
873
874 // Step 2: Extract and transform content
875 const projectInfo = extractProjectInfo(files)
876
877 if (verbose) {
878 console.log(`Project: ${projectInfo.projectName}`)
879 console.log(`Features found: ${projectInfo.features.length}`)
880 }
881
882 // Step 3: Generate each file
883 const outputs = [
884 {
885 filename: 'llm.txt',
886 content: generateShortSummary(projectInfo),
887 description: 'Short summary for LLM initialization',
888 },
889 {
890 filename: 'llm-full.txt',
891 content: generateFullContext(projectInfo, files),
892 description: 'Complete context for fine-tuned reasoning',
893 },
894 {
895 filename: 'CLAUDE.md',
896 content: generateClaudeInstructions(projectInfo),
897 description: 'Claude Sonnet 4.5 behavior instructions',
898 },
899 {
900 filename: 'AGENT.md',
901 content: generateAgentInstructions(projectInfo),
902 description: 'General AI agent behavior instructions',
903 },
904 ]
905
906 // Write all files
907 for (const output of outputs) {
908 await Bun.write(output.filename, output.content)
909
910 if (verbose) {
911 const size = (output.content.length / 1024).toFixed(2)
912 console.log(`✓ Generated ${output.filename} (${size} KB) - ${output.description}`)
913 }
914 }
915
916 console.log('\n✨ Successfully generated all LLM context files!')
917 console.log('\nGenerated files:')
918 outputs.forEach(output => {
919 console.log(` - ${output.filename}`)
920 })
921
922 return true
923 }
924 catch (error) {
925 console.error('Error generating LLM files:', error)
926 return false
927 }
928}
929
366930cli
367931 .command('llm', 'Generate LLM-friendly markdown file from documentation')
368932 .option('--dir <dir>', 'Documentation directory', { default: './docs' })
@@ -375,6 +939,16 @@ cli
375939 process.exit(1)
376940 })
377941
942cli
943 .command('generate:llm-files', 'Generate LLM context files (llm.txt, llm-full.txt, CLAUDE.md, AGENT.md)')
944 .option('--dir <dir>', 'Documentation directory', { default: './docs' })
945 .option('--verbose', 'Enable verbose logging', { default: defaultOptions.verbose })
946 .action(async (options: CliOption) => {
947 const success = await generateLlmFiles(options)
948 if (!success)
949 process.exit(1)
950 })
951
378952cli.help()
379953cli.version(version)
380954cli.parse()