also looking at this
perf(isolated): benchmark declaration generation paths
#3106
4 files
+215
-6
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -6,6 +6,7 @@ import { performance } from 'node:perf_hooks' | ||
| 6 | 6 | import { extractDeclarations } from './src/extractor' |
| 7 | 7 | import { generate } from './src/generator' |
| 8 | 8 | import { createStreamingProcessor, formatMemoryStats } from './src/memory' |
| 9 | import { processSourceDirect } from './src/process-source' | |
| 9 | 10 | import { processDeclarations } from './src/processor' |
| 10 | 11 | |
| 11 | 12 | /** |
| @@ -40,6 +41,22 @@ interface SuiteResult { | ||
| 40 | 41 | totalTimeMs: number |
| 41 | 42 | } |
| 42 | 43 | |
| 44 | /** Paired end-to-end timings for the two declaration generation paths. */ | |
| 45 | interface GenerationModeComparison { | |
| 46 | name: string | |
| 47 | inputSize: number | |
| 48 | isolated: BenchmarkResult | |
| 49 | semantic: BenchmarkResult | |
| 50 | /** Semantic time divided by isolated time. Values above 1 favor isolated mode. */ | |
| 51 | speedup: number | |
| 52 | } | |
| 53 | ||
| 54 | interface GenerationModeSuiteResult { | |
| 55 | name: string | |
| 56 | comparisons: GenerationModeComparison[] | |
| 57 | totalTimeMs: number | |
| 58 | } | |
| 59 | ||
| 43 | 60 | /** |
| 44 | 61 | * Phase timing result |
| 45 | 62 | */ |
| @@ -281,6 +298,76 @@ async function runGenerationBenchmarks(config: BenchmarkConfig): Promise<SuiteRe | ||
| 281 | 298 | } |
| 282 | 299 | } |
| 283 | 300 | |
| 301 | /** | |
| 302 | * Generate valid isolated-declarations input with expensive but fully typed | |
| 303 | * initializers. The public contract is identical in both modes; only semantic | |
| 304 | * mode needs to inspect the initializer values. | |
| 305 | */ | |
| 306 | function generateAnnotatedInitializerFile(declarations: number, entriesPerInitializer: number): string { | |
| 307 | const entries = Array.from({ length: entriesPerInitializer }, (_, index) => `'tag-${index}'`).join(', ') | |
| 308 | const lines = ['export interface BenchmarkItem { id: number; tags: readonly string[] }'] | |
| 309 | for (let index = 0; index < declarations; index++) { | |
| 310 | lines.push(`export const item${index}: BenchmarkItem = { id: ${index}, tags: [${entries}] };`) | |
| 311 | } | |
| 312 | return lines.join('\n') | |
| 313 | } | |
| 314 | ||
| 315 | /** Compare uncached end-to-end isolated and semantic declaration generation. */ | |
| 316 | async function runGenerationModeBenchmarks(config: BenchmarkConfig): Promise<GenerationModeSuiteResult> { | |
| 317 | log('\n⚡ isolatedDeclarations vs Semantic Benchmarks\n', config.verbose) | |
| 318 | ||
| 319 | const suiteStart = performance.now() | |
| 320 | const comparisons: GenerationModeComparison[] = [] | |
| 321 | const workloads = [ | |
| 322 | { name: 'Small typed initializers', declarations: 100, entries: 50 }, | |
| 323 | { name: 'Medium typed initializers', declarations: 500, entries: 100 }, | |
| 324 | { name: 'Large typed initializers', declarations: 1000, entries: 250 }, | |
| 325 | ] | |
| 326 | ||
| 327 | for (const workload of workloads) { | |
| 328 | const source = generateAnnotatedInitializerFile(workload.declarations, workload.entries) | |
| 329 | const filename = `generation-mode-${workload.declarations}.ts` | |
| 330 | const isolatedOutput = processSourceDirect(source, filename, false, ['bun'], true) | |
| 331 | const semanticOutput = processSourceDirect(source, filename, false, ['bun'], false) | |
| 332 | if (isolatedOutput !== semanticOutput) { | |
| 333 | throw new Error(`${workload.name} produced different public declarations between generation modes`) | |
| 334 | } | |
| 335 | ||
| 336 | const modeConfig = { | |
| 337 | ...config, | |
| 338 | warmupIterations: Math.max(3, config.warmupIterations), | |
| 339 | benchmarkIterations: Math.max(10, Math.min(config.benchmarkIterations, 30)), | |
| 340 | } | |
| 341 | const isolated = await runBenchmark( | |
| 342 | `${workload.name} (isolated)`, | |
| 343 | () => processSourceDirect(source, filename, false, ['bun'], true), | |
| 344 | modeConfig, | |
| 345 | source.length, | |
| 346 | ) | |
| 347 | const semantic = await runBenchmark( | |
| 348 | `${workload.name} (semantic)`, | |
| 349 | () => processSourceDirect(source, filename, false, ['bun'], false), | |
| 350 | modeConfig, | |
| 351 | source.length, | |
| 352 | ) | |
| 353 | const speedup = semantic.avgTimeMs / isolated.avgTimeMs | |
| 354 | comparisons.push({ | |
| 355 | name: workload.name, | |
| 356 | inputSize: source.length, | |
| 357 | isolated, | |
| 358 | semantic, | |
| 359 | speedup, | |
| 360 | }) | |
| 361 | log(` ${workload.name}: isolated ${isolated.avgTimeMs.toFixed(3)}ms, semantic ${semantic.avgTimeMs.toFixed(3)}ms — ${speedup.toFixed(1)}x faster`, config.verbose) | |
| 362 | } | |
| 363 | ||
| 364 | return { | |
| 365 | name: 'Generation Modes', | |
| 366 | comparisons, | |
| 367 | totalTimeMs: performance.now() - suiteStart, | |
| 368 | } | |
| 369 | } | |
| 370 | ||
| 284 | 371 | /** |
| 285 | 372 | * Memory benchmark suite |
| 286 | 373 | */ |
| @@ -720,6 +807,7 @@ interface BenchmarkOutput { | ||
| 720 | 807 | platform: string |
| 721 | 808 | nodeVersion: string |
| 722 | 809 | suites: SuiteResult[] |
| 810 | generationModes: GenerationModeSuiteResult | |
| 723 | 811 | phaseTimings: PhaseTimingSuiteResult[] |
| 724 | 812 | summary: { |
| 725 | 813 | totalTimeMs: number |
| @@ -766,6 +854,7 @@ async function main() { | ||
| 766 | 854 | } |
| 767 | 855 | |
| 768 | 856 | const suites: SuiteResult[] = [] |
| 857 | let generationModes: GenerationModeSuiteResult = { name: 'Generation Modes', comparisons: [], totalTimeMs: 0 } | |
| 769 | 858 | let phaseTimings: PhaseTimingSuiteResult[] = [] |
| 770 | 859 | |
| 771 | 860 | try { |
| @@ -773,6 +862,7 @@ async function main() { | ||
| 773 | 862 | suites.push(await runExtractionBenchmarks(config)) |
| 774 | 863 | suites.push(await runSyntheticBenchmarks(config)) |
| 775 | 864 | suites.push(await runMemoryBenchmarks(config)) |
| 865 | generationModes = await runGenerationModeBenchmarks(config) | |
| 776 | 866 | |
| 777 | 867 | // Run real-world benchmarks unless skipped |
| 778 | 868 | if (!args.includes('--skip-real-world')) { |
| @@ -789,9 +879,12 @@ async function main() { | ||
| 789 | 879 | } |
| 790 | 880 | |
| 791 | 881 | // Calculate summary |
| 792 | const totalTimeMs = suites.reduce((sum, s) => sum + s.totalTimeMs, 0) | |
| 793 | const totalBenchmarks = suites.reduce((sum, s) => sum + s.results.length, 0) | |
| 794 | const avgTimeMs = suites.flatMap(s => s.results).reduce((sum, r) => sum + r.avgTimeMs, 0) / totalBenchmarks | |
| 882 | const modeResults = generationModes.comparisons.flatMap(comparison => [comparison.isolated, comparison.semantic]) | |
| 883 | const suiteResults = suites.flatMap(suite => suite.results) | |
| 884 | const allResults = [...suiteResults, ...modeResults] | |
| 885 | const totalTimeMs = suites.reduce((sum, suite) => sum + suite.totalTimeMs, generationModes.totalTimeMs) | |
| 886 | const totalBenchmarks = allResults.length | |
| 887 | const avgTimeMs = allResults.reduce((sum, result) => sum + result.avgTimeMs, 0) / totalBenchmarks | |
| 795 | 888 | |
| 796 | 889 | // Output results |
| 797 | 890 | if (jsonOutput) { |
| @@ -800,6 +893,7 @@ async function main() { | ||
| 800 | 893 | platform: `${process.platform} ${process.arch}`, |
| 801 | 894 | nodeVersion: process.version, |
| 802 | 895 | suites, |
| 896 | generationModes, | |
| 803 | 897 | phaseTimings, |
| 804 | 898 | summary: { |
| 805 | 899 | totalTimeMs, |
| @@ -822,6 +916,13 @@ async function main() { | ||
| 822 | 916 | // Print summary |
| 823 | 917 | printSummary(suites) |
| 824 | 918 | |
| 919 | if (generationModes.comparisons.length > 0) { | |
| 920 | console.log('\n⚡ isolatedDeclarations Speedup') | |
| 921 | for (const comparison of generationModes.comparisons) { | |
| 922 | console.log(` ${comparison.name}: ${comparison.speedup.toFixed(1)}x`) | |
| 923 | } | |
| 924 | } | |
| 925 | ||
| 825 | 926 | // Print phase timing summary |
| 826 | 927 | if (phaseTimings.length > 0) { |
| 827 | 928 | printPhaseTimingSummary(phaseTimings) |
| @@ -846,12 +947,16 @@ export { | ||
| 846 | 947 | type BenchmarkConfig, |
| 847 | 948 | type BenchmarkOutput, |
| 848 | 949 | type BenchmarkResult, |
| 950 | generateAnnotatedInitializerFile, | |
| 849 | 951 | generateLargeTypeScriptFile, |
| 952 | type GenerationModeComparison, | |
| 953 | type GenerationModeSuiteResult, | |
| 850 | 954 | type PhaseTimingResult, |
| 851 | 955 | type PhaseTimingSuiteResult, |
| 852 | 956 | runBenchmark, |
| 853 | 957 | runExtractionBenchmarks, |
| 854 | 958 | runGenerationBenchmarks, |
| 959 | runGenerationModeBenchmarks, | |
| 855 | 960 | runMemoryBenchmarks, |
| 856 | 961 | runPhaseTimingBenchmarks, |
| 857 | 962 | runRealWorldBenchmarks, |
| @@ -859,5 +964,5 @@ export { | ||
| 859 | 964 | type SuiteResult, |
| 860 | 965 | } |
| 861 | 966 | |
| 862 | // Run if executed directly | |
| 863 | main().catch(console.error) | |
| 967 | // Run only when executed directly; importing benchmark helpers must be side-effect free. | |
| 968 | if (import.meta.main) main().catch(console.error) | |