v1.1.0: pooled runtime, 959 tests, production hardening (0 squash)

2025-08-15 10:00:00 -07:00
commit 92deb689cd
321 changed files with 79170 additions and 0 deletions
@@ -0,0 +1,204 @@
+// bench.test.ts - Tests for imhotep-bench harness
+// Validates benchmark execution and cache correctness.
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert'
+import {
+  MemoryCache,
+  CompilationCache,
+  ExtractionCache,
+  runBenchmark,
+  BenchmarkSuite,
+  profileRun,
+  DEFAULT_PROFILES,
+  checkBudget,
+} from './index.js'
+
+describe('Cache correctness', () => {
+  it('MemoryCache stores and retrieves values', () => {
+    const cache = new MemoryCache<string>()
+    const key = { hash: 'abc', version: '1' }
+    cache.set(key, 'hello')
+    assert.strictEqual(cache.get(key), 'hello')
+    assert.strictEqual(cache.size(), 1)
+  })
+
+  it('MemoryCache returns undefined for missing keys', () => {
+    const cache = new MemoryCache<number>()
+    assert.strictEqual(cache.get({ hash: 'missing', version: '1' }), undefined)
+  })
+
+  it('MemoryCache invalidates entries', () => {
+    const cache = new MemoryCache<number>()
+    const key = { hash: 'del', version: '1' }
+    cache.set(key, 42)
+    assert.strictEqual(cache.invalidate(key), true)
+    assert.strictEqual(cache.get(key), undefined)
+    assert.strictEqual(cache.invalidate(key), false)
+  })
+
+  it('MemoryCache respects TTL expiration', async () => {
+    const cache = new MemoryCache<string>()
+    const key = { hash: 'ttl', version: '1' }
+    cache.set(key, 'temp', 10)
+    assert.strictEqual(cache.get(key), 'temp')
+    await new Promise(r => setTimeout(r, 20))
+    assert.strictEqual(cache.get(key), undefined)
+    assert.strictEqual(cache.size(), 0)
+  })
+
+  it('MemoryCache clear removes all entries', () => {
+    const cache = new MemoryCache<number>()
+    cache.set({ hash: 'a', version: '1' }, 1)
+    cache.set({ hash: 'b', version: '1' }, 2)
+    cache.clear()
+    assert.strictEqual(cache.size(), 0)
+  })
+
+  it('CompilationCache caches compile results by source', () => {
+    const inner = new MemoryCache<{ code: string }>()
+    const cache = new CompilationCache(inner, 'v1')
+    const result = { code: 'compiled' }
+    cache.set('source-a', result)
+    assert.deepStrictEqual(cache.get('source-a'), result)
+    assert.strictEqual(cache.get('source-b'), undefined)
+  })
+
+  it('CompilationCache invalidation targets specific source', () => {
+    const inner = new MemoryCache<number>()
+    const cache = new CompilationCache(inner, 'v1')
+    cache.set('src1', 1)
+    cache.set('src2', 2)
+    cache.invalidate('src1')
+    assert.strictEqual(cache.get('src1'), undefined)
+    assert.strictEqual(cache.get('src2'), 2)
+  })
+
+  it('ExtractionCache caches by selector and facts', () => {
+    const inner = new MemoryCache<{ rects: number[] }>()
+    const cache = new ExtractionCache(inner, 'v1')
+    const result = { rects: [0, 0, 10, 10] }
+    cache.set('#app', ['box', 'style'], result)
+    assert.deepStrictEqual(cache.get('#app', ['box', 'style']), result)
+    assert.strictEqual(cache.get('#app', ['box']), undefined)
+  })
+
+  it('ExtractionCache invalidation targets specific selector+facts', () => {
+    const inner = new MemoryCache<number>()
+    const cache = new ExtractionCache(inner, 'v1')
+    cache.set('sel1', ['a'], 1)
+    cache.set('sel1', ['b'], 2)
+    cache.set('sel2', ['a'], 3)
+    cache.invalidate('sel1', ['a'])
+    assert.strictEqual(cache.get('sel1', ['a']), undefined)
+    assert.strictEqual(cache.get('sel1', ['b']), 2)
+    assert.strictEqual(cache.get('sel2', ['a']), 3)
+  })
+
+  it('Cache version isolates entries', () => {
+    const cache = new MemoryCache<string>()
+    cache.set({ hash: 'x', version: 'v1' }, 'old')
+    cache.set({ hash: 'x', version: 'v2' }, 'new')
+    assert.strictEqual(cache.get({ hash: 'x', version: 'v1' }), 'old')
+    assert.strictEqual(cache.get({ hash: 'x', version: 'v2' }), 'new')
+  })
+})
+
+describe('Benchmark execution', () => {
+  it('profileRun measures duration and memory', async () => {
+    const { durationMs, memoryDeltaBytes, result } = await profileRun(() => {
+      const arr = new Array(1000).fill(0)
+      return arr.length
+    })
+    assert.strictEqual(typeof durationMs, 'number')
+    assert.strictEqual(durationMs >= 0, true)
+    assert.strictEqual(typeof memoryDeltaBytes, 'number')
+    assert.strictEqual(result, 1000)
+  })
+
+  it('runBenchmark returns aggregated stats', async () => {
+    let counter = 0
+    const result = await runBenchmark('inc', () => {
+      counter++
+      return counter
+    })
+    assert.strictEqual(result.name, 'inc')
+    assert.strictEqual(typeof result.meanDurationMs, 'number')
+    assert.strictEqual(typeof result.minDurationMs, 'number')
+    assert.strictEqual(typeof result.maxDurationMs, 'number')
+    assert.strictEqual(typeof result.stdDevDurationMs, 'number')
+    assert.strictEqual(typeof result.meanMemoryDeltaBytes, 'number')
+    assert.strictEqual(result.runs.length, 5)
+    assert.strictEqual(result.minDurationMs <= result.meanDurationMs, true)
+    assert.strictEqual(result.meanDurationMs <= result.maxDurationMs, true)
+  })
+
+  it('runBenchmark applies budget check', async () => {
+    const result = await runBenchmark('slow', () => {}, {
+      budgetName: 'compile',
+      profile: DEFAULT_PROFILES.benchmark,
+    })
+    assert.ok(result.budgetCheck)
+    assert.strictEqual(result.budgetCheck!.budgetName, 'compile')
+    assert.strictEqual(typeof result.budgetCheck!.passed, 'boolean')
+  })
+
+  it('runBenchmark uses custom run counts', async () => {
+    let calls = 0
+    await runBenchmark(
+      'count',
+      () => {
+        calls++
+      },
+      { warmupRuns: 2, measurementRuns: 3 }
+    )
+    assert.strictEqual(calls, 5)
+  })
+
+  it('BenchmarkSuite runs sequentially', async () => {
+    const suite = new BenchmarkSuite('seq')
+    const order: number[] = []
+    suite.add('a', () => order.push(1), { warmupRuns: 0, measurementRuns: 1 })
+    suite.add('b', () => order.push(2), { warmupRuns: 0, measurementRuns: 1 })
+    const result = await suite.runSequential()
+    assert.strictEqual(result.suiteName, 'seq')
+    assert.strictEqual(result.results.length, 2)
+    assert.deepStrictEqual(order, [1, 2])
+    assert.strictEqual(typeof result.allBudgetsPassed, 'boolean')
+    assert.strictEqual(typeof result.totalDurationMs, 'number')
+  })
+
+  it('BenchmarkSuite runs in parallel', async () => {
+    const suite = new BenchmarkSuite('par', {
+      poolOptions: { maxConcurrency: 2, taskTimeoutMs: 5000 },
+    })
+    suite.add('x', async () => {
+      await new Promise(r => setTimeout(r, 10))
+    }, { warmupRuns: 0, measurementRuns: 1 })
+    suite.add('y', async () => {
+      await new Promise(r => setTimeout(r, 10))
+    }, { warmupRuns: 0, measurementRuns: 1 })
+    const result = await suite.runParallel()
+    assert.strictEqual(result.results.length, 2)
+    assert.strictEqual(typeof result.totalDurationMs, 'number')
+    // Parallel total should be less than sequential sum of sleeps (20ms) plus generous overhead
+    assert.ok(result.totalDurationMs < 150)
+  })
+
+  it('Budget check passes when under budget', () => {
+    const check = checkBudget('compile', 10, undefined, DEFAULT_PROFILES.dev)
+    assert.strictEqual(check.passed, true)
+    assert.strictEqual(check.maxDurationMs, 50)
+  })
+
+  it('Budget check fails when over budget', () => {
+    const check = checkBudget('compile', 1000, undefined, DEFAULT_PROFILES.dev)
+    assert.strictEqual(check.passed, false)
+  })
+
+  it('Missing budget returns passed=true with Infinity', () => {
+    const check = checkBudget('unknown', 99999, undefined, DEFAULT_PROFILES.dev)
+    assert.strictEqual(check.passed, true)
+    assert.strictEqual(check.maxDurationMs, Infinity)
+  })
+})