Imhotep/packages/imhotep-bench/src/bench.test.ts

// bench.test.ts - Tests for imhotep-bench harness
// Validates benchmark execution and cache correctness.

import { describe, it } from 'node:test'
import assert from 'node:assert'
import {
  MemoryCache,
  CompilationCache,
  ExtractionCache,
  runBenchmark,
  BenchmarkSuite,
  profileRun,
  DEFAULT_PROFILES,
  checkBudget,
} from './index.js'

describe('Cache correctness', () => {
  it('MemoryCache stores and retrieves values', () => {
    const cache = new MemoryCache<string>()
    const key = { hash: 'abc', version: '1' }
    cache.set(key, 'hello')
    assert.strictEqual(cache.get(key), 'hello')
    assert.strictEqual(cache.size(), 1)
  })

  it('MemoryCache returns undefined for missing keys', () => {
    const cache = new MemoryCache<number>()
    assert.strictEqual(cache.get({ hash: 'missing', version: '1' }), undefined)
  })

  it('MemoryCache invalidates entries', () => {
    const cache = new MemoryCache<number>()
    const key = { hash: 'del', version: '1' }
    cache.set(key, 42)
    assert.strictEqual(cache.invalidate(key), true)
    assert.strictEqual(cache.get(key), undefined)
    assert.strictEqual(cache.invalidate(key), false)
  })

  it('MemoryCache respects TTL expiration', async () => {
    const cache = new MemoryCache<string>()
    const key = { hash: 'ttl', version: '1' }
    cache.set(key, 'temp', 10)
    assert.strictEqual(cache.get(key), 'temp')
    await new Promise(r => setTimeout(r, 20))
    assert.strictEqual(cache.get(key), undefined)
    assert.strictEqual(cache.size(), 0)
  })

  it('MemoryCache clear removes all entries', () => {
    const cache = new MemoryCache<number>()
    cache.set({ hash: 'a', version: '1' }, 1)
    cache.set({ hash: 'b', version: '1' }, 2)
    cache.clear()
    assert.strictEqual(cache.size(), 0)
  })

  it('CompilationCache caches compile results by source', () => {
    const inner = new MemoryCache<{ code: string }>()
    const cache = new CompilationCache(inner, 'v1')
    const result = { code: 'compiled' }
    cache.set('source-a', result)
    assert.deepStrictEqual(cache.get('source-a'), result)
    assert.strictEqual(cache.get('source-b'), undefined)
  })

  it('CompilationCache invalidation targets specific source', () => {
    const inner = new MemoryCache<number>()
    const cache = new CompilationCache(inner, 'v1')
    cache.set('src1', 1)
    cache.set('src2', 2)
    cache.invalidate('src1')
    assert.strictEqual(cache.get('src1'), undefined)
    assert.strictEqual(cache.get('src2'), 2)
  })

  it('ExtractionCache caches by selector and facts', () => {
    const inner = new MemoryCache<{ rects: number[] }>()
    const cache = new ExtractionCache(inner, 'v1')
    const result = { rects: [0, 0, 10, 10] }
    cache.set('#app', ['box', 'style'], result)
    assert.deepStrictEqual(cache.get('#app', ['box', 'style']), result)
    assert.strictEqual(cache.get('#app', ['box']), undefined)
  })

  it('ExtractionCache invalidation targets specific selector+facts', () => {
    const inner = new MemoryCache<number>()
    const cache = new ExtractionCache(inner, 'v1')
    cache.set('sel1', ['a'], 1)
    cache.set('sel1', ['b'], 2)
    cache.set('sel2', ['a'], 3)
    cache.invalidate('sel1', ['a'])
    assert.strictEqual(cache.get('sel1', ['a']), undefined)
    assert.strictEqual(cache.get('sel1', ['b']), 2)
    assert.strictEqual(cache.get('sel2', ['a']), 3)
  })

  it('Cache version isolates entries', () => {
    const cache = new MemoryCache<string>()
    cache.set({ hash: 'x', version: 'v1' }, 'old')
    cache.set({ hash: 'x', version: 'v2' }, 'new')
    assert.strictEqual(cache.get({ hash: 'x', version: 'v1' }), 'old')
    assert.strictEqual(cache.get({ hash: 'x', version: 'v2' }), 'new')
  })
})

describe('Benchmark execution', () => {
  it('profileRun measures duration and memory', async () => {
    const { durationMs, memoryDeltaBytes, result } = await profileRun(() => {
      const arr = new Array(1000).fill(0)
      return arr.length
    })
    assert.strictEqual(typeof durationMs, 'number')
    assert.strictEqual(durationMs >= 0, true)
    assert.strictEqual(typeof memoryDeltaBytes, 'number')
    assert.strictEqual(result, 1000)
  })

  it('runBenchmark returns aggregated stats', async () => {
    let counter = 0
    const result = await runBenchmark('inc', () => {
      counter++
      return counter
    })
    assert.strictEqual(result.name, 'inc')
    assert.strictEqual(typeof result.meanDurationMs, 'number')
    assert.strictEqual(typeof result.minDurationMs, 'number')
    assert.strictEqual(typeof result.maxDurationMs, 'number')
    assert.strictEqual(typeof result.stdDevDurationMs, 'number')
    assert.strictEqual(typeof result.meanMemoryDeltaBytes, 'number')
    assert.strictEqual(result.runs.length, 5)
    assert.strictEqual(result.minDurationMs <= result.meanDurationMs, true)
    assert.strictEqual(result.meanDurationMs <= result.maxDurationMs, true)
  })

  it('runBenchmark applies budget check', async () => {
    const result = await runBenchmark('slow', () => {}, {
      budgetName: 'compile',
      profile: DEFAULT_PROFILES.benchmark,
    })
    assert.ok(result.budgetCheck)
    assert.strictEqual(result.budgetCheck!.budgetName, 'compile')
    assert.strictEqual(typeof result.budgetCheck!.passed, 'boolean')
  })

  it('runBenchmark uses custom run counts', async () => {
    let calls = 0
    await runBenchmark(
      'count',
      () => {
        calls++
      },
      { warmupRuns: 2, measurementRuns: 3 }
    )
    assert.strictEqual(calls, 5)
  })

  it('BenchmarkSuite runs sequentially', async () => {
    const suite = new BenchmarkSuite('seq')
    const order: number[] = []
    suite.add('a', () => order.push(1), { warmupRuns: 0, measurementRuns: 1 })
    suite.add('b', () => order.push(2), { warmupRuns: 0, measurementRuns: 1 })
    const result = await suite.runSequential()
    assert.strictEqual(result.suiteName, 'seq')
    assert.strictEqual(result.results.length, 2)
    assert.deepStrictEqual(order, [1, 2])
    assert.strictEqual(typeof result.allBudgetsPassed, 'boolean')
    assert.strictEqual(typeof result.totalDurationMs, 'number')
  })

  it('BenchmarkSuite runs in parallel', async () => {
    const suite = new BenchmarkSuite('par', {
      poolOptions: { maxConcurrency: 2, taskTimeoutMs: 5000 },
    })
    suite.add('x', async () => {
      await new Promise(r => setTimeout(r, 10))
    }, { warmupRuns: 0, measurementRuns: 1 })
    suite.add('y', async () => {
      await new Promise(r => setTimeout(r, 10))
    }, { warmupRuns: 0, measurementRuns: 1 })
    const result = await suite.runParallel()
    assert.strictEqual(result.results.length, 2)
    assert.strictEqual(typeof result.totalDurationMs, 'number')
    // Parallel total should be less than sequential sum of sleeps (20ms) plus generous overhead
    assert.ok(result.totalDurationMs < 150)
  })

  it('Budget check passes when under budget', () => {
    const check = checkBudget('compile', 10, undefined, DEFAULT_PROFILES.dev)
    assert.strictEqual(check.passed, true)
    assert.strictEqual(check.maxDurationMs, 50)
  })

  it('Budget check fails when over budget', () => {
    const check = checkBudget('compile', 1000, undefined, DEFAULT_PROFILES.dev)
    assert.strictEqual(check.passed, false)
  })

  it('Missing budget returns passed=true with Infinity', () => {
    const check = checkBudget('unknown', 99999, undefined, DEFAULT_PROFILES.dev)
    assert.strictEqual(check.passed, true)
    assert.strictEqual(check.maxDurationMs, Infinity)
  })
})