fix: harden engine, enrich failure diagnostics, close adoption gaps

- P0: CLI verify now honors  test budget with seeded multi-sample
- P0: Observe sampling enforced via Math.random() gate in hook-validator
- P1: Remove misleading undici-mock-agent isolation option
- P1: Qualify reuses shared discoverRouteDetails() with warnings
- P1: Chaos/scenario config exposed via preset schema
- P1: README/docs limitations updated to current state
- P2: Nested response annotations prefer 2xx deterministically
- P2: --changed documented as heuristic in verify.md

- Add observe sink tests (sampling 0/1, sink failure non-interference)
- Add verify runs regression tests (scale, determinism, variants)
- Add configured-scenario qualify test (independent of OAuth fixture)
- Add coverageBreakdown to qualify artifacts (per-gate route coverage)
- Add production-style observe example with real sink in docs/observe.md
- Add nightly/staging vs PR gating guidance to docs/qualify.md

- Enrich VerifyFailure with formula-aware diagnostics:
  status:201 => 'HTTP 200', body field checks => actual values
- Remove stale observe CLI activation message
- Document outbound mocks as process-global in getting-started.md
- Refresh APOPHIS_ADOPTION_AUDIT.md with current state

903 tests pass, build clean, typecheck clean.
This commit is contained in:
John Dvorak
2026-05-21 20:39:36 -07:00
parent 55b0262799
commit d0523fcc2d
128 changed files with 4004 additions and 3631 deletions
+24 -9
View File
@@ -12,6 +12,7 @@
*/
import { convertSchema } from '../domain/schema-to-arbitrary.js'
import { SeededRng } from '../infrastructure/seeded-rng.js'
import { matchesTarget } from './wildcard-match.js'
import type { OutboundCallRecord, ResolvedOutboundContract } from '../types.js'
import * as fc from 'fast-check'
@@ -26,7 +27,7 @@ export interface OutboundMockRuntime {
/** Inject a specific response for the next call to a contract (for property testing) */
injectResponse(contractName: string, statusCode: number, body: unknown): void
}
interface OutboundMockOptions {
export interface OutboundMockOptions {
readonly contracts: ResolvedOutboundContract[]
readonly mode: 'example' | 'property'
readonly overrides?: Record<string, {
@@ -38,6 +39,14 @@ interface OutboundMockOptions {
readonly seed: number
/** Route-level behavioral contracts to constrain mock responses */
readonly routeEnsures?: readonly string[]
/** Runtime identifier for diagnostics */
readonly runtimeId?: string
}
let activeRuntimeId: string | undefined
export function getActiveMockRuntimeId(): string | undefined {
return activeRuntimeId
}
/** Resource store: contractName → resourceId → resourceBody */
type ResourceStore = Map<string, Map<string, unknown>>
@@ -174,9 +183,20 @@ export function createOutboundMockRuntime(opts: OutboundMockOptions): OutboundMo
return { statusCode: 200, body: generatedBody }
}
const install = (): void => {
if (activeRuntimeId !== undefined) {
throw new Error(
`OutboundMockRuntime already active (owner: ${activeRuntimeId}). ` +
'Only one outbound mock runtime can be installed at a time. ' +
'Restore the existing runtime first. ' +
'Consider running mock-dependent tests serially.'
)
}
if (originalFetch !== undefined) {
throw new Error('OutboundMockRuntime already installed')
}
if (opts.runtimeId) {
activeRuntimeId = opts.runtimeId
}
originalFetch = globalThis.fetch
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url
@@ -259,6 +279,9 @@ export function createOutboundMockRuntime(opts: OutboundMockOptions): OutboundMo
globalThis.fetch = originalFetch
originalFetch = undefined
}
if (opts.runtimeId && activeRuntimeId === opts.runtimeId) {
activeRuntimeId = undefined
}
}
const getCalls = (name?: string): ReadonlyArray<OutboundCallRecord> => {
if (name === undefined) return calls
@@ -276,11 +299,3 @@ export function createOutboundMockRuntime(opts: OutboundMockOptions): OutboundMo
}
return { install, restore, getCalls, getResource, clear, injectResponse }
}
function matchesTarget(url: string, target: string): boolean {
if (target === url) return true
if (target.includes('*')) {
const regex = new RegExp('^' + target.replace(/\*/g, '.*') + '$')
return regex.test(url)
}
return url.includes(target)
}