apophis/apophis-fastify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden engine, enrich failure diagnostics, close adoption gaps

Browse Source 
- P0: CLI verify now honors  test budget with seeded multi-sample
- P0: Observe sampling enforced via Math.random() gate in hook-validator
- P1: Remove misleading undici-mock-agent isolation option
- P1: Qualify reuses shared discoverRouteDetails() with warnings
- P1: Chaos/scenario config exposed via preset schema
- P1: README/docs limitations updated to current state
- P2: Nested response annotations prefer 2xx deterministically
- P2: --changed documented as heuristic in verify.md

- Add observe sink tests (sampling 0/1, sink failure non-interference)
- Add verify runs regression tests (scale, determinism, variants)
- Add configured-scenario qualify test (independent of OAuth fixture)
- Add coverageBreakdown to qualify artifacts (per-gate route coverage)
- Add production-style observe example with real sink in docs/observe.md
- Add nightly/staging vs PR gating guidance to docs/qualify.md

- Enrich VerifyFailure with formula-aware diagnostics:
  status:201 => 'HTTP 200', body field checks => actual values
- Remove stale observe CLI activation message
- Document outbound mocks as process-global in getting-started.md
- Refresh APOPHIS_ADOPTION_AUDIT.md with current state

903 tests pass, build clean, typecheck clean.
This commit is contained in:
John Dvorak
2026-05-21 20:39:36 -07:00
parent 55b0262799
commit d0523fcc2d
128 changed files with 4004 additions and 3631 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/commands/verify/runner.ts
+129 -29
View File
@@ -20,6 +20,10 @@ import { executeHttp } from '../../../infrastructure/http-executor.js'
import { parse } from '../../../formula/parser.js'
import { evaluateAsync } from '../../../formula/evaluator.js'
import { createOperationResolver } from '../../../formula/runtime.js'
import { buildRequest } from '../../../domain/request-builder.js'
import { convertSchema } from '../../../domain/schema-to-arbitrary.js'
import * as fc from 'fast-check'
import { resolveRuns } from '../../../types.js'
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
import type { RouteResult } from '../../core/types.js'
@@ -33,6 +37,8 @@ export interface VerifyFailure {
expected: string
observed: string
artifactPath?: string
formula?: string
category?: string
}
export interface VerifyRunResult {
@@ -48,6 +54,8 @@ export interface VerifyRunResult {
noRelevantChanges?: boolean
availableRoutes?: string[]
artifactPaths: string[]
discoveryWarnings?: string[]
runs: number
}
export interface VerifyRunnerDeps {
@@ -57,6 +65,7 @@ export interface VerifyRunnerDeps {
routeFilters?: string[]
changed?: boolean
profileRoutes?: string[]
runs?: number
}
// ---------------------------------------------------------------------------
@@ -203,28 +212,58 @@ async function filterChangedRoutes(
/**
* Build a request for a route.
*/
function buildRouteRequest(route: RouteContract): {
function buildRouteRequest(
route: RouteContract,
seed?: number,
runIndex?: number,
variant?: { name: string; headers?: Record<string, string> },
): {
method: string
url: string
body?: unknown
query?: Record<string, string>
headers: Record<string, string>
} {
const headers: Record<string, string> = {
'content-type': 'application/json',
let generatedData: Record<string, unknown> = {}
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
if (bodySchema && seed !== undefined) {
try {
const bodyArb = convertSchema(bodySchema, { context: 'request' })
const bodySeed = seed + (runIndex ?? 0) * 31
const samples = fc.sample(bodyArb, { numRuns: 1, seed: bodySeed })
const bodySample = samples[0]
if (bodySample !== null && typeof bodySample === 'object') {
generatedData = bodySample as Record<string, unknown>
}
} catch {
// fall through to example-based generation
}
}
// Build body from schema if available
let body: unknown = undefined
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
if (bodySchema && route.method === 'POST') {
body = buildExampleBody(bodySchema)
if (!generatedData || Object.keys(generatedData).length === 0) {
generatedData = buildExampleBody(bodySchema ?? {}) as Record<string, unknown> ?? {}
}
const request = buildRequest(
route,
generatedData,
variant?.headers ?? ({} as Record<string, string>),
{ resources: new Map(), counters: new Map() },
)
const headers: Record<string, string> = {
'content-type': 'application/json',
...request.headers,
...(variant?.headers ?? {}),
}
return {
method: route.method,
url: route.path,
body,
method: request.method,
url: request.url,
body: request.body,
headers,
query: request.query,
}
}
@@ -268,18 +307,62 @@ function buildExampleValue(schema: Record<string, unknown>): unknown {
return undefined
}
/**
* Execute a single contract for a route.
* Returns the evaluation context and any failure.
*/
function buildFailureDiagnostic(
contract: string,
evalCtx: EvalContext,
evalError?: string,
): Pick<VerifyFailure, 'expected' | 'observed' | 'formula' | 'category'> {
if (evalError) {
return { expected: 'true', observed: evalError, formula: contract, category: 'runtime' }
}
const status = evalCtx.response.statusCode
const statusMatch = contract.match(/^status:(\d+)$/i)
if (statusMatch) {
return { expected: `HTTP ${statusMatch[1]}`, observed: `HTTP ${status}`, formula: contract, category: 'runtime' }
}
const fieldMatch = contract.match(/response_body\(this\)\.([\w.]+)\s*(!=|==)\s*null/i)
if (fieldMatch) {
const path = fieldMatch[1]!
const negated = fieldMatch[2] === '!='
const parts = path.split('.')
let value: unknown = evalCtx.response.body
for (const p of parts) {
if (value != null && typeof value === 'object') {
value = (value as Record<string, unknown>)[p]
} else { value = undefined; break }
}
const actual = value === undefined ? 'undefined' : value === null ? 'null' : `"${String(value)}"`
return {
expected: negated ? `${path} != null` : `${path} == null`,
observed: `${path} is ${actual}`,
formula: contract, category: 'runtime',
}
}
const codeMatch = contract.match(/response_code\(this\)\s*(==|!=|<|>|<=|>=)\s*(\d+)/i)
if (codeMatch) {
return {
expected: `response_code == ${codeMatch[2]}`,
observed: `response_code is ${status}`,
formula: contract, category: 'runtime',
}
}
return { expected: contract, observed: `false`, formula: contract, category: 'runtime' }
}
async function executeContract(
fastify: FastifyInjectInstance,
route: RouteContract,
contract: string,
timeout?: number,
variant?: { name: string; headers?: Record<string, string> },
seed?: number,
runIndex?: number,
): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> {
const request = buildRouteRequest(route)
const request = buildRouteRequest(route, seed, runIndex, variant)
// Merge variant headers if provided
const headers = variant?.headers
@@ -307,6 +390,7 @@ async function executeContract(
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
const diagnostic = buildFailureDiagnostic(contract, evalCtx, result.success ? undefined : result.error)
return {
ctx: evalCtx,
failure: {
@@ -314,14 +398,17 @@ async function executeContract(
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: result.success ? String(result.value) : result.error,
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
},
}
}
return { ctx: evalCtx }
} catch (error) {
const diagnostic = buildFailureDiagnostic(contract, evalCtx, error instanceof Error ? error.message : String(error))
return {
ctx: evalCtx,
failure: {
@@ -329,8 +416,10 @@ async function executeContract(
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: error instanceof Error ? error.message : String(error),
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
},
}
}
@@ -397,9 +486,10 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noRoutesMatched: false,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
notGitRepo: true,
}
artifactPaths: [],
notGitRepo: true,
runs: 0,
}
}
routes = await filterChangedRoutes(routes, cwd)
}
@@ -417,6 +507,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noContractsFound: false,
availableRoutes,
artifactPaths: [],
runs: 0,
}
}
@@ -437,10 +528,13 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noContractsFound: true,
availableRoutes,
artifactPaths: [],
runs: 0,
}
}
// 4. Execute contracts (with variant expansion)
const runConfig = resolveRuns(deps.runs)
const runs = runConfig.contractRuns
const failures: VerifyFailure[] = []
let total = 0
let passedCount = 0
@@ -453,13 +547,18 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
for (const variant of variants) {
for (const contract of contracts) {
total++
const result = await executeContract(fastify, route, contract, deps.timeout, variant)
for (let runIndex = 0; runIndex < runs; runIndex++) {
total++
const result = await executeContract(
fastify, route, contract, deps.timeout,
variant, deps.seed, runIndex,
)
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
}
}
}
}
@@ -485,5 +584,6 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noContractsFound: false,
availableRoutes,
artifactPaths: [],
runs,
}
}