fix: harden engine, enrich failure diagnostics, close adoption gaps
- P0: CLI verify now honors test budget with seeded multi-sample - P0: Observe sampling enforced via Math.random() gate in hook-validator - P1: Remove misleading undici-mock-agent isolation option - P1: Qualify reuses shared discoverRouteDetails() with warnings - P1: Chaos/scenario config exposed via preset schema - P1: README/docs limitations updated to current state - P2: Nested response annotations prefer 2xx deterministically - P2: --changed documented as heuristic in verify.md - Add observe sink tests (sampling 0/1, sink failure non-interference) - Add verify runs regression tests (scale, determinism, variants) - Add configured-scenario qualify test (independent of OAuth fixture) - Add coverageBreakdown to qualify artifacts (per-gate route coverage) - Add production-style observe example with real sink in docs/observe.md - Add nightly/staging vs PR gating guidance to docs/qualify.md - Enrich VerifyFailure with formula-aware diagnostics: status:201 => 'HTTP 200', body field checks => actual values - Remove stale observe CLI activation message - Document outbound mocks as process-global in getting-started.md - Refresh APOPHIS_ADOPTION_AUDIT.md with current state 903 tests pass, build clean, typecheck clean.
This commit is contained in:
John Dvorak
@@ -37,7 +37,6 @@ import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../..
|
||||
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
|
||||
import type { OutputContext } from '../../renderers/shared.js'
|
||||
import { resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
|
||||
|
||||
@@ -98,24 +97,11 @@ export function generateSeed(): number {
|
||||
|
||||
/**
|
||||
* Discover routes from the Fastify app for chaos execution.
|
||||
* Injected fastify instance must have routes registered.
|
||||
* Uses the shared discovery module for capture + fallback support.
|
||||
*/
|
||||
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
|
||||
// Cast to access routes
|
||||
const app = fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }> }
|
||||
if (!app.routes) return []
|
||||
|
||||
return app.routes.map(r => ({
|
||||
path: r.url,
|
||||
method: r.method as RouteContract['method'],
|
||||
category: 'observer',
|
||||
requires: [],
|
||||
ensures: [],
|
||||
invariants: [],
|
||||
regexPatterns: {},
|
||||
validateRuntime: false,
|
||||
schema: r.schema,
|
||||
}))
|
||||
const { discoverRoutes } = await import('../../../domain/discovery.js')
|
||||
return discoverRoutes(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -288,12 +274,51 @@ export function buildArtifact(
|
||||
// Build execution summary from runner result
|
||||
const executionSummary = runResult.executionSummary
|
||||
|
||||
// Build per-gate coverage breakdown for clear artifact interpretation
|
||||
const scenarioRoutes = [...new Set(
|
||||
runResult.stepTraces
|
||||
.filter(t => t.status === 'passed' || t.status === 'failed')
|
||||
.map(t => t.route)
|
||||
.filter(Boolean)
|
||||
)]
|
||||
const statefulRoutes = runResult.statefulResult
|
||||
? [...new Set(runResult.statefulResult.tests.map(t => normalizeRouteIdentity(t.name)))]
|
||||
: []
|
||||
const chaosRoutesPlanned: string[] = []
|
||||
const chaosRoutesExecuted: string[] = []
|
||||
for (const r of runResult.chaosResults ?? []) {
|
||||
if (r.applied) chaosRoutesExecuted.push(r.route)
|
||||
}
|
||||
// Infer planned from chaos results or executionSummary
|
||||
if (executionSummary.chaosRoutesPlanned > 0 && chaosRoutesExecuted.length > 0) {
|
||||
chaosRoutesPlanned.push(...chaosRoutesExecuted)
|
||||
}
|
||||
|
||||
const coverageBreakdown = {
|
||||
scenario: {
|
||||
routesCovered: scenarioRoutes,
|
||||
stepsTotal: executionSummary.totalSteps,
|
||||
stepsPassed: runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0),
|
||||
},
|
||||
stateful: {
|
||||
routesCovered: statefulRoutes,
|
||||
testsTotal: executionSummary.statefulTestsRun,
|
||||
testsPassed: runResult.statefulResult?.summary.passed ?? 0,
|
||||
},
|
||||
chaos: {
|
||||
routesPlanned: chaosRoutesPlanned,
|
||||
routesExecuted: chaosRoutesExecuted,
|
||||
runsTotal: executionSummary.chaosRunsRun,
|
||||
runsPassed: (runResult.chaosResults ?? []).filter(r => r.contractsPassed).length,
|
||||
},
|
||||
}
|
||||
|
||||
// Build profile gates from the result context
|
||||
// We need to pass gates through or infer from results
|
||||
const profileGates = {
|
||||
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
|
||||
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
|
||||
chaos: (runResult.chaosResult !== undefined) || executionSummary.chaosRunsRun > 0,
|
||||
chaos: executionSummary.chaosRoutesPlanned > 0,
|
||||
}
|
||||
|
||||
// Deterministic parameters for audit
|
||||
@@ -320,6 +345,7 @@ export function buildArtifact(
|
||||
failed: failures.length,
|
||||
},
|
||||
executionSummary,
|
||||
coverageBreakdown,
|
||||
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
|
||||
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
|
||||
route: sr.route,
|
||||
@@ -386,117 +412,6 @@ async function emitArtifact(
|
||||
// Output formatting
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function formatHumanOutput(
|
||||
result: QualifyRunResult,
|
||||
options: { profile?: string; seed: number; env: string },
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
|
||||
lines.push(`Qualify run for environment "${options.env}"`)
|
||||
if (options.profile) {
|
||||
lines.push(`Profile: ${options.profile}`)
|
||||
}
|
||||
lines.push(`Seed: ${options.seed}`)
|
||||
lines.push('')
|
||||
|
||||
// Scenario results
|
||||
for (const scenario of result.scenarioResults) {
|
||||
lines.push(`Scenario: ${scenario.name}`)
|
||||
for (const step of scenario.steps) {
|
||||
const icon = step.ok ? '✓' : '✗'
|
||||
lines.push(` ${icon} ${step.name} (${step.statusCode ?? 'no-status'})`)
|
||||
if (!step.ok && step.diagnostics) {
|
||||
lines.push(` Expected: ${step.diagnostics.expected || 'success'}`)
|
||||
lines.push(` Observed: ${step.diagnostics.error || 'failure'}`)
|
||||
if (step.diagnostics.actual) {
|
||||
lines.push(` Actual: ${step.diagnostics.actual}`)
|
||||
}
|
||||
if (step.diagnostics.diff) {
|
||||
lines.push(` Diff:`)
|
||||
for (const line of String(step.diagnostics.diff).split('\n')) {
|
||||
lines.push(` ${line}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Stateful results
|
||||
if (result.statefulResult) {
|
||||
lines.push(`Stateful: ${result.statefulResult.summary.passed} passed, ${result.statefulResult.summary.failed} failed`)
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Chaos results
|
||||
if (result.chaosResult) {
|
||||
lines.push(`Chaos: ${result.chaosResult.applied ? 'applied' : 'none'}`)
|
||||
if (result.chaosResult.events.length > 0) {
|
||||
for (const event of result.chaosResult.events) {
|
||||
lines.push(` ${event}`)
|
||||
}
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Step traces
|
||||
if (result.stepTraces.length > 0) {
|
||||
lines.push('Step traces:')
|
||||
for (const trace of result.stepTraces.slice(0, 20)) {
|
||||
const icon = trace.status === 'passed' ? '✓' : trace.status === 'skipped' ? '⊘' : '✗'
|
||||
lines.push(` ${icon} ${trace.name} (${trace.durationMs}ms)`)
|
||||
}
|
||||
if (result.stepTraces.length > 20) {
|
||||
lines.push(` ... and ${result.stepTraces.length - 20} more`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Cleanup failures
|
||||
if (result.cleanupFailures.length > 0) {
|
||||
lines.push('Cleanup failures (reported separately):')
|
||||
for (const cf of result.cleanupFailures) {
|
||||
lines.push(` ⚠ ${cf.resource}: ${cf.error}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Per-profile gate execution counts
|
||||
lines.push('Profile gate execution counts:')
|
||||
lines.push(` Scenario: ${result.executionSummary.scenariosRun} run`)
|
||||
lines.push(` Stateful: ${result.executionSummary.statefulTestsRun} tests run`)
|
||||
lines.push(` Chaos: ${result.executionSummary.chaosRunsRun} runs run`)
|
||||
lines.push('')
|
||||
|
||||
// Executed routes
|
||||
if (result.executedRoutes.length > 0) {
|
||||
lines.push(`Executed routes (${result.executedRoutes.length}):`)
|
||||
for (const route of result.executedRoutes) {
|
||||
lines.push(` ${route}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Skipped routes
|
||||
if (result.skippedRoutes.length > 0) {
|
||||
lines.push(`Skipped routes (${result.skippedRoutes.length}):`)
|
||||
for (const sr of result.skippedRoutes) {
|
||||
lines.push(` ${sr.route}: ${sr.reason}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Summary
|
||||
if (result.passed) {
|
||||
lines.push('All qualifications passed.')
|
||||
} else {
|
||||
lines.push('Qualification failed.')
|
||||
lines.push(`Replay: apophis replay --artifact <artifact-path>`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main command handler
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -583,9 +498,26 @@ export async function qualifyCommand(
|
||||
const profileDef = profile ? config.profiles?.[profile] : undefined
|
||||
const gates = resolveProfileGates(profileDef?.features)
|
||||
|
||||
// 5. Build scenario configs from profile routes
|
||||
// 5. Build scenario configs from profile routes and config
|
||||
const routes = profileDef?.routes ?? []
|
||||
const scenarios = buildScenarioConfigs(routes, seed)
|
||||
const fixtureScenarios = buildScenarioConfigs(routes, seed)
|
||||
|
||||
const configScenarios: ScenarioConfig[] = (config.scenarios ?? []).map(s => ({
|
||||
name: s.name,
|
||||
steps: s.steps.map(step => ({
|
||||
name: step.name,
|
||||
request: {
|
||||
method: step.request.method as ScenarioConfig['steps'][0]['request']['method'],
|
||||
url: step.request.url,
|
||||
body: step.request.body,
|
||||
headers: step.request.headers,
|
||||
},
|
||||
expect: step.expect,
|
||||
capture: step.capture,
|
||||
})),
|
||||
}))
|
||||
|
||||
const scenarios = [...fixtureScenarios, ...configScenarios]
|
||||
|
||||
// 6. Build stateful config
|
||||
const presetName = profileDef?.preset
|
||||
@@ -603,6 +535,9 @@ export async function qualifyCommand(
|
||||
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
|
||||
? {
|
||||
probability: 0.5,
|
||||
strategy: preset.chaosStrategy as ChaosConfig['strategy'],
|
||||
sampleSize: preset.chaosSampleSize,
|
||||
sampleRoutes: preset.chaosSampleRoutes,
|
||||
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
|
||||
error: { probability: 0.2, statusCode: 503 },
|
||||
dropout: { probability: 0.2, statusCode: 504 },
|
||||
@@ -611,32 +546,41 @@ export async function qualifyCommand(
|
||||
: undefined
|
||||
|
||||
// 8. Load the Fastify app for execution
|
||||
// Try to import the app from the fixture
|
||||
let fastify: FastifyAppLike | undefined
|
||||
let appEntrypoint: string | undefined
|
||||
try {
|
||||
const appPath = resolve(workingDir, 'app.js')
|
||||
const appUrl = pathToFileURL(appPath)
|
||||
appUrl.searchParams.set('apophisRun', String(Date.now()))
|
||||
const appModule = await import(appUrl.href)
|
||||
fastify = (appModule.default || appModule) as FastifyAppLike
|
||||
const { loadApp } = await import('../../core/app-loader.js')
|
||||
const loaded = await loadApp(workingDir)
|
||||
fastify = loaded.fastify as FastifyAppLike
|
||||
appEntrypoint = loaded.entrypoint
|
||||
if (fastify && typeof fastify.ready === 'function') {
|
||||
await fastify.ready()
|
||||
}
|
||||
} catch (err) {
|
||||
// App not available — return a result indicating no app to test
|
||||
if (process.env.APOPHIS_DEBUG === '1') {
|
||||
console.error('Failed to load app:', err)
|
||||
}
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: 'No Fastify app found. Ensure app.js exports a Fastify instance or a factory function.\n\nSupported patterns:\n export default app\n export const createApp = () => app\n module.exports = app',
|
||||
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// 9. Discover routes for chaos
|
||||
// 9. Discover routes for chaos (with discovery metadata)
|
||||
const appRoutes = await discoverAppRoutes(fastify)
|
||||
|
||||
// Collect discovery warnings
|
||||
const { discoverRouteDetails } = await import('../../../domain/discovery.js')
|
||||
const discoveryResult = discoverRouteDetails(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
|
||||
const discoveryWarnings: string[] = [
|
||||
...discoveryResult.warnings,
|
||||
]
|
||||
if (discoveryResult.source === 'print-routes' && !discoveryResult.hasSchemaMetadata) {
|
||||
discoveryWarnings.push(
|
||||
'Routes were discovered without schema metadata. Behavioral contracts and chaos injection may be limited. ' +
|
||||
'Register APOPHIS before defining routes or use createFastify().'
|
||||
)
|
||||
}
|
||||
|
||||
// 10. Run qualify execution
|
||||
const deps = {
|
||||
fastify: fastify as any,
|
||||
@@ -658,7 +602,7 @@ export async function qualifyCommand(
|
||||
|
||||
// 12. Signal quality guardrails — fail if zero checks executed
|
||||
const execSummary = runResult.executionSummary
|
||||
const warnings: string[] = [...artifact.warnings]
|
||||
const warnings: string[] = [...artifact.warnings, ...discoveryWarnings]
|
||||
|
||||
if (execSummary.totalExecuted === 0) {
|
||||
await emitArtifact(artifact, {
|
||||
|
||||
Reference in New Issue
Block a user