fix: harden engine, enrich failure diagnostics, close adoption gaps
- P0: CLI verify now honors test budget with seeded multi-sample - P0: Observe sampling enforced via Math.random() gate in hook-validator - P1: Remove misleading undici-mock-agent isolation option - P1: Qualify reuses shared discoverRouteDetails() with warnings - P1: Chaos/scenario config exposed via preset schema - P1: README/docs limitations updated to current state - P2: Nested response annotations prefer 2xx deterministically - P2: --changed documented as heuristic in verify.md - Add observe sink tests (sampling 0/1, sink failure non-interference) - Add verify runs regression tests (scale, determinism, variants) - Add configured-scenario qualify test (independent of OAuth fixture) - Add coverageBreakdown to qualify artifacts (per-gate route coverage) - Add production-style observe example with real sink in docs/observe.md - Add nightly/staging vs PR gating guidance to docs/qualify.md - Enrich VerifyFailure with formula-aware diagnostics: status:201 => 'HTTP 200', body field checks => actual values - Remove stale observe CLI activation message - Document outbound mocks as process-global in getting-started.md - Refresh APOPHIS_ADOPTION_AUDIT.md with current state 903 tests pass, build clean, typecheck clean.
This commit is contained in:
John Dvorak
@@ -117,20 +117,26 @@ function buildArtifact(
|
||||
}
|
||||
return {
|
||||
route,
|
||||
contract: f.contract,
|
||||
contract: f.formula || f.contract,
|
||||
expected: f.expected,
|
||||
observed: f.observed,
|
||||
seed: options.seed,
|
||||
replayCommand: `apophis replay --artifact ${f.artifactPath || '<artifact-path-unavailable>'}`,
|
||||
category: f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME,
|
||||
category: f.category ?? (f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME),
|
||||
}
|
||||
})
|
||||
|
||||
if (runResult.noContractsFound) {
|
||||
warnings.push('No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.')
|
||||
if (runResult.discoveryWarnings && runResult.discoveryWarnings.length > 0) {
|
||||
warnings.push(...runResult.discoveryWarnings)
|
||||
}
|
||||
}
|
||||
if (runResult.noRoutesMatched) {
|
||||
warnings.push(`No routes matched the filter. Available routes: ${runResult.availableRoutes?.join(', ') || 'none'}`)
|
||||
if (runResult.discoveryWarnings && runResult.discoveryWarnings.length > 0) {
|
||||
warnings.push(...runResult.discoveryWarnings)
|
||||
}
|
||||
}
|
||||
if (runResult.notGitRepo) {
|
||||
warnings.push('--changed requires a git repository. Current directory is not inside a git repo.')
|
||||
@@ -149,7 +155,7 @@ function buildArtifact(
|
||||
|
||||
return {
|
||||
version: 'apophis-artifact/1',
|
||||
cliVersion: '2.0.0',
|
||||
cliVersion: '2.7.0',
|
||||
command: 'verify',
|
||||
mode: 'verify',
|
||||
cwd: options.cwd,
|
||||
@@ -255,106 +261,6 @@ function formatHumanFailure(failure: FailureRecord, profile?: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Format human-readable output for verify results.
|
||||
*/
|
||||
function formatHumanOutput(
|
||||
runResult: VerifyRunResult,
|
||||
options: { profile?: string; seed: number; env: string; routeFilters?: string[] },
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
|
||||
if (runResult.notGitRepo) {
|
||||
lines.push(`--changed requires a git repository.`)
|
||||
lines.push(`Current directory is not inside a git repo.`)
|
||||
lines.push('')
|
||||
lines.push('Next:')
|
||||
lines.push(` Initialize git with \`git init\`, or run verify without --changed.`)
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
if (runResult.noRelevantChanges) {
|
||||
lines.push(`No relevant changes detected.`)
|
||||
lines.push(`Git shows no modified files that match any route.`)
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
if (runResult.noRoutesMatched) {
|
||||
lines.push(`No routes matched the filter.`)
|
||||
lines.push(`Filters applied: ${options.routeFilters?.join(', ') || 'none'}`)
|
||||
lines.push(`Available routes:`)
|
||||
for (const r of runResult.availableRoutes || []) {
|
||||
lines.push(` ${r}`)
|
||||
}
|
||||
lines.push('')
|
||||
lines.push('Next:')
|
||||
lines.push(` Adjust --routes filter or add routes to your app.`)
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
if (runResult.noContractsFound) {
|
||||
lines.push('No behavioral contracts found.')
|
||||
lines.push('')
|
||||
lines.push('APOPHIS discovered routes, but none have behavioral contracts.')
|
||||
lines.push('Schema-only routes (with response schemas) are not enough.')
|
||||
lines.push('You must add x-ensures or x-requires clauses that check behavior.')
|
||||
lines.push('')
|
||||
lines.push('Example — add this to your route schema:')
|
||||
lines.push(' "x-ensures": [')
|
||||
lines.push(' "response_code(GET /users/{response_body(this).id}) == 200"')
|
||||
lines.push(' ]')
|
||||
lines.push('')
|
||||
lines.push('Next steps:')
|
||||
lines.push(' 1. Open your route file (e.g., app.js or src/routes/users.js)')
|
||||
lines.push(' 2. Find the route you want to test')
|
||||
lines.push(' 3. Add an "x-ensures" array inside the schema object')
|
||||
lines.push(' 4. Run: apophis verify --profile quick --routes "POST /users"')
|
||||
lines.push('')
|
||||
lines.push('For more examples, see docs/getting-started.md')
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
// Print failures using canonical format
|
||||
for (const failure of runResult.failures) {
|
||||
const failureRecord: FailureRecord = {
|
||||
route: failure.route,
|
||||
contract: failure.contract,
|
||||
expected: failure.expected,
|
||||
observed: failure.observed,
|
||||
seed: options.seed,
|
||||
replayCommand: `apophis replay --artifact ${failure.artifactPath || 'reports/apophis/failure-*.json'}`,
|
||||
}
|
||||
lines.push(formatHumanFailure(failureRecord, options.profile))
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Summary
|
||||
if (runResult.passed) {
|
||||
lines.push(`All ${runResult.total} contract(s) passed.`)
|
||||
} else {
|
||||
lines.push(`Failed: ${runResult.failed} of ${runResult.total} contract(s) failed.`)
|
||||
}
|
||||
lines.push(`Seed: ${options.seed}`)
|
||||
|
||||
// Replay command on failure
|
||||
if (!runResult.passed && runResult.failures.length > 0) {
|
||||
lines.push('')
|
||||
lines.push('Replay')
|
||||
lines.push(` apophis replay --artifact <path-to-artifact>`)
|
||||
lines.push('')
|
||||
lines.push('Determinism')
|
||||
lines.push(` This run used seed ${options.seed}.`)
|
||||
lines.push(` Same seed + same app state = same results.`)
|
||||
lines.push(` If results differ on re-run, the app has nondeterministic behavior.`)
|
||||
lines.push(` Stabilize: reset app state, mock external services, avoid time-dependent logic.`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main command handler
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -454,10 +360,12 @@ export async function verifyCommand(
|
||||
|
||||
// 5. Load the Fastify app
|
||||
let fastify: unknown
|
||||
let appEntrypoint: string | undefined
|
||||
try {
|
||||
const { loadApp } = await import('../../core/app-loader.js')
|
||||
const loaded = await loadApp(workingDir)
|
||||
fastify = loaded.fastify
|
||||
appEntrypoint = loaded.entrypoint
|
||||
if (fastify && typeof (fastify as any).ready === 'function') {
|
||||
await (fastify as any).ready()
|
||||
}
|
||||
@@ -465,7 +373,7 @@ export async function verifyCommand(
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: `No Fastify app found. Ensure app.js exports a Fastify instance or a factory function.\n\nSupported patterns:\n export default app\n export const createApp = () => app\n module.exports = app\n\nError: ${errorMessage}\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
||||
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -479,6 +387,9 @@ export async function verifyCommand(
|
||||
routeFilters,
|
||||
changed,
|
||||
profileRoutes: config.profiles?.[profile || '']?.routes,
|
||||
runs: typeof config.presets?.[loadResult.presetName || '']?.runs === 'number'
|
||||
? (config.presets[loadResult.presetName || ''] as { runs?: number }).runs
|
||||
: undefined,
|
||||
})
|
||||
|
||||
// 7. Build artifact
|
||||
|
||||
Reference in New Issue
Block a user