apophis/apophis-fastify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: plugin contract e2e, qualify --changed, production observe, regressions

Browse Source
This commit is contained in:
John Dvorak
2026-05-22 11:05:52 -07:00
parent d0523fcc2d
commit 1de735ee08
34 changed files with 1392 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/__fixtures__/plugin-contracts/apophis.config.js
+4
View File
@@ -0,0 +1,4 @@
export default {
profiles: { quick: { preset: "safe-ci" } },
presets: { "safe-ci": { runs: 3 } },
}
src/cli/__fixtures__/plugin-contracts/app.js
+34
View File
@@ -0,0 +1,34 @@
import Fastify from "fastify";
const app = Fastify({ logger: false });
await app.register(import("@fastify/swagger"), {
openapi: { info: { title: "Plugin Contracts Fixture", version: "1.0.0" } },
});
let apophisPlugin;
try {
({ default: apophisPlugin } = await import("../../../index.js"));
} catch {
({ default: apophisPlugin } = await import("../../../../dist/index.js"));
}
await app.register(apophisPlugin, {
runtime: "off",
pluginContracts: {
"required-header": {
appliesTo: "/api/**",
hooks: { onRequest: { requires: ["request_headers(this).x-custom != null"] } },
},
},
});
app.get("/api/data", {
schema: {
"x-category": "observer",
"x-ensures": ["status:200"],
response: { 200: { type: "object", properties: { ok: { type: "boolean" } } } },
},
}, async () => ({ ok: true }));
export default app;
src/cli/__fixtures__/plugin-contracts/package.json
+1
View File
@@ -0,0 +1 @@
{"name":"plugin-contracts-fixture","type":"module","private":true}
src/cli/commands/qualify/chaos-handler.ts
+19 -1
View File
@@ -63,8 +63,26 @@ export async function runChaosOnRoute(
const nonDelayEvents = events.filter(e => e.type !== 'inbound-delay')
const application = applyChaosToExecution(ctx, nonDelayEvents)
// 5. Validate post-chaos contracts: did the route still satisfy its contracts?
// Validate route contracts after chaos. Compose plugin contracts if available.
let contractsPassed = true
if (deps.pluginContractRegistry) {
const composed = deps.pluginContractRegistry.composeContracts(route)
if (!route.formulaSources) route.formulaSources = {}
for (const phase of Object.values(composed.phases)) {
for (const req of phase.requires) {
if (!route.requires.includes(req.formula)) {
route.requires.push(req.formula)
route.formulaSources[req.formula] = req.source
}
}
for (const ens of phase.ensures) {
if (!route.ensures.includes(ens.formula)) {
route.ensures.push(ens.formula)
route.formulaSources[ens.formula] = ens.source
}
}
}
}
if (application.applied && route.ensures.length > 0) {
const { validatePostconditionsAsync } = await import('../../../domain/contract-validation.js')
const result = await validatePostconditionsAsync(
src/cli/commands/qualify/index.ts
+55 -4
View File
@@ -30,6 +30,7 @@ import {
type StepTrace,
type CleanupFailure,
} from './runner.js'
import { isGitRepo, filterChangedRoutes } from '../verify/runner.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
import { renderHumanArtifact } from '../../renderers/human.js'
@@ -64,6 +65,7 @@ function coerceTimeout(value: unknown): number | undefined {
export interface QualifyOptions {
profile?: string
seed?: number
changed?: boolean
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson'
@@ -252,7 +254,8 @@ export function buildArtifact(
const totalTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
(runResult.statefulResult?.tests.length ?? 0)
(runResult.statefulResult?.summary.passed ?? 0) +
(runResult.statefulResult?.summary.failed ?? 0)
const passedTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
@@ -282,7 +285,11 @@ export function buildArtifact(
.filter(Boolean)
)]
const statefulRoutes = runResult.statefulResult
? [...new Set(runResult.statefulResult.tests.map(t => normalizeRouteIdentity(t.name)))]
? [...new Set(
runResult.statefulResult.tests
.filter(t => !t.directive)
.map(t => normalizeRouteIdentity(t.name))
)]
: []
const chaosRoutesPlanned: string[] = []
const chaosRoutesExecuted: string[] = []
@@ -343,6 +350,7 @@ export function buildArtifact(
total: totalTests,
passed: passedTests,
failed: failures.length,
skipped: (runResult.statefulResult?.summary.skipped ?? 0),
},
executionSummary,
coverageBreakdown,
@@ -437,6 +445,7 @@ export async function qualifyCommand(
const {
profile,
seed: explicitSeed,
changed,
config: configPath,
cwd,
artifactDir,
@@ -566,7 +575,31 @@ export async function qualifyCommand(
try {
// 9. Discover routes for chaos (with discovery metadata)
const appRoutes = await discoverAppRoutes(fastify)
const allRoutes = await discoverAppRoutes(fastify)
let appRoutes = allRoutes
// 9a. Apply --changed filter
if (changed) {
const inGit = await isGitRepo(workingDir)
if (!inGit) {
return {
exitCode: USAGE_ERROR,
message: '--changed requires a git repository. Current directory is not inside a git repo.',
}
}
appRoutes = await filterChangedRoutes(allRoutes, workingDir)
const matched = appRoutes.length
const total = allRoutes.length
if (!ctx.options.quiet && format === 'human') {
console.log(`${matched}/${total} routes match --changed filter`)
}
if (matched === 0) {
return {
exitCode: SUCCESS,
message: `No routes changed since HEAD. All ${total} routes skipped.`,
}
}
}
// Collect discovery warnings
const { discoverRouteDetails } = await import('../../../domain/discovery.js')
@@ -586,6 +619,8 @@ export async function qualifyCommand(
fastify: fastify as any,
seed,
timeout: presetTimeout,
pluginContractRegistry: (fastify as unknown as Record<string, unknown>)?._apophisPluginContractRegistry as
import('../../../domain/plugin-contracts.js').PluginContractRegistry | undefined,
}
const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
@@ -600,6 +635,15 @@ export async function qualifyCommand(
seed,
})
// Drain plugin contract registry warnings if app registered APOPHIS
const pcr = (fastify as any)?._apophisPluginContractRegistry
if (pcr?.drainWarnings) {
const pcrWarnings = pcr.drainWarnings()
if (pcrWarnings.length > 0) {
artifact.warnings.push(...pcrWarnings)
}
}
// 12. Signal quality guardrails — fail if zero checks executed
const execSummary = runResult.executionSummary
const warnings: string[] = [...artifact.warnings, ...discoveryWarnings]
@@ -624,7 +668,9 @@ export async function qualifyCommand(
if (gates.scenario && execSummary.scenariosRun === 0) {
warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
}
if (gates.stateful && execSummary.statefulTestsRun === 0) {
if (gates.stateful && execSummary.statefulTestsRun === 0 && execSummary.statefulSkipped > 0) {
warnings.push(`WARNING: stateful gate enabled but all ${execSummary.statefulSkipped} commands were skipped (preconditions not met). Check required headers, scopes, or plugin contract settings.`)
} else if (gates.stateful && execSummary.statefulTestsRun === 0) {
warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
}
if (gates.chaos && execSummary.chaosRunsRun === 0) {
@@ -706,6 +752,7 @@ export async function handleQualify(
const options: QualifyOptions = {
profile: ctx.options.profile || undefined,
seed: undefined,
changed: false,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as QualifyOptions['format'],
@@ -722,6 +769,10 @@ export async function handleQualify(
}
}
if (args.includes('--changed')) {
options.changed = true
}
const result = await qualifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
src/cli/commands/qualify/runner.ts
+11 -4
View File
@@ -29,6 +29,7 @@ import type {
FastifyInjectInstance,
} from '../../../types.js'
import type { ExtensionRegistry } from '../../../extension/types.js'
import type { PluginContractRegistry } from '../../../domain/plugin-contracts.js'
// ---------------------------------------------------------------------------
// Types
@@ -60,6 +61,7 @@ export interface QualifyRunResult {
totalFailed: number
scenariosRun: number
statefulTestsRun: number
statefulSkipped: number
chaosRunsRun: number
chaosRoutesPlanned: number
chaosRoutesExecuted: number
@@ -85,6 +87,7 @@ export interface CleanupFailure {
export interface QualifyRunnerDeps {
fastify: FastifyInjectInstance
extensionRegistry?: ExtensionRegistry
pluginContractRegistry?: PluginContractRegistry
seed: number
timeout?: number
}
@@ -237,17 +240,20 @@ export async function runQualify(
const statefulPassed = !statefulResult || statefulResult.summary.failed === 0
const chaosPassed = chaosResults.every(r => !r.applied || r.contractsPassed)
// Count execution metrics
// Count execution metrics — distinguish executed from skipped
const scenariosRun = scenarioResults.length
const statefulTestsRun = statefulResult?.tests.length ?? 0
const statefulPassedCount = statefulResult?.summary.passed ?? 0
const statefulFailedCount = statefulResult?.summary.failed ?? 0
const statefulSkippedCount = statefulResult?.summary.skipped ?? 0
const statefulTestsRun = statefulPassedCount + statefulFailedCount
const chaosRunsRun = chaosResults.length
const totalSteps = allTraces.length
const totalExecuted = scenariosRun + statefulTestsRun + chaosRunsRun
const totalPassed = scenarioResults.reduce((sum, r) => sum + r.summary.passed, 0) +
(statefulResult?.summary.passed ?? 0) +
statefulPassedCount +
chaosResults.reduce((sum, r) => sum + (r.applied && r.contractsPassed ? 1 : 0), 0)
const totalFailed = scenarioResults.reduce((sum, r) => sum + r.summary.failed, 0) +
(statefulResult?.summary.failed ?? 0)
statefulFailedCount
// Track executed and skipped routes for transparency
const executedRoutes: string[] = []
@@ -316,6 +322,7 @@ export async function runQualify(
totalFailed,
scenariosRun,
statefulTestsRun,
statefulSkipped: statefulSkippedCount,
chaosRunsRun,
chaosRoutesPlanned,
chaosRoutesExecuted,
src/cli/commands/qualify/scenario-handler.ts
+1 -1
View File
@@ -27,7 +27,7 @@ export async function runScenarioWithTraces(
): Promise<{ result: ScenarioResult; traces: StepTrace[] }> {
const scopeHeaders: Record<string, string> = {}
const result = await runScenario(deps.fastify, config, scopeHeaders, deps.extensionRegistry)
const result = await runScenario(deps.fastify, config, scopeHeaders, deps.extensionRegistry, deps.pluginContractRegistry)
const traces: StepTrace[] = result.steps.map((step, idx) => {
const trace: StepTrace = {
src/cli/commands/qualify/stateful-handler.ts
+1 -1
View File
@@ -45,7 +45,7 @@ export async function runStatefulWithTraces(
cleanupManager,
minimalScopeRegistry,
deps.extensionRegistry,
undefined, // pluginContractRegistry — will be passed from runner when available
deps.pluginContractRegistry,
undefined, // outboundContractRegistry
)
src/cli/commands/replay/index.ts
+29
View File
@@ -172,9 +172,36 @@ async function executeContractDirect(
validateRuntime: false,
}
// Compose plugin contracts if the app registered the APOPHIS plugin
const pluginContractRegistry = (fastify as any)?._apophisPluginContractRegistry
if (pluginContractRegistry) {
const composed = pluginContractRegistry.composeContracts(routeContract)
if (!routeContract.formulaSources) routeContract.formulaSources = {}
for (const phase of Object.values(composed.phases) as Array<{ requires: Array<{ formula: string; source: string }>; ensures: Array<{ formula: string; source: string }> }>) {
for (const req of phase.requires) {
if (!routeContract.requires.includes(req.formula)) {
routeContract.requires.push(req.formula)
routeContract.formulaSources[req.formula] = req.source as 'route' | `plugin:${string}`
}
}
for (const ens of phase.ensures) {
if (!routeContract.ensures.includes(ens.formula)) {
routeContract.ensures.push(ens.formula)
routeContract.formulaSources[ens.formula] = ens.source as 'route' | `plugin:${string}`
}
}
}
}
// Build request
const headers: Record<string, string> = {}
// Auto-inject headers required by plugin contract preconditions
for (const f of routeContract.requires) {
const m = f.match(/request_headers\(this\)\.([\w-]+)\s*(!=\s*null|==\s*"([^"]+)")/)
if (m) headers[m[1]!] = m[3] || 'test-value'
}
// Execute request
try {
const ctx = await executeHttp(fastify, routeContract, {
@@ -264,10 +291,12 @@ async function executeReplay(
}
// Try to run verify first (works if app has APOPHIS plugin)
const pluginContractRegistry = (fastify as any)?._apophisPluginContractRegistry
let runResult = await runVerify({
fastify: fastify as any,
seed: artifact.seed || 42,
routeFilters: [failure.route],
pluginContractRegistry,
})
// If no routes matched, or route found but no contracts (plugin not registered before routes),
src/cli/commands/verify/index.ts
+19 -35
View File
@@ -170,6 +170,7 @@ function buildArtifact(
total: runResult.total,
passed: runResult.passedCount,
failed: runResult.failed,
skipped: runResult.skipped,
},
deterministicParams: {
seed: options.seed,
@@ -390,6 +391,7 @@ export async function verifyCommand(
runs: typeof config.presets?.[loadResult.presetName || '']?.runs === 'number'
? (config.presets[loadResult.presetName || ''] as { runs?: number }).runs
: undefined,
pluginContractRegistry: (fastify as any)?._apophisPluginContractRegistry,
})
// 7. Build artifact
@@ -403,6 +405,15 @@ export async function verifyCommand(
routeFilters,
})
// Drain plugin contract registry warnings if app registered APOPHIS
const pcr = (fastify as any)?._apophisPluginContractRegistry
if (pcr?.drainWarnings) {
const pcrWarnings = pcr.drainWarnings()
if (pcrWarnings.length > 0) {
artifact.warnings.push(...pcrWarnings)
}
}
// 8. Write artifact if configured or on failure
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
await emitArtifact(artifact, {
@@ -638,54 +649,27 @@ export async function handleVerify(
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
if (!ctx.options.quiet) {
// Machine-readable formats always emit regardless of --quiet
if (machineMode) {
if (format === 'json') {
if (result.artifact) {
console.log(renderJsonArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
console.log(result.artifact ? renderJsonArtifact(result.artifact) : renderJson({ exitCode: result.exitCode, message: result.message, warnings: result.warnings }))
} else if (format === 'json-summary') {
if (result.artifact) {
console.log(renderJsonSummaryArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
console.log(result.artifact ? renderJsonSummaryArtifact(result.artifact) : renderJson({ exitCode: result.exitCode, message: result.message, warnings: result.warnings }))
} else if (format === 'ndjson') {
if (result.artifact) {
renderNdjsonArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'verify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
process.stdout.write(JSON.stringify({ type: 'run.completed', command: 'verify', exitCode: result.exitCode, message: result.message, warnings: result.warnings }) + '\n')
}
} else if (format === 'ndjson-summary') {
if (result.artifact) {
renderNdjsonSummaryArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'verify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
process.stdout.write(JSON.stringify({ type: 'run.completed', command: 'verify', exitCode: result.exitCode, message: result.message, warnings: result.warnings }) + '\n')
}
} else if (result.message) {
console.log(result.message)
}
} else if (!ctx.options.quiet && result.message) {
console.log(result.message)
}
// Print warnings in human mode only
src/cli/commands/verify/runner.ts
+129 -15
View File
@@ -26,6 +26,7 @@ import * as fc from 'fast-check'
import { resolveRuns } from '../../../types.js'
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
import type { RouteResult } from '../../core/types.js'
import type { PluginContractRegistry } from '../../../domain/plugin-contracts.js'
// ---------------------------------------------------------------------------
// Types
@@ -46,6 +47,7 @@ export interface VerifyRunResult {
total: number
passedCount: number
failed: number
skipped: number
failures: VerifyFailure[]
durationMs: number
noRoutesMatched: boolean
@@ -66,6 +68,7 @@ export interface VerifyRunnerDeps {
changed?: boolean
profileRoutes?: string[]
runs?: number
pluginContractRegistry?: PluginContractRegistry
}
// ---------------------------------------------------------------------------
@@ -161,7 +164,7 @@ function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): Ro
/**
* Check if cwd is inside a git repository.
*/
async function isGitRepo(cwd: string): Promise<boolean> {
export async function isGitRepo(cwd: string): Promise<boolean> {
try {
const { execSync } = await import('node:child_process')
execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' })
@@ -174,7 +177,7 @@ async function isGitRepo(cwd: string): Promise<boolean> {
/**
* Get git-modified files for --changed filtering.
*/
async function getGitChangedFiles(cwd: string): Promise<string[]> {
export async function getGitChangedFiles(cwd: string): Promise<string[]> {
try {
const { execSync } = await import('node:child_process')
const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' })
@@ -187,7 +190,7 @@ async function getGitChangedFiles(cwd: string): Promise<string[]> {
/**
* Filter routes to only those modified in git.
*/
async function filterChangedRoutes(
export async function filterChangedRoutes(
routes: RouteContract[],
cwd: string,
): Promise<RouteContract[]> {
@@ -481,6 +484,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total: 0,
passedCount: 0,
failed: 0,
skipped: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
@@ -501,6 +505,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total: 0,
passedCount: 0,
failed: 0,
skipped: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: true,
@@ -511,6 +516,28 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
}
}
// 3a. Compose plugin contracts into route contracts
if (deps.pluginContractRegistry) {
for (const route of routes) {
const composed = deps.pluginContractRegistry.composeContracts(route)
if (!route.formulaSources) route.formulaSources = {}
for (const phase of Object.values(composed.phases)) {
for (const req of phase.requires) {
if (!route.requires.includes(req.formula)) {
route.requires.push(req.formula)
route.formulaSources[req.formula] = req.source
}
}
for (const ens of phase.ensures) {
if (!route.ensures.includes(ens.formula)) {
route.ensures.push(ens.formula)
route.formulaSources[ens.formula] = ens.source
}
}
}
}
}
// 3. Check for behavioral contracts
const routesWithContracts = routes.filter(route =>
route.ensures.length > 0 || route.requires.length > 0
@@ -522,6 +549,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total: 0,
passedCount: 0,
failed: 0,
skipped: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
@@ -533,33 +561,118 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
}
// 4. Execute contracts (with variant expansion)
// Execute one HTTP request per (route, variant, runIndex), then validate
// all contracts against the same request context.
// Requires (preconditions) that fail → SKIP the ensures and count as skipped.
// Ensures (postconditions) that fail → count as failures.
const runConfig = resolveRuns(deps.runs)
const runs = runConfig.contractRuns
const failures: VerifyFailure[] = []
let total = 0
let passedCount = 0
let skippedCount = 0
// Extract required headers from precondition formulas and auto-inject them
// so that plugin contracts (e.g., x-tenant-id != null) are satisfied.
function extractRequiredHeaders(formulas: string[]): Record<string, string> {
const headers: Record<string, string> = {}
for (const f of formulas) {
const nullCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*!=\s*null/i)
if (nullCheck) { headers[nullCheck[1]!] = 'test-value'; continue }
const eqCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*==\s*"([^"]+)"/i)
if (eqCheck) { headers[eqCheck[1]!] = eqCheck[2]!; continue }
}
return headers
}
for (const route of routesWithContracts) {
const contracts = [...route.requires, ...route.ensures]
const defaultHeaders = extractRequiredHeaders(route.requires)
const variants = route.variants && route.variants.length > 0
? route.variants
: [{ name: 'default' }]
for (const variant of variants) {
for (const contract of contracts) {
for (let runIndex = 0; runIndex < runs; runIndex++) {
total++
const result = await executeContract(
fastify, route, contract, deps.timeout,
variant, deps.seed, runIndex,
)
for (let runIndex = 0; runIndex < runs; runIndex++) {
// Execute one HTTP request per run
const request = buildRouteRequest(route, deps.seed, runIndex, variant)
const headers = {
...defaultHeaders,
...request.headers,
...(variant.headers ?? {}),
}
const ctx = await executeHttp(fastify, route, {
method: request.method,
url: request.url,
body: request.body,
headers,
query: {},
}, undefined, deps.timeout)
const evalCtx: EvalContext = {
...ctx,
operationResolver: createOperationResolver(fastify, headers, ctx),
}
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
// Evaluate requires first. If any fail, skip ensures for this run.
let requiresPassed = true
for (const formula of route.requires) {
total++
try {
const parsed = parse(formula)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
requiresPassed = false
skippedCount++
} else {
passedCount++
}
} catch {
requiresPassed = false
skippedCount++
}
}
// Only evaluate ensures if all requires passed
if (requiresPassed) {
for (const formula of route.ensures) {
total++
try {
const parsed = parse(formula)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
const diagnostic = buildFailureDiagnostic(formula, evalCtx, result.success ? undefined : result.error)
failures.push({
route: variant.name && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract: formula,
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
})
} else {
passedCount++
}
} catch (error) {
const diagnostic = buildFailureDiagnostic(formula, evalCtx, error instanceof Error ? error.message : String(error))
failures.push({
route: variant.name && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract: formula,
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
})
}
}
} else {
// Preconditions failed: ensures are skipped but not evaluated.
// We still count each ensure as "total" and "skipped" for transparency.
total += route.ensures.length
skippedCount += route.ensures.length
}
}
}
}
@@ -578,6 +691,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total,
passedCount,
failed: failures.length,
skipped: skippedCount,
failures: sortedFailures,
durationMs,
noRoutesMatched: false,
src/cli/core/config-loader.ts
+1 -1
View File
@@ -140,7 +140,7 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
optional: true,
items: { type: 'object' },
},
chaos: {
pluginContracts: {
type: 'object',
optional: true,
properties: {},
src/cli/core/context.ts
+1 -1
View File
@@ -93,7 +93,7 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
const packageManager = detectPackageManager(cwd);
// Normalize options
const format = options.format === 'json' || options.format === 'ndjson'
const format = options.format === 'json' || options.format === 'ndjson' || options.format === 'json-summary' || options.format === 'ndjson-summary'
? options.format
: 'human';
src/cli/core/index.ts
+7 -4
View File
@@ -102,9 +102,11 @@ function getCommandHelp(command: string): string {
${pc.dim('Options:')}
--profile <name> Profile name from config
--seed <number> Deterministic seed
--changed Filter to git-modified routes
${pc.dim('Examples:')}
apophis qualify --profile oauth-nightly --seed 42
apophis qualify --changed
`,
replay: `
${pc.bold('apophis replay')} — Replay a failure using seed and stored trace
@@ -167,18 +169,18 @@ function printInternalError(error: unknown): void {
console.error();
}
function resolveRequestedFormat(argv: string[]): 'human' | 'json' | 'ndjson' {
function resolveRequestedFormat(argv: string[]): 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary' {
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (!arg) continue;
if (arg === '--format' && argv[i + 1]) {
const value = argv[i + 1];
if (value === 'json' || value === 'ndjson') return value;
if (value === 'json' || value === 'ndjson' || value === 'json-summary' || value === 'ndjson-summary') return value as 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
return 'human';
}
if (arg.startsWith('--format=')) {
const value = arg.slice('--format='.length);
if (value === 'json' || value === 'ndjson') return value;
if (value === 'json' || value === 'ndjson' || value === 'json-summary' || value === 'ndjson-summary') return value as 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
return 'human';
}
}
@@ -277,6 +279,7 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
case 'qualify':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--seed <number>', 'Deterministic seed');
cmd.option('--changed', 'Filter to git-modified routes');
break;
case 'replay':
cmd.option('--artifact <path>', 'Path to failure artifact');
@@ -375,7 +378,7 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']),
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config', '--workspace']),
qualify: new Set(['--profile', '--seed', '--workspace']),
qualify: new Set(['--profile', '--seed', '--workspace', '--changed']),
replay: new Set(['--artifact', '--route']),
doctor: new Set(['--mode', '--strict', '--workspace']),
migrate: new Set(['--check', '--dry-run', '--write']),
src/cli/core/types.ts
+1
View File
@@ -297,6 +297,7 @@ export interface Artifact {
total: number;
passed: number;
failed: number;
skipped?: number;
};
executionSummary?: ExecutionSummary;
coverageBreakdown?: {
src/cli/renderers/human.ts
+24 -4
View File
@@ -202,6 +202,10 @@ function renderSummary(
lines.push(` Failed: ${summary.failed}`);
}
if (summary.skipped !== undefined && summary.skipped > 0) {
lines.push(` ${c.yellow('Skipped:')} ${summary.skipped} (preconditions not met)`);
}
lines.push(` Duration: ${formatDuration(artifact.durationMs)}`);
if (artifact.seed !== undefined) {
@@ -404,15 +408,31 @@ export function renderHumanArtifact(
lines.push(c.bold('Next steps'));
if (artifact.command === 'verify') {
if (artifact.summary.failed === 0) {
lines.push(` ${c.green('✓')} All contracts passed.`);
lines.push(` ${c.dim('→ Add more behavioral contracts with')} x-ensures ${c.dim('and')} x-requires ${c.dim('to cover more routes.')}`);
lines.push(` ${c.dim('→ Run')} apophis observe ${c.dim('to enable runtime contract monitoring in production.')}`);
lines.push(` ${c.dim('→ Run')} apophis qualify --profile standard ${c.dim('for stateful/chaos testing.')}`);
if (artifact.summary.skipped && artifact.summary.skipped > 0) {
lines.push(` ${c.yellow('!')} No contract violations, but ${artifact.summary.skipped} precondition(s) were unmet and skipped.`);
lines.push(` ${c.dim('→ Preconditions (x-requires + plugin contract requires) must be satisfied for tests to run.')}`);
lines.push(` ${c.dim('→ Send required headers via --variant or scope configuration.')}`);
lines.push(` ${c.dim('→ Add more behavioral contracts with')} x-ensures ${c.dim('and')} x-requires ${c.dim('to cover more routes.')}`);
} else {
lines.push(` ${c.green('✓')} All contracts passed.`);
lines.push(` ${c.dim('→ Add more behavioral contracts with')} x-ensures ${c.dim('and')} x-requires ${c.dim('to cover more routes.')}`);
lines.push(` ${c.dim('→ Run')} apophis observe ${c.dim('to enable runtime contract monitoring in production.')}`);
lines.push(` ${c.dim('→ Run')} apophis qualify --profile standard ${c.dim('for stateful/chaos testing.')}`);
}
} else {
lines.push(` ${c.yellow('!')} Fix failing contracts and rerun with:`);
lines.push(` ${c.dim('→')} apophis verify --seed ${artifact.seed} ${artifact.profile ? `--profile ${artifact.profile}` : ''}`);
lines.push(` ${c.dim('→ Or replay the artifact:')} apophis replay --artifact <path>`);
}
} else if (artifact.command === 'qualify') {
if (artifact.summary.skipped && artifact.summary.skipped > 0) {
lines.push(` ${c.yellow('!')} ${artifact.summary.skipped} stateful command(s) skipped — preconditions not met.`);
lines.push(` ${c.dim('→ Check plugin contracts and route headers. Qualify needs preconditions satisfied.')}`);
}
if (artifact.summary.failed === 0 && (!artifact.summary.skipped || artifact.summary.skipped === 0)) {
lines.push(` ${c.green('✓')} All checks passed.`);
}
lines.push(` ${c.dim('→ Review')} reports/apophis/ ${c.dim('for detailed artifact output.')}`);
}
return lines.join('\n');