apophis/apophis-fastify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden engine, enrich failure diagnostics, close adoption gaps

Browse Source 
- P0: CLI verify now honors  test budget with seeded multi-sample
- P0: Observe sampling enforced via Math.random() gate in hook-validator
- P1: Remove misleading undici-mock-agent isolation option
- P1: Qualify reuses shared discoverRouteDetails() with warnings
- P1: Chaos/scenario config exposed via preset schema
- P1: README/docs limitations updated to current state
- P2: Nested response annotations prefer 2xx deterministically
- P2: --changed documented as heuristic in verify.md

- Add observe sink tests (sampling 0/1, sink failure non-interference)
- Add verify runs regression tests (scale, determinism, variants)
- Add configured-scenario qualify test (independent of OAuth fixture)
- Add coverageBreakdown to qualify artifacts (per-gate route coverage)
- Add production-style observe example with real sink in docs/observe.md
- Add nightly/staging vs PR gating guidance to docs/qualify.md

- Enrich VerifyFailure with formula-aware diagnostics:
  status:201 => 'HTTP 200', body field checks => actual values
- Remove stale observe CLI activation message
- Document outbound mocks as process-global in getting-started.md
- Refresh APOPHIS_ADOPTION_AUDIT.md with current state

903 tests pass, build clean, typecheck clean.
This commit is contained in:
John Dvorak
2026-05-21 20:39:36 -07:00
parent 55b0262799
commit d0523fcc2d
128 changed files with 4004 additions and 3631 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/__goldens__/help.txt
+1 -1
View File
@@ -6,7 +6,7 @@ Options:
--config <path> Path to config file
--profile <name> Profile name from config
--cwd <path> Working directory
--format <human|json|ndjson> Output format (default: human)
--format <human|json|ndjson|json-summary|ndjson-summary> Output format (default: human)
--color <auto|always|never> Color mode (default: auto)
--quiet Suppress non-essential output
--verbose Verbose output
src/cli/commands/doctor/checks/dependencies.ts
+20 -6
View File
@@ -33,7 +33,7 @@ export interface DependencyCheckOptions {
// Constants
// ---------------------------------------------------------------------------
const MIN_NODE_VERSION = 18;
const MIN_NODE_VERSION = 20;
const REQUIRED_PEER_DEPS = ['fastify', '@fastify/swagger'];
// ---------------------------------------------------------------------------
@@ -54,13 +54,27 @@ function parseNodeMajor(version: string): number {
export function checkNodeVersion(nodeVersion: string): DependencyCheckResult {
const major = parseNodeMajor(nodeVersion);
if (major < MIN_NODE_VERSION) {
const tooOld = major < MIN_NODE_VERSION;
const untested = major >= 23 || major === 21;
if (tooOld) {
return {
name: 'node-version',
status: 'fail',
message: `Node.js ${nodeVersion} is not supported. Minimum required: ${MIN_NODE_VERSION}.x`,
detail: `APOPHIS requires Node.js ${MIN_NODE_VERSION} or higher for ESM and modern features.`,
remediation: `Upgrade Node.js to ${MIN_NODE_VERSION}.x or higher (use nvm, fnm, or your package manager).`,
message: `Node.js ${nodeVersion} is not supported. APOPHIS requires Node.js >=${MIN_NODE_VERSION}.18.1.`,
detail: `Detected Node.js ${nodeVersion} (major ${major}). APOPHIS requires Node ${MIN_NODE_VERSION} LTS or 22 LTS.`,
remediation: `Install Node.js ${MIN_NODE_VERSION}.x or 22.x (use nvm, fnm, or your package manager).`,
mode: 'all',
};
}
if (untested) {
return {
name: 'node-version',
status: 'warn',
message: `Node.js ${nodeVersion} is not in the tested range (20.x or 22.x).`,
detail: `Detected Node.js ${nodeVersion} (major ${major}). APOPHIS is tested on Node 20 LTS and 22 LTS.`,
remediation: `Use Node.js 20.x or 22.x for best compatibility. Current version may work but is untested.`,
mode: 'all',
};
}
@@ -68,7 +82,7 @@ export function checkNodeVersion(nodeVersion: string): DependencyCheckResult {
return {
name: 'node-version',
status: 'pass',
message: `Node.js ${nodeVersion} meets minimum requirement (${MIN_NODE_VERSION}+)`,
message: `Node.js ${nodeVersion} is supported (>=${MIN_NODE_VERSION}.18.1 <21 || >=22 <23)`,
mode: 'all',
};
}
src/cli/commands/doctor/checks/routes.ts
+36 -44
View File
@@ -7,8 +7,7 @@
* - Is the app file loadable?
*/
import { existsSync } from 'node:fs';
import { resolve } from 'node:path';
import { APP_CANDIDATES, findAppFile } from '../../../core/app-loader.js';
// ---------------------------------------------------------------------------
// Types
@@ -28,38 +27,6 @@ export interface RouteCheckOptions {
configPath?: string;
}
// ---------------------------------------------------------------------------
// App file detection
// ---------------------------------------------------------------------------
const APP_CANDIDATES = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
];
/**
* Find the Fastify app entrypoint file.
*/
function findAppFile(cwd: string): string | null {
for (const candidate of APP_CANDIDATES) {
const fullPath = resolve(cwd, candidate);
if (existsSync(fullPath)) {
return candidate;
}
}
return null;
}
/**
* Check if app file exists and is readable.
*/
@@ -106,7 +73,7 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
}
try {
const appPath = resolve(options.cwd, appFile);
const appPath = appFile;
const appModule = await import(appPath);
const app = appModule.default || appModule;
@@ -147,18 +114,32 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
}
// Check for routes
let routeCount = 0;
let routeCount = 0
let discoverySource: string | undefined
// Fastify 5+ routes access
if (app.routes && typeof app.routes === 'function') {
const routes = app.routes();
routeCount = Array.isArray(routes) ? routes.length : 0;
const routes = app.routes()
routeCount = Array.isArray(routes) ? routes.length : 0
}
// Fallback: check if we can get routes via inject or other methods
if (routeCount === 0 && app.hasRoute) {
// We can't enumerate, but we can at least verify the app is functional
routeCount = -1; // Unknown but app seems functional
routeCount = -1
}
// Use discoverRouteDetails for metadata-aware reporting
try {
const { discoverRouteDetails } = await import('../../../../domain/discovery.js')
const discovery = discoverRouteDetails(app as Parameters<typeof discoverRouteDetails>[0])
if (discovery.source === 'print-routes') {
discoverySource = 'print-routes'
if (routeCount === 0 && discovery.routes.length > 0) {
routeCount = discovery.routes.length
}
}
} catch {
// discovery diagnostics are optional
}
if (routeCount === 0) {
@@ -170,7 +151,7 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
'APOPHIS discovers routes via the onRoute hook.',
remediation: 'Register routes before exporting the app, or ensure the APOPHIS plugin is registered.',
mode: 'all',
};
}
}
if (routeCount < 0) {
@@ -180,7 +161,18 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
message: `App loaded from ${appFile}. Route enumeration not available (app is functional).`,
detail: 'Route count could not be determined, but the app appears to be a valid Fastify instance.',
mode: 'all',
};
}
}
if (discoverySource === 'print-routes') {
return {
name: 'route-discovery',
status: 'warn',
message: `Discovered ${routeCount} route(s) from ${appFile} but without schema metadata.`,
detail: 'Routes were discovered through printRoutes() fallback. Schema annotations (x-ensures, x-requires, x-outbound, x-timeout, x-variants) are unavailable. APOPHIS can detect paths but not behavioral contracts.',
remediation: 'Register APOPHIS or install route discovery before defining routes, or use createFastify() from @apophis/fastify.',
mode: 'all',
}
}
return {
@@ -233,8 +225,8 @@ export async function checkSwaggerRegistration(options: RouteCheckOptions): Prom
}
try {
const appPath = resolve(options.cwd, appFile);
const content = (await import('node:fs')).readFileSync(appPath, 'utf-8');
const { readFileSync } = await import('node:fs');
const content = readFileSync(appFile, 'utf-8');
if (content.includes('@fastify/swagger') || content.includes('fastify-swagger')) {
return {
src/cli/commands/doctor/index.ts
-6
View File
@@ -78,12 +78,6 @@ export interface DoctorResult {
// Check filtering
// ---------------------------------------------------------------------------
function shouldRunCheck(checkMode: string | undefined, modeFilter: DoctorMode): boolean {
if (!modeFilter) return true;
if (!checkMode || checkMode === 'all') return true;
return checkMode === modeFilter;
}
// ---------------------------------------------------------------------------
// Monorepo detection
// ---------------------------------------------------------------------------
src/cli/commands/init/index.ts
+3 -17
View File
@@ -8,6 +8,7 @@ import { resolve } from 'node:path';
import type { CliContext } from '../../core/types.js';
import { USAGE_ERROR, SUCCESS } from '../../core/exit-codes.js';
import { getScaffoldForPreset, getPresetNames, type ScaffoldResult } from './scaffolds/index.js';
import { APP_CANDIDATES } from '../../core/app-loader.js';
// ─────────────────────────────────────────────────────────────────────────────
// Types
@@ -63,22 +64,7 @@ function renderInstallCommand(
* - Common server file names (server.js, app.js, index.js, etc.)
*/
export async function detectFastifyEntrypoint(cwd: string): Promise<string | null> {
const candidates = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
];
for (const candidate of candidates) {
for (const candidate of APP_CANDIDATES) {
const fullPath = resolve(cwd, candidate);
if (!existsSync(fullPath)) continue;
@@ -217,7 +203,7 @@ function generateConfigContent(config: ScaffoldResult['config'], isTypeScript: b
lines.push('');
if (isTypeScript) {
lines.push('import type { ApophisConfig } from "apophis-fastify/cli";');
lines.push('import type { ApophisConfig } from "@apophis/fastify";');
lines.push('');
lines.push('const config: ApophisConfig = ' + stringifyConfig(config) + ';');
lines.push('');
src/cli/commands/observe/index.ts
+10 -1
View File
@@ -264,7 +264,16 @@ function formatActivationOutput(
}
lines.push('');
lines.push('To activate observation, run without --check-config.');
lines.push('The CLI validates configuration and reports readiness.');
lines.push('To activate runtime observation, register the APOPHIS plugin in your');
lines.push('application with observe options:');
lines.push('');
lines.push(' await fastify.register(apophisPlugin, {');
lines.push(' runtime: \'warn\',');
lines.push(' observe: { enabled: true, sinks: [...] }');
lines.push(' });');
lines.push('');
lines.push('See docs/observe.md for the full programmatic activation guide.');
return lines.join('\n');
}
src/cli/commands/qualify/chaos-handler.ts
+38 -14
View File
@@ -11,7 +11,7 @@
* - No optional imports — everything is passed via parameters
*/
import { applyChaosToExecution, createChaosEventArbitrary, formatChaosEvents } from '../../../quality/chaos-v3.js'
import { applyChaosToExecution, formatChaosEvents, extractDelays, sleep as chaosSleep } from '../../../quality/chaos-v3.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type {
RouteContract,
@@ -23,6 +23,12 @@ import type { QualifyRunnerDeps, ChaosRunResult } from './runner.js'
/**
* Run a single route with chaos injection and collect traces.
* Uses chaos-v3 pure functions for deterministic adversity.
*
* Flow:
* 1. Generate deterministic chaos events
* 2. Apply transport-level delays (sleep before HTTP request)
* 3. Execute the HTTP request
* 4. Apply remaining chaos (error, dropout, corruption) to the response context
*/
export async function runChaosOnRoute(
deps: QualifyRunnerDeps,
@@ -31,12 +37,16 @@ export async function runChaosOnRoute(
): Promise<{ ctx: EvalContext; chaosResult: ChaosRunResult }> {
const started = Date.now()
// Generate chaos events using seeded RNG via fast-check
// For CLI qualify, we use a deterministic subset
const rng = new SeededRng(deps.seed)
const contractNames: string[] = []
// 1. Generate deterministic chaos events
const events = generateDeterministicChaosEvents(chaosConfig, deps.seed)
// Build a minimal request for the route
// 2. Apply transport-level delays BEFORE the HTTP request
const { totalMs: delayMs } = extractDelays(events)
if (delayMs > 0) {
await chaosSleep(delayMs)
}
// 3. Build a minimal request and execute it
const request = {
method: route.method,
url: route.path,
@@ -45,25 +55,39 @@ export async function runChaosOnRoute(
body: undefined as unknown,
}
// Execute the request
const { executeHttp } = await import('../../../infrastructure/http-executor.js')
const ctx = await executeHttp(deps.fastify, route, request, undefined, deps.timeout)
// Generate and apply chaos events
const chaosArb = createChaosEventArbitrary(chaosConfig, contractNames)
// For deterministic CLI runs, we generate a fixed small set of events
// In practice, fast-check would be used in property tests; here we simulate
const events = generateDeterministicChaosEvents(chaosConfig, deps.seed)
// 4. Apply remaining chaos (error, dropout, corruption) to response context
// Filter out delay events since they were already applied at the transport level
const nonDelayEvents = events.filter(e => e.type !== 'inbound-delay')
const application = applyChaosToExecution(ctx, nonDelayEvents)
const application = applyChaosToExecution(ctx, events)
// 5. Validate post-chaos contracts: did the route still satisfy its contracts?
let contractsPassed = true
if (application.applied && route.ensures.length > 0) {
const { validatePostconditionsAsync } = await import('../../../domain/contract-validation.js')
const result = await validatePostconditionsAsync(
route.ensures,
application.ctx,
route,
undefined as unknown as import('../../../extension/types.js').ExtensionRegistry
)
contractsPassed = result.success
}
// Consider delay as "applied chaos" for reporting purposes
const hadDelay = delayMs > 0
const chaosApplied = application.applied || hadDelay
const chaosResult: ChaosRunResult = {
applied: application.applied,
applied: chaosApplied,
events: application.events
.filter(e => e.type !== 'none')
.map(e => formatChaosEvents([e])),
route: `${route.method} ${route.path}`,
durationMs: Date.now() - started,
contractsPassed,
}
return { ctx: application.ctx, chaosResult }
src/cli/commands/qualify/index.ts
+88 -144
View File
@@ -37,7 +37,6 @@ import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../..
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
import type { OutputContext } from '../../renderers/shared.js'
import { resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
@@ -98,24 +97,11 @@ export function generateSeed(): number {
/**
* Discover routes from the Fastify app for chaos execution.
* Injected fastify instance must have routes registered.
* Uses the shared discovery module for capture + fallback support.
*/
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
// Cast to access routes
const app = fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }> }
if (!app.routes) return []
return app.routes.map(r => ({
path: r.url,
method: r.method as RouteContract['method'],
category: 'observer',
requires: [],
ensures: [],
invariants: [],
regexPatterns: {},
validateRuntime: false,
schema: r.schema,
}))
const { discoverRoutes } = await import('../../../domain/discovery.js')
return discoverRoutes(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
}
// ---------------------------------------------------------------------------
@@ -288,12 +274,51 @@ export function buildArtifact(
// Build execution summary from runner result
const executionSummary = runResult.executionSummary
// Build per-gate coverage breakdown for clear artifact interpretation
const scenarioRoutes = [...new Set(
runResult.stepTraces
.filter(t => t.status === 'passed' || t.status === 'failed')
.map(t => t.route)
.filter(Boolean)
)]
const statefulRoutes = runResult.statefulResult
? [...new Set(runResult.statefulResult.tests.map(t => normalizeRouteIdentity(t.name)))]
: []
const chaosRoutesPlanned: string[] = []
const chaosRoutesExecuted: string[] = []
for (const r of runResult.chaosResults ?? []) {
if (r.applied) chaosRoutesExecuted.push(r.route)
}
// Infer planned from chaos results or executionSummary
if (executionSummary.chaosRoutesPlanned > 0 && chaosRoutesExecuted.length > 0) {
chaosRoutesPlanned.push(...chaosRoutesExecuted)
}
const coverageBreakdown = {
scenario: {
routesCovered: scenarioRoutes,
stepsTotal: executionSummary.totalSteps,
stepsPassed: runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0),
},
stateful: {
routesCovered: statefulRoutes,
testsTotal: executionSummary.statefulTestsRun,
testsPassed: runResult.statefulResult?.summary.passed ?? 0,
},
chaos: {
routesPlanned: chaosRoutesPlanned,
routesExecuted: chaosRoutesExecuted,
runsTotal: executionSummary.chaosRunsRun,
runsPassed: (runResult.chaosResults ?? []).filter(r => r.contractsPassed).length,
},
}
// Build profile gates from the result context
// We need to pass gates through or infer from results
const profileGates = {
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
chaos: (runResult.chaosResult !== undefined) || executionSummary.chaosRunsRun > 0,
chaos: executionSummary.chaosRoutesPlanned > 0,
}
// Deterministic parameters for audit
@@ -320,6 +345,7 @@ export function buildArtifact(
failed: failures.length,
},
executionSummary,
coverageBreakdown,
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
route: sr.route,
@@ -386,117 +412,6 @@ async function emitArtifact(
// Output formatting
// ---------------------------------------------------------------------------
function formatHumanOutput(
result: QualifyRunResult,
options: { profile?: string; seed: number; env: string },
): string {
const lines: string[] = []
lines.push(`Qualify run for environment "${options.env}"`)
if (options.profile) {
lines.push(`Profile: ${options.profile}`)
}
lines.push(`Seed: ${options.seed}`)
lines.push('')
// Scenario results
for (const scenario of result.scenarioResults) {
lines.push(`Scenario: ${scenario.name}`)
for (const step of scenario.steps) {
const icon = step.ok ? '✓' : '✗'
lines.push(` ${icon} ${step.name} (${step.statusCode ?? 'no-status'})`)
if (!step.ok && step.diagnostics) {
lines.push(` Expected: ${step.diagnostics.expected || 'success'}`)
lines.push(` Observed: ${step.diagnostics.error || 'failure'}`)
if (step.diagnostics.actual) {
lines.push(` Actual: ${step.diagnostics.actual}`)
}
if (step.diagnostics.diff) {
lines.push(` Diff:`)
for (const line of String(step.diagnostics.diff).split('\n')) {
lines.push(` ${line}`)
}
}
}
}
lines.push('')
}
// Stateful results
if (result.statefulResult) {
lines.push(`Stateful: ${result.statefulResult.summary.passed} passed, ${result.statefulResult.summary.failed} failed`)
lines.push('')
}
// Chaos results
if (result.chaosResult) {
lines.push(`Chaos: ${result.chaosResult.applied ? 'applied' : 'none'}`)
if (result.chaosResult.events.length > 0) {
for (const event of result.chaosResult.events) {
lines.push(` ${event}`)
}
}
lines.push('')
}
// Step traces
if (result.stepTraces.length > 0) {
lines.push('Step traces:')
for (const trace of result.stepTraces.slice(0, 20)) {
const icon = trace.status === 'passed' ? '✓' : trace.status === 'skipped' ? '⊘' : '✗'
lines.push(` ${icon} ${trace.name} (${trace.durationMs}ms)`)
}
if (result.stepTraces.length > 20) {
lines.push(` ... and ${result.stepTraces.length - 20} more`)
}
lines.push('')
}
// Cleanup failures
if (result.cleanupFailures.length > 0) {
lines.push('Cleanup failures (reported separately):')
for (const cf of result.cleanupFailures) {
lines.push(` ⚠ ${cf.resource}: ${cf.error}`)
}
lines.push('')
}
// Per-profile gate execution counts
lines.push('Profile gate execution counts:')
lines.push(` Scenario: ${result.executionSummary.scenariosRun} run`)
lines.push(` Stateful: ${result.executionSummary.statefulTestsRun} tests run`)
lines.push(` Chaos: ${result.executionSummary.chaosRunsRun} runs run`)
lines.push('')
// Executed routes
if (result.executedRoutes.length > 0) {
lines.push(`Executed routes (${result.executedRoutes.length}):`)
for (const route of result.executedRoutes) {
lines.push(` ${route}`)
}
lines.push('')
}
// Skipped routes
if (result.skippedRoutes.length > 0) {
lines.push(`Skipped routes (${result.skippedRoutes.length}):`)
for (const sr of result.skippedRoutes) {
lines.push(` ${sr.route}: ${sr.reason}`)
}
lines.push('')
}
// Summary
if (result.passed) {
lines.push('All qualifications passed.')
} else {
lines.push('Qualification failed.')
lines.push(`Replay: apophis replay --artifact <artifact-path>`)
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
@@ -583,9 +498,26 @@ export async function qualifyCommand(
const profileDef = profile ? config.profiles?.[profile] : undefined
const gates = resolveProfileGates(profileDef?.features)
// 5. Build scenario configs from profile routes
// 5. Build scenario configs from profile routes and config
const routes = profileDef?.routes ?? []
const scenarios = buildScenarioConfigs(routes, seed)
const fixtureScenarios = buildScenarioConfigs(routes, seed)
const configScenarios: ScenarioConfig[] = (config.scenarios ?? []).map(s => ({
name: s.name,
steps: s.steps.map(step => ({
name: step.name,
request: {
method: step.request.method as ScenarioConfig['steps'][0]['request']['method'],
url: step.request.url,
body: step.request.body,
headers: step.request.headers,
},
expect: step.expect,
capture: step.capture,
})),
}))
const scenarios = [...fixtureScenarios, ...configScenarios]
// 6. Build stateful config
const presetName = profileDef?.preset
@@ -603,6 +535,9 @@ export async function qualifyCommand(
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
? {
probability: 0.5,
strategy: preset.chaosStrategy as ChaosConfig['strategy'],
sampleSize: preset.chaosSampleSize,
sampleRoutes: preset.chaosSampleRoutes,
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
error: { probability: 0.2, statusCode: 503 },
dropout: { probability: 0.2, statusCode: 504 },
@@ -611,32 +546,41 @@ export async function qualifyCommand(
: undefined
// 8. Load the Fastify app for execution
// Try to import the app from the fixture
let fastify: FastifyAppLike | undefined
let appEntrypoint: string | undefined
try {
const appPath = resolve(workingDir, 'app.js')
const appUrl = pathToFileURL(appPath)
appUrl.searchParams.set('apophisRun', String(Date.now()))
const appModule = await import(appUrl.href)
fastify = (appModule.default || appModule) as FastifyAppLike
const { loadApp } = await import('../../core/app-loader.js')
const loaded = await loadApp(workingDir)
fastify = loaded.fastify as FastifyAppLike
appEntrypoint = loaded.entrypoint
if (fastify && typeof fastify.ready === 'function') {
await fastify.ready()
}
} catch (err) {
// App not available — return a result indicating no app to test
if (process.env.APOPHIS_DEBUG === '1') {
console.error('Failed to load app:', err)
}
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: 'No Fastify app found. Ensure app.js exports a Fastify instance or a factory function.\n\nSupported patterns:\n export default app\n export const createApp = () => app\n module.exports = app',
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
}
}
try {
// 9. Discover routes for chaos
// 9. Discover routes for chaos (with discovery metadata)
const appRoutes = await discoverAppRoutes(fastify)
// Collect discovery warnings
const { discoverRouteDetails } = await import('../../../domain/discovery.js')
const discoveryResult = discoverRouteDetails(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
const discoveryWarnings: string[] = [
...discoveryResult.warnings,
]
if (discoveryResult.source === 'print-routes' && !discoveryResult.hasSchemaMetadata) {
discoveryWarnings.push(
'Routes were discovered without schema metadata. Behavioral contracts and chaos injection may be limited. ' +
'Register APOPHIS before defining routes or use createFastify().'
)
}
// 10. Run qualify execution
const deps = {
fastify: fastify as any,
@@ -658,7 +602,7 @@ export async function qualifyCommand(
// 12. Signal quality guardrails — fail if zero checks executed
const execSummary = runResult.executionSummary
const warnings: string[] = [...artifact.warnings]
const warnings: string[] = [...artifact.warnings, ...discoveryWarnings]
if (execSummary.totalExecuted === 0) {
await emitArtifact(artifact, {
src/cli/commands/qualify/runner.ts
+91 -19
View File
@@ -48,6 +48,7 @@ export interface QualifyRunResult {
scenarioResults: ScenarioResult[]
statefulResult?: TestSuite
chaosResult?: ChaosRunResult
chaosResults: ChaosRunResult[]
stepTraces: StepTrace[]
cleanupFailures: CleanupFailure[]
durationMs: number
@@ -60,6 +61,8 @@ export interface QualifyRunResult {
scenariosRun: number
statefulTestsRun: number
chaosRunsRun: number
chaosRoutesPlanned: number
chaosRoutesExecuted: number
totalSteps: number
}
executedRoutes: string[]
@@ -71,6 +74,7 @@ export interface ChaosRunResult {
events: string[]
route: string
durationMs: number
contractsPassed: boolean
}
export interface CleanupFailure {
@@ -110,6 +114,45 @@ export function resolveProfileGates(features?: string[]): ProfileGates {
}
}
/**
* Select routes for chaos injection based on strategy.
*
* - 'one': picks one route deterministically (default, backward-compatible)
* - 'all': every provided route
* - 'sample': randomly sampled subset
* - 'routes': explicit list matched against route strings "METHOD /path"
*/
export function selectChaosRoutes(
allRoutes: RouteContract[],
config: ChaosConfig,
seed: number,
): RouteContract[] {
const strategy = config.strategy ?? 'one'
const rng = new SeededRng(seed)
if (strategy === 'all') {
return [...allRoutes]
}
if (strategy === 'sample') {
const size = Math.min(config.sampleSize ?? 3, allRoutes.length)
const shuffled = [...allRoutes].sort(() => rng.next() - 0.5)
return shuffled.slice(0, size)
}
if (strategy === 'routes' && config.sampleRoutes) {
return allRoutes.filter(r => {
const id = `${r.method} ${r.path}`
return config.sampleRoutes!.some(p => id === p)
})
}
// 'one' — default: pick one route deterministically
const idx = Math.floor(rng.next() * allRoutes.length)
const route = allRoutes[idx]
return route ? [route] : []
}
// ---------------------------------------------------------------------------
// Main qualify runner
// ---------------------------------------------------------------------------
@@ -125,13 +168,14 @@ export async function runQualify(
statefulConfig?: TestConfig,
chaosConfig?: ChaosConfig,
routes?: RouteContract[],
cleanupManager?: import('../../../infrastructure/cleanup-manager.js').CleanupManager,
): Promise<QualifyRunResult> {
const started = Date.now()
const scenarioResults: ScenarioResult[] = []
const allTraces: StepTrace[] = []
const cleanupFailures: CleanupFailure[] = []
let statefulResult: TestSuite | undefined
let chaosResult: ChaosRunResult | undefined
const chaosResults: ChaosRunResult[] = []
// Run scenarios
if (gates.scenario) {
@@ -149,37 +193,59 @@ export async function runQualify(
allTraces.push(...traces)
}
// Run chaos on routes
// Run chaos on selected routes
let chaosRoutesPlanned = 0
let chaosRoutesExecuted = 0
if (gates.chaos && chaosConfig && routes && routes.length > 0) {
// Pick one route deterministically for CLI chaos demo
const rng = new SeededRng(deps.seed)
const route = routes[Math.floor(rng.next() * routes.length)]
if (route) {
const { chaosResult: cr } = await runChaosOnRoute(deps, route, chaosConfig)
chaosResult = cr
const selectedRoutes = selectChaosRoutes(routes, chaosConfig, deps.seed)
chaosRoutesPlanned = selectedRoutes.length
for (const route of selectedRoutes) {
try {
const { chaosResult: cr } = await runChaosOnRoute(deps, route, chaosConfig)
chaosResults.push(cr)
if (cr.applied) {
chaosRoutesExecuted++
}
} catch {
// Individual chaos run failure should not abort the qualify run
}
}
}
// Simulate cleanup tracking
// In real usage, cleanupManager would be injected and tracked
// For now, cleanup failures are empty unless injected by caller
// Run cleanup and track failures
if (cleanupManager) {
try {
const outcomes = await cleanupManager.cleanup()
for (const outcome of outcomes) {
if (outcome.error) {
cleanupFailures.push({
resource: `${outcome.resource.type}/${outcome.resource.id} (${outcome.resource.url})`,
error: outcome.error,
})
}
}
} catch {
// Cleanup failures are tracked but should not block the qualify result
}
}
const durationMs = Date.now() - started
// Determine overall pass/fail
const scenarioPassed = scenarioResults.every(r => r.ok)
const statefulPassed = !statefulResult || statefulResult.summary.failed === 0
const chaosPassed = !chaosResult || chaosResult.applied // chaos "passes" if it applied
const chaosPassed = chaosResults.every(r => !r.applied || r.contractsPassed)
// Count execution metrics
const scenariosRun = scenarioResults.length
const statefulTestsRun = statefulResult?.tests.length ?? 0
const chaosRunsRun = chaosResult ? 1 : 0
const chaosRunsRun = chaosResults.length
const totalSteps = allTraces.length
const totalExecuted = scenariosRun + statefulTestsRun + chaosRunsRun
const totalPassed = scenarioResults.reduce((sum, r) => sum + r.summary.passed, 0) +
(statefulResult?.summary.passed ?? 0) +
(chaosResult?.applied ? 1 : 0)
chaosResults.reduce((sum, r) => sum + (r.applied && r.contractsPassed ? 1 : 0), 0)
const totalFailed = scenarioResults.reduce((sum, r) => sum + r.summary.failed, 0) +
(statefulResult?.summary.failed ?? 0)
@@ -204,9 +270,9 @@ export async function runQualify(
}
}
// Track chaos route
if (chaosResult) {
executedRoutes.push(chaosResult.route)
// Track chaos routes
for (const cr of chaosResults) {
executedRoutes.push(cr.route)
}
// Track skipped routes from profile filters
@@ -224,6 +290,9 @@ export async function runQualify(
reason = 'Stateful config missing or invalid'
} else if (gates.chaos && !chaosConfig) {
reason = 'Chaos config missing or invalid'
} else if (gates.chaos && chaosConfig) {
const strategy = chaosConfig.strategy ?? 'one'
reason = `Not selected by chaos strategy: ${strategy}`
}
skippedRoutes.push({ route: routeStr, reason })
}
@@ -234,19 +303,22 @@ export async function runQualify(
passed: scenarioPassed && statefulPassed && chaosPassed,
scenarioResults,
statefulResult,
chaosResult,
chaosResult: chaosResults[0],
chaosResults,
stepTraces: allTraces,
cleanupFailures,
durationMs,
seed: deps.seed,
executionSummary: {
totalPlanned: scenarios.length + (statefulConfig ? 1 : 0) + (chaosConfig && routes && routes.length > 0 ? 1 : 0),
totalPlanned: scenarios.length + (statefulConfig ? 1 : 0) + chaosRoutesPlanned,
totalExecuted,
totalPassed,
totalFailed,
scenariosRun,
statefulTestsRun,
chaosRunsRun,
chaosRoutesPlanned,
chaosRoutesExecuted,
totalSteps,
},
executedRoutes: [...new Set(executedRoutes)],
src/cli/commands/qualify/scenario-handler.ts
+1 -1
View File
@@ -10,7 +10,7 @@
* - No optional imports — everything is passed via parameters
*/
import { runScenario } from '../../../test/scenario-runner.js'
import { runScenario } from '../../../quality/scenario-runner.js'
import type {
ScenarioConfig,
ScenarioResult,
src/cli/commands/qualify/stateful-handler.ts
+1 -1
View File
@@ -10,7 +10,7 @@
* - No optional imports — everything is passed via parameters
*/
import { runStatefulTests } from '../../../test/stateful-runner.js'
import { runStatefulTests } from '../../../quality/stateful-runner.js'
import { CleanupManager } from '../../../infrastructure/cleanup-manager.js'
import type {
TestConfig,
src/cli/commands/replay/index.ts
+1 -1
View File
@@ -256,7 +256,7 @@ async function executeReplay(
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: `Cannot load Fastify app from ${workingDir}/app.js: ${errorMessage}`,
message: `Cannot load Fastify app: ${errorMessage}`,
warnings,
reproduced: false,
originalFailure: failure,
src/cli/commands/replay/loader.ts
+1 -1
View File
@@ -26,7 +26,7 @@ import type { Artifact, FailureRecord } from '../../core/types.js';
const SUPPORTED_ARTIFACT_VERSION = 'apophis-artifact/1';
/** Current CLI version for compatibility checks */
const CLI_VERSION = '2.0.0';
const CLI_VERSION = '2.7.0';
// ---------------------------------------------------------------------------
// Types
src/cli/commands/verify/index.ts
+15 -104
View File
@@ -117,20 +117,26 @@ function buildArtifact(
}
return {
route,
contract: f.contract,
contract: f.formula || f.contract,
expected: f.expected,
observed: f.observed,
seed: options.seed,
replayCommand: `apophis replay --artifact ${f.artifactPath || '<artifact-path-unavailable>'}`,
category: f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME,
category: f.category ?? (f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME),
}
})
if (runResult.noContractsFound) {
warnings.push('No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.')
if (runResult.discoveryWarnings && runResult.discoveryWarnings.length > 0) {
warnings.push(...runResult.discoveryWarnings)
}
}
if (runResult.noRoutesMatched) {
warnings.push(`No routes matched the filter. Available routes: ${runResult.availableRoutes?.join(', ') || 'none'}`)
if (runResult.discoveryWarnings && runResult.discoveryWarnings.length > 0) {
warnings.push(...runResult.discoveryWarnings)
}
}
if (runResult.notGitRepo) {
warnings.push('--changed requires a git repository. Current directory is not inside a git repo.')
@@ -149,7 +155,7 @@ function buildArtifact(
return {
version: 'apophis-artifact/1',
cliVersion: '2.0.0',
cliVersion: '2.7.0',
command: 'verify',
mode: 'verify',
cwd: options.cwd,
@@ -255,106 +261,6 @@ function formatHumanFailure(failure: FailureRecord, profile?: string): string {
}
/**
* Format human-readable output for verify results.
*/
function formatHumanOutput(
runResult: VerifyRunResult,
options: { profile?: string; seed: number; env: string; routeFilters?: string[] },
): string {
const lines: string[] = []
if (runResult.notGitRepo) {
lines.push(`--changed requires a git repository.`)
lines.push(`Current directory is not inside a git repo.`)
lines.push('')
lines.push('Next:')
lines.push(` Initialize git with \`git init\`, or run verify without --changed.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noRelevantChanges) {
lines.push(`No relevant changes detected.`)
lines.push(`Git shows no modified files that match any route.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noRoutesMatched) {
lines.push(`No routes matched the filter.`)
lines.push(`Filters applied: ${options.routeFilters?.join(', ') || 'none'}`)
lines.push(`Available routes:`)
for (const r of runResult.availableRoutes || []) {
lines.push(` ${r}`)
}
lines.push('')
lines.push('Next:')
lines.push(` Adjust --routes filter or add routes to your app.`)
lines.push('')
return lines.join('\n')
}
if (runResult.noContractsFound) {
lines.push('No behavioral contracts found.')
lines.push('')
lines.push('APOPHIS discovered routes, but none have behavioral contracts.')
lines.push('Schema-only routes (with response schemas) are not enough.')
lines.push('You must add x-ensures or x-requires clauses that check behavior.')
lines.push('')
lines.push('Example — add this to your route schema:')
lines.push(' "x-ensures": [')
lines.push(' "response_code(GET /users/{response_body(this).id}) == 200"')
lines.push(' ]')
lines.push('')
lines.push('Next steps:')
lines.push(' 1. Open your route file (e.g., app.js or src/routes/users.js)')
lines.push(' 2. Find the route you want to test')
lines.push(' 3. Add an "x-ensures" array inside the schema object')
lines.push(' 4. Run: apophis verify --profile quick --routes "POST /users"')
lines.push('')
lines.push('For more examples, see docs/getting-started.md')
lines.push('')
return lines.join('\n')
}
// Print failures using canonical format
for (const failure of runResult.failures) {
const failureRecord: FailureRecord = {
route: failure.route,
contract: failure.contract,
expected: failure.expected,
observed: failure.observed,
seed: options.seed,
replayCommand: `apophis replay --artifact ${failure.artifactPath || 'reports/apophis/failure-*.json'}`,
}
lines.push(formatHumanFailure(failureRecord, options.profile))
lines.push('')
}
// Summary
if (runResult.passed) {
lines.push(`All ${runResult.total} contract(s) passed.`)
} else {
lines.push(`Failed: ${runResult.failed} of ${runResult.total} contract(s) failed.`)
}
lines.push(`Seed: ${options.seed}`)
// Replay command on failure
if (!runResult.passed && runResult.failures.length > 0) {
lines.push('')
lines.push('Replay')
lines.push(` apophis replay --artifact <path-to-artifact>`)
lines.push('')
lines.push('Determinism')
lines.push(` This run used seed ${options.seed}.`)
lines.push(` Same seed + same app state = same results.`)
lines.push(` If results differ on re-run, the app has nondeterministic behavior.`)
lines.push(` Stabilize: reset app state, mock external services, avoid time-dependent logic.`)
}
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
@@ -454,10 +360,12 @@ export async function verifyCommand(
// 5. Load the Fastify app
let fastify: unknown
let appEntrypoint: string | undefined
try {
const { loadApp } = await import('../../core/app-loader.js')
const loaded = await loadApp(workingDir)
fastify = loaded.fastify
appEntrypoint = loaded.entrypoint
if (fastify && typeof (fastify as any).ready === 'function') {
await (fastify as any).ready()
}
@@ -465,7 +373,7 @@ export async function verifyCommand(
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: `No Fastify app found. Ensure app.js exports a Fastify instance or a factory function.\n\nSupported patterns:\n export default app\n export const createApp = () => app\n module.exports = app\n\nError: ${errorMessage}\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
}
}
@@ -479,6 +387,9 @@ export async function verifyCommand(
routeFilters,
changed,
profileRoutes: config.profiles?.[profile || '']?.routes,
runs: typeof config.presets?.[loadResult.presetName || '']?.runs === 'number'
? (config.presets[loadResult.presetName || ''] as { runs?: number }).runs
: undefined,
})
// 7. Build artifact
src/cli/commands/verify/runner.ts
+129 -29
View File
@@ -20,6 +20,10 @@ import { executeHttp } from '../../../infrastructure/http-executor.js'
import { parse } from '../../../formula/parser.js'
import { evaluateAsync } from '../../../formula/evaluator.js'
import { createOperationResolver } from '../../../formula/runtime.js'
import { buildRequest } from '../../../domain/request-builder.js'
import { convertSchema } from '../../../domain/schema-to-arbitrary.js'
import * as fc from 'fast-check'
import { resolveRuns } from '../../../types.js'
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
import type { RouteResult } from '../../core/types.js'
@@ -33,6 +37,8 @@ export interface VerifyFailure {
expected: string
observed: string
artifactPath?: string
formula?: string
category?: string
}
export interface VerifyRunResult {
@@ -48,6 +54,8 @@ export interface VerifyRunResult {
noRelevantChanges?: boolean
availableRoutes?: string[]
artifactPaths: string[]
discoveryWarnings?: string[]
runs: number
}
export interface VerifyRunnerDeps {
@@ -57,6 +65,7 @@ export interface VerifyRunnerDeps {
routeFilters?: string[]
changed?: boolean
profileRoutes?: string[]
runs?: number
}
// ---------------------------------------------------------------------------
@@ -203,28 +212,58 @@ async function filterChangedRoutes(
/**
* Build a request for a route.
*/
function buildRouteRequest(route: RouteContract): {
function buildRouteRequest(
route: RouteContract,
seed?: number,
runIndex?: number,
variant?: { name: string; headers?: Record<string, string> },
): {
method: string
url: string
body?: unknown
query?: Record<string, string>
headers: Record<string, string>
} {
const headers: Record<string, string> = {
'content-type': 'application/json',
let generatedData: Record<string, unknown> = {}
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
if (bodySchema && seed !== undefined) {
try {
const bodyArb = convertSchema(bodySchema, { context: 'request' })
const bodySeed = seed + (runIndex ?? 0) * 31
const samples = fc.sample(bodyArb, { numRuns: 1, seed: bodySeed })
const bodySample = samples[0]
if (bodySample !== null && typeof bodySample === 'object') {
generatedData = bodySample as Record<string, unknown>
}
} catch {
// fall through to example-based generation
}
}
// Build body from schema if available
let body: unknown = undefined
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
if (bodySchema && route.method === 'POST') {
body = buildExampleBody(bodySchema)
if (!generatedData || Object.keys(generatedData).length === 0) {
generatedData = buildExampleBody(bodySchema ?? {}) as Record<string, unknown> ?? {}
}
const request = buildRequest(
route,
generatedData,
variant?.headers ?? ({} as Record<string, string>),
{ resources: new Map(), counters: new Map() },
)
const headers: Record<string, string> = {
'content-type': 'application/json',
...request.headers,
...(variant?.headers ?? {}),
}
return {
method: route.method,
url: route.path,
body,
method: request.method,
url: request.url,
body: request.body,
headers,
query: request.query,
}
}
@@ -268,18 +307,62 @@ function buildExampleValue(schema: Record<string, unknown>): unknown {
return undefined
}
/**
* Execute a single contract for a route.
* Returns the evaluation context and any failure.
*/
function buildFailureDiagnostic(
contract: string,
evalCtx: EvalContext,
evalError?: string,
): Pick<VerifyFailure, 'expected' | 'observed' | 'formula' | 'category'> {
if (evalError) {
return { expected: 'true', observed: evalError, formula: contract, category: 'runtime' }
}
const status = evalCtx.response.statusCode
const statusMatch = contract.match(/^status:(\d+)$/i)
if (statusMatch) {
return { expected: `HTTP ${statusMatch[1]}`, observed: `HTTP ${status}`, formula: contract, category: 'runtime' }
}
const fieldMatch = contract.match(/response_body\(this\)\.([\w.]+)\s*(!=|==)\s*null/i)
if (fieldMatch) {
const path = fieldMatch[1]!
const negated = fieldMatch[2] === '!='
const parts = path.split('.')
let value: unknown = evalCtx.response.body
for (const p of parts) {
if (value != null && typeof value === 'object') {
value = (value as Record<string, unknown>)[p]
} else { value = undefined; break }
}
const actual = value === undefined ? 'undefined' : value === null ? 'null' : `"${String(value)}"`
return {
expected: negated ? `${path} != null` : `${path} == null`,
observed: `${path} is ${actual}`,
formula: contract, category: 'runtime',
}
}
const codeMatch = contract.match(/response_code\(this\)\s*(==|!=|<|>|<=|>=)\s*(\d+)/i)
if (codeMatch) {
return {
expected: `response_code == ${codeMatch[2]}`,
observed: `response_code is ${status}`,
formula: contract, category: 'runtime',
}
}
return { expected: contract, observed: `false`, formula: contract, category: 'runtime' }
}
async function executeContract(
fastify: FastifyInjectInstance,
route: RouteContract,
contract: string,
timeout?: number,
variant?: { name: string; headers?: Record<string, string> },
seed?: number,
runIndex?: number,
): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> {
const request = buildRouteRequest(route)
const request = buildRouteRequest(route, seed, runIndex, variant)
// Merge variant headers if provided
const headers = variant?.headers
@@ -307,6 +390,7 @@ async function executeContract(
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
const diagnostic = buildFailureDiagnostic(contract, evalCtx, result.success ? undefined : result.error)
return {
ctx: evalCtx,
failure: {
@@ -314,14 +398,17 @@ async function executeContract(
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: result.success ? String(result.value) : result.error,
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
},
}
}
return { ctx: evalCtx }
} catch (error) {
const diagnostic = buildFailureDiagnostic(contract, evalCtx, error instanceof Error ? error.message : String(error))
return {
ctx: evalCtx,
failure: {
@@ -329,8 +416,10 @@ async function executeContract(
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract,
expected: 'true',
observed: error instanceof Error ? error.message : String(error),
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
},
}
}
@@ -397,9 +486,10 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noRoutesMatched: false,
noContractsFound: false,
availableRoutes,
artifactPaths: [],
notGitRepo: true,
}
artifactPaths: [],
notGitRepo: true,
runs: 0,
}
}
routes = await filterChangedRoutes(routes, cwd)
}
@@ -417,6 +507,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noContractsFound: false,
availableRoutes,
artifactPaths: [],
runs: 0,
}
}
@@ -437,10 +528,13 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noContractsFound: true,
availableRoutes,
artifactPaths: [],
runs: 0,
}
}
// 4. Execute contracts (with variant expansion)
const runConfig = resolveRuns(deps.runs)
const runs = runConfig.contractRuns
const failures: VerifyFailure[] = []
let total = 0
let passedCount = 0
@@ -453,13 +547,18 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
for (const variant of variants) {
for (const contract of contracts) {
total++
const result = await executeContract(fastify, route, contract, deps.timeout, variant)
for (let runIndex = 0; runIndex < runs; runIndex++) {
total++
const result = await executeContract(
fastify, route, contract, deps.timeout,
variant, deps.seed, runIndex,
)
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
}
}
}
}
@@ -485,5 +584,6 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
noContractsFound: false,
availableRoutes,
artifactPaths: [],
runs,
}
}
src/cli/core/app-loader.ts
+157 -45
View File
@@ -1,99 +1,211 @@
/**
* App loader utility for CLI commands.
* Handles various app export patterns and module systems.
*
* Supports auto-detection of entrypoint files:
* app.js, app.ts, server.js, server.ts, index.js, index.ts,
* src/app.js, src/app.ts, src/server.js, src/server.ts,
* src/index.js, src/index.ts
*/
import { resolve } from 'node:path'
import { existsSync } from 'node:fs'
import { resolve, extname } from 'node:path'
import { pathToFileURL } from 'node:url'
function isTsxAvailable(): boolean {
return process.execArgv.some(arg => arg.includes('tsx') || arg.includes('ts-node')) ||
process.argv[0]?.includes('tsx') ||
!!process.env.TSX_TSCONFIG_PATH
}
export const APP_CANDIDATES = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
]
export interface LoadedApp {
fastify: unknown
source: 'default' | 'named' | 'commonjs'
entrypoint: string
}
/**
* Load a Fastify app from app.js in the given directory.
* Supports:
* Find the first existing app entrypoint file in the given directory.
* Returns the resolved absolute path, or null if no candidate exists.
*/
export function findAppFile(cwd: string): string | null {
for (const candidate of APP_CANDIDATES) {
const fullPath = resolve(cwd, candidate)
if (existsSync(fullPath)) {
return fullPath
}
}
return null
}
/**
* Load a Fastify app from the given directory.
*
* Auto-detects the entrypoint by searching APP_CANDIDATES for the first
* existing file. Supports:
* - ESM default export: export default fastifyInstance
* - ESM named export: export const createApp = () => fastifyInstance
* - CommonJS: module.exports = fastifyInstance
* - CommonJS named: exports.createApp = () => fastifyInstance
*
* If an explicit entrypoint is provided, it takes precedence over auto-detection.
* For .ts entrypoints, a clear error message suggests installing tsx.
*
* After loading the app, this function ensures route discovery works even if the
* APOPHIS plugin is not registered or was registered after routes.
*/
export async function loadApp(cwd: string): Promise<LoadedApp> {
const appPath = resolve(cwd, 'app.js')
const appUrl = pathToFileURL(appPath).href + '?t=' + Date.now()
export async function loadApp(cwd: string, entrypoint?: string): Promise<LoadedApp> {
const resolvedEntrypoint = entrypoint
? resolve(cwd, entrypoint)
: findAppFile(cwd)
if (!resolvedEntrypoint) {
throw new AppLoadError(
`No Fastify app entrypoint found. Searched for: ${APP_CANDIDATES.join(', ')}.\n` +
'Create an app.js, server.js, or similar that exports a Fastify instance.',
'not_found',
)
}
const ext = extname(resolvedEntrypoint)
const isTypeScript = ext === '.ts'
if (isTypeScript && !isTsxAvailable()) {
throw new AppLoadError(
`TypeScript entrypoint ${resolvedEntrypoint} requires a TS loader.\n` +
'Install tsx and run:\n' +
' npm install -D tsx\n' +
' npx tsx ./node_modules/.bin/apophis verify\n\n' +
'Or convert your entrypoint to JavaScript.',
'import_failed',
)
}
const appUrl = pathToFileURL(resolvedEntrypoint).href + '?t=' + Date.now()
let appModule: Record<string, unknown>
try {
appModule = await import(appUrl) as Record<string, unknown>
} catch (err) {
const errMessage = err instanceof Error ? err.message : String(err)
if (isTypeScript) {
throw new AppLoadError(
`Cannot load TypeScript entrypoint ${resolvedEntrypoint}: ${errMessage}\n\n` +
'TypeScript entrypoints require tsx. Install tsx and run:\n' +
' npm install -D tsx\n' +
' npx tsx ./node_modules/.bin/apophis verify\n\n' +
'Or convert your entrypoint to JavaScript.',
'import_failed',
)
}
throw new AppLoadError(
`Cannot load app.js: ${err instanceof Error ? err.message : String(err)}`,
`Cannot load ${resolvedEntrypoint}: ${errMessage}`,
'import_failed',
)
}
// Try default export first
let fastify: unknown
let source: 'default' | 'named' | 'commonjs' = 'default'
if (appModule.default && isFastifyInstance(appModule.default)) {
return { fastify: appModule.default, source: 'default' }
}
fastify = appModule.default
source = 'default'
} else {
let found = false
for (const [key, value] of Object.entries(appModule)) {
if (key === 'default') continue
// Try named exports that look like Fastify instances or factory functions
for (const [key, value] of Object.entries(appModule)) {
if (key === 'default') continue
if (isFastifyInstance(value)) {
fastify = value
source = 'named'
found = true
break
}
if (isFastifyInstance(value)) {
return { fastify: value, source: 'named' }
}
// Try calling factory functions
if (typeof value === 'function' && !isClass(value)) {
try {
const result = await value()
if (isFastifyInstance(result)) {
return { fastify: result, source: 'named' }
if (typeof value === 'function' && !isClass(value)) {
try {
const result = await value()
if (isFastifyInstance(result)) {
fastify = result
source = 'named'
found = true
break
}
} catch {
// Factory function failed, try next
}
} catch {
// Factory function failed, try next
}
}
if (!found && isFastifyInstance(appModule)) {
fastify = appModule
source = 'commonjs'
found = true
}
if (!found) {
throw new AppLoadError(
`No Fastify instance found in ${resolvedEntrypoint}. ` +
'Ensure the file exports a Fastify instance or a factory function.\n\n' +
'Supported patterns:\n' +
' export default app\n' +
' export { app }\n' +
' export const createApp = () => app\n' +
' module.exports = app',
'no_fastify',
)
}
}
// If module itself is a Fastify instance (CommonJS)
if (isFastifyInstance(appModule)) {
return { fastify: appModule, source: 'commonjs' }
}
await ensureRouteDiscovery(fastify as Record<string, unknown>)
throw new AppLoadError(
'No Fastify instance found in app.js. Ensure app.js exports a Fastify instance or a factory function.',
'no_fastify',
)
return { fastify, source, entrypoint: resolvedEntrypoint }
}
/**
* Check if a value looks like a Fastify instance.
* Install a lightweight onRoute hook for route discovery.
* This runs before avvio's boot phase, so routes registered
* inside deferred plugins will be captured even if apophis
* is not registered first.
*/
async function ensureRouteDiscovery(f: Record<string, unknown>): Promise<void> {
if (typeof f.addHook !== 'function') return
try {
const { installRouteDiscovery } = await import('../../infrastructure/discovery-hook.js')
installRouteDiscovery(f as { addHook: Function })
} catch {
// discovery module not importable, skip auto-capture
}
}
function isFastifyInstance(value: unknown): boolean {
return value !== null &&
typeof value === 'object' &&
typeof (value as Record<string, unknown>).ready === 'function'
if (value === null || typeof value !== 'object') return false
const obj = value as Record<string, unknown>
return typeof obj.ready === 'function'
}
/**
* Check if a function is a class constructor.
*/
function isClass(fn: unknown): boolean {
return typeof fn === 'function' &&
fn.toString().startsWith('class ')
}
/**
* Error type for app loading failures.
*/
export class AppLoadError extends Error {
constructor(
message: string,
public readonly code: 'import_failed' | 'no_fastify',
public readonly code: 'import_failed' | 'no_fastify' | 'not_found',
) {
super(message)
this.name = 'AppLoadError'
src/cli/core/config-loader.ts
+63 -3
View File
@@ -30,6 +30,7 @@ export interface Config {
environments?: Record<string, EnvironmentPolicy>;
profiles?: Record<string, ProfileDefinition>;
presets?: Record<string, PresetDefinition>;
scenarios?: ScenarioConfigDef[];
[key: string]: unknown;
}
@@ -53,9 +54,28 @@ export interface PresetDefinition {
routes?: string[];
seed?: number;
features?: string[];
chaos?: boolean;
chaosStrategy?: 'one' | 'all' | 'sample' | 'routes';
chaosSampleSize?: number;
chaosSampleRoutes?: string[];
[key: string]: unknown;
}
export interface ScenarioConfigDef {
name: string;
steps: Array<{
name: string;
request: {
method: string;
url: string;
body?: unknown;
headers?: Record<string, string>;
};
expect: string[];
capture?: Record<string, string>;
}>;
}
export interface LoadConfigOptions {
cwd: string;
configPath?: string;
@@ -111,6 +131,20 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
optional: true,
items: { type: 'string' },
},
metadata: {
type: 'object',
optional: true,
},
scenarios: {
type: 'array',
optional: true,
items: { type: 'object' },
},
chaos: {
type: 'object',
optional: true,
properties: {},
},
};
// Schema for EnvironmentPolicy values (inside environments.<name>)
@@ -140,6 +174,10 @@ const PROFILE_SCHEMA: Record<string, SchemaField> = {
sampling: { type: 'number', optional: true },
blocking: { type: 'boolean', optional: true },
sinks: { type: 'object', optional: true },
chaos: { type: 'object', optional: true },
chaosStrategy: { type: 'string', optional: true, enumValues: ['one', 'all', 'sample', 'routes'] },
chaosSampleSize: { type: 'number', optional: true, min: 1 },
chaosSampleRoutes: { type: 'array', optional: true, items: { type: 'string' } },
};
// Schema for PresetDefinition values (inside presets.<name>)
@@ -148,6 +186,9 @@ const PRESET_SCHEMA: Record<string, SchemaField> = {
timeout: { type: 'number', optional: true, min: 0 },
parallel: { type: 'boolean', optional: true },
chaos: { type: 'boolean', optional: true },
chaosStrategy: { type: 'string', optional: true, enumValues: ['one', 'all', 'sample', 'routes'] },
chaosSampleSize: { type: 'number', optional: true, min: 1 },
chaosSampleRoutes: { type: 'array', optional: true, items: { type: 'string' } },
observe: { type: 'boolean', optional: true },
features: { type: 'array', optional: true, items: { type: 'string' } },
sampling: { type: 'number', optional: true },
@@ -201,7 +242,7 @@ export function loadPackageJsonConfig(cwd: string): { config: Config | null; pat
/**
* Load a config file by path.
* Supports .js, .ts (via dynamic import, assumes tsx available), and .json.
* Supports .js, .ts (via dynamic import when tsx is available), and .json.
*/
export async function loadConfigFile(configPath: string): Promise<Config> {
if (configPath.endsWith('.json')) {
@@ -209,6 +250,14 @@ export async function loadConfigFile(configPath: string): Promise<Config> {
return JSON.parse(content) as Config;
}
if (configPath.endsWith('.ts') && !process.execArgv.some(a => a.includes('tsx') || a.includes('ts-node'))) {
throw new Error(
`TypeScript config file ${configPath} requires a TS loader.\n` +
'Convert to apophis.config.js or apophis.config.json, or run through tsx:\n' +
' npx tsx ./node_modules/.bin/apophis verify'
);
}
// For .js and .ts, use dynamic import.
// tsx handles .ts files in dev environments.
const fileUrl = pathToFileURL(configPath).href;
@@ -259,6 +308,15 @@ function isInsideDynamicContainer(path: string): boolean {
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.');
}
/**
* User-managed extension keys that pass through schema validation.
* APOPHIS will never interpret these fields; they are reserved for team metadata.
* Keys starting with 'x-' are always allowed at any level.
*/
function isUserExtensionKey(key: string): boolean {
return key.startsWith('x-')
}
/**
* Validate that a value matches the expected type for a schema field.
* Throws ConfigValidationError on type mismatch.
@@ -406,13 +464,14 @@ export function validateConfigAgainstSchema(
}
}
} else {
if (isUserExtensionKey(key)) continue
// Unknown key inside a profile/preset/environment object
throw new ConfigValidationError(
`Unknown config key at ${currentPath}`,
currentPath,
key,
obj[key],
`Valid keys for ${parentContainer} entries: ${Object.keys(childSchema || {}).join(', ')}.`,
`Valid keys for ${parentContainer} entries: ${Object.keys(childSchema || {}).join(', ')}. Use x- prefixed keys for team metadata.`,
);
}
}
@@ -420,12 +479,13 @@ export function validateConfigAgainstSchema(
}
if (!fieldSchema) {
if (isUserExtensionKey(key)) continue
throw new ConfigValidationError(
`Unknown config key at ${currentPath}`,
currentPath,
key,
obj[key],
`Valid top-level keys: ${Object.keys(CONFIG_SCHEMA).join(', ')}.`,
`Valid top-level keys: ${Object.keys(CONFIG_SCHEMA).join(', ')}. Use the metadata field or x- prefixed keys for team data.`,
);
}
src/cli/core/index.ts
+5 -5
View File
@@ -2,7 +2,7 @@ import { cac } from 'cac';
import pc from 'picocolors';
import { createContext, type CliContext } from './context.js';
const CLI_VERSION = '2.0.0';
const CLI_VERSION = '2.7.0';
const HELP_HEADER = `
${pc.bold('apophis')} — Contract-driven API testing for Fastify
@@ -23,7 +23,7 @@ const HELP_HEADER = `
--config <path> Config file path
--profile <name> Profile name from config
--cwd <path> Working directory override
--format <mode> Output format: human | json | ndjson (default: human)
--format <mode> Output format: human | json | ndjson | json-summary | ndjson-summary (default: human)
--color <mode> Color mode: auto | always | never (default: auto)
--quiet Suppress non-error output
--verbose Enable verbose logging
@@ -223,7 +223,7 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
cli.option('--config <path>', 'Config file path');
cli.option('--profile <name>', 'Profile name from config');
cli.option('--cwd <path>', 'Working directory override');
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
cli.option('--format <mode>', 'Output format: human | json | ndjson | json-summary | ndjson-summary', { default: 'human' });
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
cli.option('--quiet', 'Suppress non-error output');
cli.option('--verbose', 'Enable verbose logging');
@@ -372,11 +372,11 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
]);
const commandSpecificFlags: Record<string, Set<string>> = {
init: new Set(['--preset', '--force', '--noninteractive']),
init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']),
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config', '--workspace']),
qualify: new Set(['--profile', '--seed', '--workspace']),
replay: new Set(['--artifact']),
replay: new Set(['--artifact', '--route']),
doctor: new Set(['--mode', '--strict', '--workspace']),
migrate: new Set(['--check', '--dry-run', '--write']),
};
src/cli/core/types.ts
+5
View File
@@ -299,6 +299,11 @@ export interface Artifact {
failed: number;
};
executionSummary?: ExecutionSummary;
coverageBreakdown?: {
scenario: { routesCovered: string[]; stepsTotal: number; stepsPassed: number }
stateful: { routesCovered: string[]; testsTotal: number; testsPassed: number }
chaos: { routesPlanned: string[]; routesExecuted: string[]; runsTotal: number; runsPassed: number }
}
executedRoutes?: string[];
skippedRoutes?: RouteExecutionInfo[];
stepTraces?: StepTrace[];
src/cli/renderers/human.ts
+2 -26
View File
@@ -21,7 +21,7 @@ import { shouldUseColor, getColors, truncate, indent, formatDuration } from './s
// Types
// ---------------------------------------------------------------------------
export interface HumanRendererOptions {
interface HumanRendererOptions {
ctx: OutputContext;
profile?: string;
seed?: number;
@@ -180,34 +180,10 @@ function generateNextSteps(failure: FailureRecord): string {
}
// ---------------------------------------------------------------------------
// Progress and summary rendering
// ---------------------------------------------------------------------------
/**
* Render a simple ASCII progress bar.
*/
function renderProgressBar(
current: number,
total: number,
width: number,
ctx: OutputContext,
): string {
const c = getColorizer(ctx);
if (total === 0) return c.dim('[' + ' '.repeat(width) + ']');
const filled = Math.round((current / total) * width);
const empty = width - filled;
const filledChar = '█';
const emptyChar = '░';
return '[' + c.green(filledChar.repeat(filled)) + c.dim(emptyChar.repeat(empty)) + ']';
}
/**
* Render summary for verify/observe/qualify results.
*/
export function renderSummary(
function renderSummary(
artifact: Artifact,
ctx: OutputContext,
): string {
src/cli/renderers/json.ts
+1 -19
View File
@@ -14,7 +14,7 @@ import type { Artifact, CommandResult } from '../core/types.js';
// Types
// ---------------------------------------------------------------------------
export interface JsonRendererOptions {
interface JsonRendererOptions {
indent?: number;
}
@@ -141,24 +141,6 @@ export function renderJsonArtifact(
* If an artifact is present, it is rendered.
* Otherwise, a minimal JSON with the message and exit code is returned.
*/
export function renderJsonResult(
result: CommandResult,
options: JsonRendererOptions = {},
): string {
if (result.artifact) {
return renderJsonArtifact(result.artifact, options);
}
// Minimal JSON for results without artifacts
const minimal = {
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
};
return JSON.stringify(minimal, null, options.indent ?? 2);
}
/**
* Render a concise summary artifact for CI/machine parsers.
* Omits stepTraces, cleanupOutcomes, and profileGates to reduce noise.
src/cli/renderers/ndjson.ts
+2 -23
View File
@@ -14,7 +14,7 @@ import type { Artifact, FailureRecord, NdjsonEvent } from '../core/types.js';
// Types
// ---------------------------------------------------------------------------
export interface NdjsonRendererOptions {
interface NdjsonRendererOptions {
/** Output stream to write to (defaults to process.stdout) */
output?: NodeJS.WriteStream;
}
@@ -118,7 +118,7 @@ export function renderNdjsonEvent(event: NdjsonEvent): string {
* Write an NDJSON event to the output stream.
* Flushes after each write.
*/
export function writeNdjsonEvent(
function writeNdjsonEvent(
event: NdjsonEvent,
options: NdjsonRendererOptions = {},
): void {
@@ -216,25 +216,4 @@ export function renderNdjsonSummaryArtifact(
writeNdjsonEvent(createRunCompletedEvent(artifact.summary), options);
}
/**
* Create concise NDJSON events for an artifact without writing.
* Useful for testing summary mode.
*/
export function createNdjsonSummaryEvents(artifact: Artifact): NdjsonEvent[] {
const events: NdjsonEvent[] = [];
events.push(createRunStartedEvent(artifact.command, artifact.seed));
events.push({
type: 'run.summary',
summary: artifact.summary,
executionSummary: artifact.executionSummary,
profileGates: artifact.profileGates,
deterministicParams: artifact.deterministicParams,
timestamp: getTimestamp(),
} as unknown as NdjsonEvent);
events.push(createRunCompletedEvent(artifact.summary));
return events;
}
src/cli/renderers/shared.ts
+1 -71
View File
@@ -15,7 +15,7 @@ import pc from 'picocolors';
// Types
// ---------------------------------------------------------------------------
export interface TruncationOptions {
interface TruncationOptions {
maxLength?: number;
suffix?: string;
}
@@ -77,15 +77,6 @@ export function truncate(str: string, options: TruncationOptions = {}): string {
return str.slice(0, truncatedLength) + suffix;
}
/**
* Truncate an object for terminal display.
* Converts to JSON and truncates.
*/
export function truncateObject(obj: unknown, options: TruncationOptions = {}): string {
const str = typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2);
return truncate(str, options);
}
// ---------------------------------------------------------------------------
// Indentation
// ---------------------------------------------------------------------------
@@ -113,13 +104,6 @@ export function formatDuration(ms: number): string {
return `${(ms / 1000).toFixed(2)}s`;
}
/**
* Format a timestamp as ISO string.
*/
export function formatTimestamp(date?: Date): string {
return (date || new Date()).toISOString();
}
/**
* Strip ANSI escape codes from a string.
*/
@@ -136,58 +120,4 @@ export function hasAnsi(str: string): boolean {
return /\u001b\[[0-9;]*m/.test(str);
}
// ---------------------------------------------------------------------------
// TTY/CI output helpers
// ---------------------------------------------------------------------------
/**
* Determine if spinners should be shown.
* Never show spinners in CI or non-TTY environments.
*/
export function shouldShowSpinner(ctx: OutputContext): boolean {
return ctx.isTTY && !ctx.isCI;
}
/**
* Write to stdout with optional flushing.
* In non-TTY mode, always flush.
*/
export function writeStdout(str: string): void {
process.stdout.write(str);
}
/**
* Write line to stdout.
*/
export function writeLine(str: string = ''): void {
process.stdout.write(str + '\n');
}
// ---------------------------------------------------------------------------
// Progress helpers
// ---------------------------------------------------------------------------
/**
* Format a progress indicator (no spinner, just text).
* Safe for CI/non-TTY.
*/
export function formatProgress(current: number, total: number, label?: string): string {
const pct = total > 0 ? Math.round((current / total) * 100) : 0;
const prefix = label ? `${label} ` : '';
return `${prefix}[${current}/${total}] ${pct}%`;
}
// ---------------------------------------------------------------------------
// Summary formatting
// ---------------------------------------------------------------------------
/**
* Format a summary block for human output.
*/
export function formatSummary(total: number, passed: number, failed: number): string {
const lines: string[] = [];
lines.push(`Total: ${total}`);
lines.push(`Passed: ${passed}`);
lines.push(`Failed: ${failed}`);
return lines.join('\n');
}