/** * S6: Qualify thread - Qualify command handler * * Responsibilities: * - Load config and resolve profile * - Block prod runs by default (policy engine) * - Run scenario/stateful/chaos based on profile * - Generate seed if omitted, always print it * - Rich artifact emission with step traces * - Handle cleanup failures separately * - Exit 0 on pass, 1 on qualification failure, 2 on safety violation * * Architecture: * - Dependency injection: all dependencies passed explicitly * - No optional imports — everything is required or injected * - Inline comments for documentation */ import type { CliContext } from '../../core/context.js' import { loadConfig } from '../../core/config-loader.js' import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js' import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js' import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js' import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js' import { runQualify, resolveProfileGates, type QualifyRunResult, type StepTrace, type CleanupFailure, } from './runner.js' import { isGitRepo, filterChangedRoutes } from '../verify/runner.js' import { SeededRng } from '../../../infrastructure/seeded-rng.js' import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js' import { renderHumanArtifact } from '../../renderers/human.js' import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js' import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js' import type { OutputContext } from '../../renderers/shared.js' import { resolve } from 'node:path' const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/ function normalizeRouteIdentity(route: string): string { const normalized = route.trim().replace(/\s+/g, ' ') const [method, ...pathParts] = normalized.split(' ') if (!method || pathParts.length === 0) { return normalized } return `${method.toUpperCase()} ${pathParts.join(' ')}` } function isReplayCompatibleRoute(route: string): boolean { return ROUTE_IDENTITY_PATTERN.test(route) } function coerceTimeout(value: unknown): number | undefined { return typeof value === 'number' ? value : undefined } // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- export interface QualifyOptions { profile?: string seed?: number changed?: boolean config?: string cwd?: string format?: 'human' | 'json' | 'ndjson' quiet?: boolean verbose?: boolean artifactDir?: string } interface FastifyAppLike { ready?: () => Promise close?: () => Promise } // --------------------------------------------------------------------------- // Seed generation // --------------------------------------------------------------------------- /** * Generate a deterministic seed if none provided. * Uses current time + process pid + counter for uniqueness. */ let seedCounter = 0 export function generateSeed(): number { seedCounter++ return Date.now() + (process.pid || 0) + seedCounter } // --------------------------------------------------------------------------- // Route discovery helper // --------------------------------------------------------------------------- /** * Discover routes from the Fastify app for chaos execution. * Uses the shared discovery module for capture + fallback support. */ async function discoverAppRoutes(fastify: unknown): Promise { const { discoverRoutes } = await import('../../../domain/discovery.js') return discoverRoutes(fastify as { routes?: Array<{ method: string; url: string; schema?: Record } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string }) } // --------------------------------------------------------------------------- // Scenario builder from profile // --------------------------------------------------------------------------- /** * Build scenario configs from profile routes for protocol-lab fixture. * Creates an OAuth-like multi-step scenario. */ function buildScenarioConfigs(routes: string[], seed: number): ScenarioConfig[] { // For the protocol-lab fixture, build the OAuth scenario const hasOAuth = routes.some(r => r.includes('/oauth/authorize')) if (!hasOAuth) return [] const rng = new SeededRng(seed) const clientId = `client-${Math.floor(rng.next() * 10000)}` return [{ name: 'oauth-flow', steps: [ { name: 'authorize', request: { method: 'POST', url: '/oauth/authorize', body: { client_id: clientId, redirect_uri: 'http://localhost/callback', scope: 'read', }, }, expect: ['status:200', 'response_body(this).code != null'], capture: { code: 'response_body(this).code' }, }, { name: 'token', request: { method: 'POST', url: '/oauth/token', body: { code: '$authorize.code', client_id: clientId, client_secret: 'secret', redirect_uri: 'http://localhost/callback', }, }, expect: ['status:200', 'response_body(this).access_token != null'], capture: { accessToken: 'response_body(this).access_token' }, }, { name: 'user', request: { method: 'GET', url: '/api/user', headers: { authorization: 'Bearer $token.accessToken', }, }, expect: ['status:200', 'response_body(this).id != null'], }, ], }] } // --------------------------------------------------------------------------- // Artifact builder // --------------------------------------------------------------------------- /** * Build a rich artifact document from qualify results. * Includes step traces, cleanup failures, and replay info. */ export function buildArtifact( runResult: QualifyRunResult, options: { cwd: string configPath?: string profile?: string preset?: string env: string seed: number }, ): Artifact { const failures: FailureRecord[] = [] const warnings: string[] = [] const replayCompatibleExecutedRoutes = (runResult.executedRoutes || []) .map(normalizeRouteIdentity) .filter(isReplayCompatibleRoute) // Collect scenario failures for (const scenario of runResult.scenarioResults) { if (!scenario.ok) { for (let stepIdx = 0; stepIdx < scenario.steps.length; stepIdx++) { const step = scenario.steps[stepIdx]! if (!step.ok && step.diagnostics) { // Use actual HTTP route from step trace for stable replay identity const trace = runResult.stepTraces.find( t => t.name === step.name && t.status === 'failed' ) const route = normalizeRouteIdentity(trace?.route || `${scenario.name} / ${step.name}`) if (!isReplayCompatibleRoute(route)) { warnings.push(`Scenario step "${scenario.name}/${step.name}" did not resolve to METHOD /path route identity.`) } failures.push({ route, contract: step.diagnostics.formula || 'scenario-step', expected: step.diagnostics.expected || 'success', observed: step.diagnostics.error || 'failure', seed: runResult.seed, replayCommand: `apophis replay --artifact `, category: step.diagnostics.error ? classifyError(step.diagnostics.error) : ErrorTaxonomy.RUNTIME, diff: step.diagnostics.diff ?? undefined, actual: step.diagnostics.actual ?? undefined, }) } } } } // Collect stateful failures if (runResult.statefulResult) { let fallbackRouteIdx = 0 for (const test of runResult.statefulResult.tests) { if (!test.ok) { let route = normalizeRouteIdentity(test.name) if (!isReplayCompatibleRoute(route)) { route = replayCompatibleExecutedRoutes[fallbackRouteIdx] || route fallbackRouteIdx++ } if (!isReplayCompatibleRoute(route)) { warnings.push(`Stateful failure "${test.name}" did not resolve to METHOD /path route identity.`) } failures.push({ route, contract: test.diagnostics?.formula || 'stateful-test', expected: test.diagnostics?.expected || 'success', observed: test.diagnostics?.error || 'failure', seed: runResult.seed, replayCommand: `apophis replay --artifact `, category: test.diagnostics?.error ? classifyError(test.diagnostics.error) : ErrorTaxonomy.RUNTIME, diff: test.diagnostics?.diff ?? undefined, actual: test.diagnostics?.actual ?? undefined, }) } } } const totalTests = runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) + (runResult.statefulResult?.summary.passed ?? 0) + (runResult.statefulResult?.summary.failed ?? 0) const passedTests = runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) + (runResult.statefulResult?.summary.passed ?? 0) if (runResult.cleanupFailures.length > 0) { warnings.push( `Cleanup failures: ${runResult.cleanupFailures.map(c => `${c.resource}: ${c.error}`).join(', ')}` ) } // Build cleanup outcomes from cleanup failures const cleanupOutcomes = runResult.cleanupFailures.map(cf => ({ resource: cf.resource, cleaned: false, error: cf.error, })) // Build execution summary from runner result const executionSummary = runResult.executionSummary // Build per-gate coverage breakdown for clear artifact interpretation const scenarioRoutes = [...new Set( runResult.stepTraces .filter(t => t.status === 'passed' || t.status === 'failed') .map(t => t.route) .filter(Boolean) )] const statefulRoutes = runResult.statefulResult ? [...new Set( runResult.statefulResult.tests .filter(t => !t.directive) .map(t => normalizeRouteIdentity(t.name)) )] : [] const chaosRoutesPlanned: string[] = [] const chaosRoutesExecuted: string[] = [] for (const r of runResult.chaosResults ?? []) { if (r.applied) chaosRoutesExecuted.push(r.route) } // Infer planned from chaos results or executionSummary if (executionSummary.chaosRoutesPlanned > 0 && chaosRoutesExecuted.length > 0) { chaosRoutesPlanned.push(...chaosRoutesExecuted) } const coverageBreakdown = { scenario: { routesCovered: scenarioRoutes, stepsTotal: executionSummary.totalSteps, stepsPassed: runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0), }, stateful: { routesCovered: statefulRoutes, testsTotal: executionSummary.statefulTestsRun, testsPassed: runResult.statefulResult?.summary.passed ?? 0, }, chaos: { routesPlanned: chaosRoutesPlanned, routesExecuted: chaosRoutesExecuted, runsTotal: executionSummary.chaosRunsRun, runsPassed: (runResult.chaosResults ?? []).filter(r => r.contractsPassed).length, }, } // Build profile gates from the result context // We need to pass gates through or infer from results const profileGates = { scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0, stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0, chaos: executionSummary.chaosRoutesPlanned > 0, } // Deterministic parameters for audit const deterministicParams = { seed: runResult.seed, profileGates, } return { version: 'apophis-artifact/1', command: 'qualify', mode: 'qualify', cwd: options.cwd, configPath: options.configPath, profile: options.profile, preset: options.preset, env: options.env, seed: options.seed, startedAt: new Date(Date.now() - runResult.durationMs).toISOString(), durationMs: runResult.durationMs, summary: { total: totalTests, passed: passedTests, failed: failures.length, skipped: (runResult.statefulResult?.summary.skipped ?? 0), }, executionSummary, coverageBreakdown, executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity), skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({ route: sr.route, executed: false, reason: sr.reason, })), stepTraces: runResult.stepTraces, cleanupOutcomes, profileGates, deterministicParams, failures, artifacts: [], warnings, exitReason: runResult.passed ? 'success' : 'behavioral_failure', } } function attachReplayCommands(artifact: Artifact, artifactPath: string): void { for (const failure of artifact.failures) { failure.replayCommand = `apophis replay --artifact ${artifactPath}` } } async function emitArtifact( artifact: Artifact, options: { command: 'qualify' cwd: string preferredDir?: string force: boolean }, ): Promise { if (!options.force && !options.preferredDir) { return undefined } const defaultDir = resolve(options.cwd, 'reports', 'apophis') const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[] const attempted = new Set() for (const dir of candidateDirs) { if (attempted.has(dir)) continue attempted.add(dir) try { const { mkdirSync, writeFileSync } = await import('node:fs') const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`) mkdirSync(dir, { recursive: true }) attachReplayCommands(artifact, artifactPath) writeFileSync(artifactPath, JSON.stringify(artifact, null, 2)) if (!artifact.artifacts.includes(artifactPath)) { artifact.artifacts.push(artifactPath) } return artifactPath } catch { // Try fallback directory if available. } } artifact.warnings.push('Failed to write artifact to disk') return undefined } // --------------------------------------------------------------------------- // Output formatting // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // Main command handler // --------------------------------------------------------------------------- /** * Main qualify command handler. * * Flow: * 1. Load and resolve config * 2. Run policy engine checks (block prod by default) * 3. Generate seed if omitted, always print it * 4. Resolve profile gates (scenario/stateful/chaos) * 5. Build scenario configs from profile routes * 6. Run execution modes * 7. Build rich artifact with step traces * 8. Handle cleanup failures separately * 9. Return appropriate exit code */ export async function qualifyCommand( options: QualifyOptions, ctx: CliContext, ): Promise { const { profile, seed: explicitSeed, changed, config: configPath, cwd, artifactDir, } = options const workingDir = cwd || ctx.cwd const format = options.format || ctx.options.format || 'human' // Detect environment const env = detectEnvironment() try { // 1. Load config const loadResult = await loadConfig({ cwd: workingDir, configPath, profileName: profile, env, }) if (!loadResult.configPath) { return { exitCode: USAGE_ERROR, message: 'No config found. Run "apophis init" to create one.', } } const config = loadResult.config // 2. Run policy engine checks const policyEngine = new PolicyEngine({ config, env, mode: 'qualify', profileName: profile || undefined, presetName: loadResult.presetName || undefined, }) const policyResult = policyEngine.check() if (!policyResult.allowed) { const message = [ 'Policy check failed:', ...policyResult.errors.map(e => ` ✗ ${e}`), ].join('\n') return { exitCode: USAGE_ERROR, message, } } // 3. Generate seed if omitted const seed = explicitSeed ?? generateSeed() if (!ctx.options.quiet && format === 'human') { console.log(`Seed: ${seed}`) } // 4. Resolve profile gates const profileDef = profile ? config.profiles?.[profile] : undefined const gates = resolveProfileGates(profileDef?.features) // 5. Build scenario configs from profile routes and config const routes = profileDef?.routes ?? [] const fixtureScenarios = buildScenarioConfigs(routes, seed) const configScenarios: ScenarioConfig[] = (config.scenarios ?? []).map(s => ({ name: s.name, steps: s.steps.map(step => ({ name: step.name, request: { method: step.request.method as ScenarioConfig['steps'][0]['request']['method'], url: step.request.url, body: step.request.body, headers: step.request.headers, }, expect: step.expect, capture: step.capture, })), })) const scenarios = [...fixtureScenarios, ...configScenarios] // 6. Build stateful config const presetName = profileDef?.preset const preset = presetName ? config.presets?.[presetName] : undefined const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout) const statefulConfig: TestConfig | undefined = gates.stateful ? { seed, timeout: presetTimeout, routes: profileDef?.routes, } : undefined // 7. Build chaos config const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos ? { probability: 0.5, strategy: preset.chaosStrategy as ChaosConfig['strategy'], sampleSize: preset.chaosSampleSize, sampleRoutes: preset.chaosSampleRoutes, delay: { probability: 0.3, minMs: 100, maxMs: 500 }, error: { probability: 0.2, statusCode: 503 }, dropout: { probability: 0.2, statusCode: 504 }, corruption: { probability: 0.1 }, } : undefined // 8. Load the Fastify app for execution let fastify: FastifyAppLike | undefined let appEntrypoint: string | undefined try { const { loadApp } = await import('../../core/app-loader.js') const loaded = await loadApp(workingDir) fastify = loaded.fastify as FastifyAppLike appEntrypoint = loaded.entrypoint if (fastify && typeof fastify.ready === 'function') { await fastify.ready() } } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err) return { exitCode: USAGE_ERROR, message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`, } } try { // 9. Discover routes for chaos (with discovery metadata) const allRoutes = await discoverAppRoutes(fastify) let appRoutes = allRoutes // 9a. Apply --changed filter if (changed) { const inGit = await isGitRepo(workingDir) if (!inGit) { return { exitCode: USAGE_ERROR, message: '--changed requires a git repository. Current directory is not inside a git repo.', } } appRoutes = await filterChangedRoutes(allRoutes, workingDir) const matched = appRoutes.length const total = allRoutes.length if (!ctx.options.quiet && format === 'human') { console.log(`${matched}/${total} routes match --changed filter`) } if (matched === 0) { return { exitCode: SUCCESS, message: `No routes changed since HEAD. All ${total} routes skipped.`, } } } // Collect discovery warnings const { discoverRouteDetails } = await import('../../../domain/discovery.js') const discoveryResult = discoverRouteDetails(fastify as { routes?: Array<{ method: string; url: string; schema?: Record } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string }) const discoveryWarnings: string[] = [ ...discoveryResult.warnings, ] if (discoveryResult.source === 'print-routes' && !discoveryResult.hasSchemaMetadata) { discoveryWarnings.push( 'Routes were discovered without schema metadata. Behavioral contracts and chaos injection may be limited. ' + 'Register APOPHIS before defining routes or use createFastify().' ) } // 10. Run qualify execution const deps = { fastify: fastify as any, seed, timeout: presetTimeout, pluginContractRegistry: (fastify as unknown as Record)?._apophisPluginContractRegistry as import('../../../domain/plugin-contracts.js').PluginContractRegistry | undefined, } const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes) // 11. Build artifact first so we can reference it for guardrails const artifact = buildArtifact(runResult, { cwd: workingDir, configPath: loadResult.configPath, profile: profile || undefined, preset: presetName, env, seed, }) // Drain plugin contract registry warnings if app registered APOPHIS const pcr = (fastify as any)?._apophisPluginContractRegistry if (pcr?.drainWarnings) { const pcrWarnings = pcr.drainWarnings() if (pcrWarnings.length > 0) { artifact.warnings.push(...pcrWarnings) } } // 12. Signal quality guardrails — fail if zero checks executed const execSummary = runResult.executionSummary const warnings: string[] = [...artifact.warnings, ...discoveryWarnings] if (execSummary.totalExecuted === 0) { await emitArtifact(artifact, { command: 'qualify', cwd: workingDir, preferredDir: artifactDir || config.artifactDir, force: true, }) return { exitCode: BEHAVIORAL_FAILURE, message: 'Qualify failed: zero checks executed. No scenarios, stateful tests, or chaos runs were performed. Verify profile gates and app configuration.', artifact, warnings: artifact.warnings, } } // Warn if execution counts are suspiciously low if (gates.scenario && execSummary.scenariosRun === 0) { warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.') } if (gates.stateful && execSummary.statefulTestsRun === 0 && execSummary.statefulSkipped > 0) { warnings.push(`WARNING: stateful gate enabled but all ${execSummary.statefulSkipped} commands were skipped (preconditions not met). Check required headers, scopes, or plugin contract settings.`) } else if (gates.stateful && execSummary.statefulTestsRun === 0) { warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.') } if (gates.chaos && execSummary.chaosRunsRun === 0) { warnings.push('WARNING: chaos gate enabled but zero chaos runs executed. Check chaos config and route availability.') } // 12. Write artifact if configured or on failure const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed) await emitArtifact(artifact, { command: 'qualify', cwd: workingDir, preferredDir: artifactDir || config.artifactDir, force: shouldEmitArtifact, }) // 13. Format output based on format option const outputCtx: OutputContext = { isTTY: ctx.isTTY, isCI: ctx.isCI, colorMode: ctx.options.color, } let message = '' if (!ctx.options.quiet) { if (format === 'json') { message = renderJsonArtifact(artifact) } else if (format === 'json-summary') { message = renderJsonSummaryArtifact(artifact) } else if (format === 'ndjson') { // For ndjson, we don't return a message string; events are streamed message = '' } else if (format === 'ndjson-summary') { // Concise ndjson: only summary events message = '' } else { // human format message = renderHumanArtifact(artifact, outputCtx) } } return { exitCode: runResult.passed ? SUCCESS : BEHAVIORAL_FAILURE, artifact, message, warnings: artifact.warnings, } } finally { if (fastify && typeof fastify.close === 'function') { try { await fastify.close() } catch (closeErr) { if (process.env.APOPHIS_DEBUG === '1') { console.error('Failed to close Fastify app after qualify run:', closeErr) } } } } } catch (error) { const message = error instanceof Error ? error.message : String(error) return { exitCode: INTERNAL_ERROR, message: `Internal error in qualify command: ${message}`, } } } // --------------------------------------------------------------------------- // CLI adapter // --------------------------------------------------------------------------- /** * Adapter that bridges the CLI framework (cac) to the qualify command handler. * This function signature matches what the CLI core expects. */ export async function handleQualify( args: string[], ctx: CliContext, ): Promise { const options: QualifyOptions = { profile: ctx.options.profile || undefined, seed: undefined, changed: false, config: ctx.options.config || undefined, cwd: ctx.cwd, format: ctx.options.format as QualifyOptions['format'], quiet: ctx.options.quiet, verbose: ctx.options.verbose, artifactDir: ctx.options.artifactDir || undefined, } const seedIdx = args.indexOf('--seed') if (seedIdx !== -1 && args[seedIdx + 1]) { const parsed = parseInt(args[seedIdx + 1]!, 10) if (!isNaN(parsed)) { options.seed = parsed } } if (args.includes('--changed')) { options.changed = true } const result = await qualifyCommand(options, ctx) const format = options.format || ctx.options.format || 'human' const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary' if (!ctx.options.quiet) { if (format === 'json') { if (result.artifact) { console.log(renderJsonArtifact(result.artifact)) } else { console.log(renderJson({ exitCode: result.exitCode, message: result.message, warnings: result.warnings, })) } } else if (format === 'json-summary') { if (result.artifact) { console.log(renderJsonSummaryArtifact(result.artifact)) } else { console.log(renderJson({ exitCode: result.exitCode, message: result.message, warnings: result.warnings, })) } } else if (format === 'ndjson') { if (result.artifact) { renderNdjsonArtifact(result.artifact) } else { process.stdout.write(JSON.stringify({ type: 'run.completed', command: 'qualify', exitCode: result.exitCode, message: result.message, warnings: result.warnings, }) + '\n') } } else if (format === 'ndjson-summary') { if (result.artifact) { renderNdjsonSummaryArtifact(result.artifact) } else { process.stdout.write(JSON.stringify({ type: 'run.completed', command: 'qualify', exitCode: result.exitCode, message: result.message, warnings: result.warnings, }) + '\n') } } else if (result.message) { console.log(result.message) } } // Print warnings in human mode only if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) { for (const warning of result.warnings) { console.warn(`Warning: ${warning}`) } } return result.exitCode }