Files
apophis-fastify/src/cli/commands/qualify/index.ts
T

788 lines
26 KiB
TypeScript
Raw Normal View History

/**
* S6: Qualify thread - Qualify command handler
*
* Responsibilities:
* - Load config and resolve profile
* - Block prod runs by default (policy engine)
* - Run scenario/stateful/chaos based on profile
* - Generate seed if omitted, always print it
* - Rich artifact emission with step traces
* - Handle cleanup failures separately
* - Exit 0 on pass, 1 on qualification failure, 2 on safety violation
*
* Architecture:
* - Dependency injection: all dependencies passed explicitly
* - No optional imports — everything is required or injected
* - Inline comments for documentation
*/
import type { CliContext } from '../../core/context.js'
import { loadConfig } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
import {
runQualify,
resolveProfileGates,
type QualifyRunResult,
type StepTrace,
type CleanupFailure,
} from './runner.js'
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
import { renderHumanArtifact } from '../../renderers/human.js'
import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js'
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
import type { OutputContext } from '../../renderers/shared.js'
import { resolve } from 'node:path'
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
function normalizeRouteIdentity(route: string): string {
const normalized = route.trim().replace(/\s+/g, ' ')
const [method, ...pathParts] = normalized.split(' ')
if (!method || pathParts.length === 0) {
return normalized
}
return `${method.toUpperCase()} ${pathParts.join(' ')}`
}
function isReplayCompatibleRoute(route: string): boolean {
return ROUTE_IDENTITY_PATTERN.test(route)
}
function coerceTimeout(value: unknown): number | undefined {
return typeof value === 'number' ? value : undefined
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface QualifyOptions {
profile?: string
seed?: number
config?: string
cwd?: string
format?: 'human' | 'json' | 'ndjson'
quiet?: boolean
verbose?: boolean
artifactDir?: string
}
interface FastifyAppLike {
ready?: () => Promise<void>
close?: () => Promise<void>
}
// ---------------------------------------------------------------------------
// Seed generation
// ---------------------------------------------------------------------------
/**
* Generate a deterministic seed if none provided.
* Uses current time + process pid + counter for uniqueness.
*/
let seedCounter = 0
export function generateSeed(): number {
seedCounter++
return Date.now() + (process.pid || 0) + seedCounter
}
// ---------------------------------------------------------------------------
// Route discovery helper
// ---------------------------------------------------------------------------
/**
* Discover routes from the Fastify app for chaos execution.
* Uses the shared discovery module for capture + fallback support.
*/
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
const { discoverRoutes } = await import('../../../domain/discovery.js')
return discoverRoutes(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
}
// ---------------------------------------------------------------------------
// Scenario builder from profile
// ---------------------------------------------------------------------------
/**
* Build scenario configs from profile routes for protocol-lab fixture.
* Creates an OAuth-like multi-step scenario.
*/
function buildScenarioConfigs(routes: string[], seed: number): ScenarioConfig[] {
// For the protocol-lab fixture, build the OAuth scenario
const hasOAuth = routes.some(r => r.includes('/oauth/authorize'))
if (!hasOAuth) return []
const rng = new SeededRng(seed)
const clientId = `client-${Math.floor(rng.next() * 10000)}`
return [{
name: 'oauth-flow',
steps: [
{
name: 'authorize',
request: {
method: 'POST',
url: '/oauth/authorize',
body: {
client_id: clientId,
redirect_uri: 'http://localhost/callback',
scope: 'read',
},
},
expect: ['status:200', 'response_body(this).code != null'],
capture: { code: 'response_body(this).code' },
},
{
name: 'token',
request: {
method: 'POST',
url: '/oauth/token',
body: {
code: '$authorize.code',
client_id: clientId,
client_secret: 'secret',
redirect_uri: 'http://localhost/callback',
},
},
expect: ['status:200', 'response_body(this).access_token != null'],
capture: { accessToken: 'response_body(this).access_token' },
},
{
name: 'user',
request: {
method: 'GET',
url: '/api/user',
headers: {
authorization: 'Bearer $token.accessToken',
},
},
expect: ['status:200', 'response_body(this).id != null'],
},
],
}]
}
// ---------------------------------------------------------------------------
// Artifact builder
// ---------------------------------------------------------------------------
/**
* Build a rich artifact document from qualify results.
* Includes step traces, cleanup failures, and replay info.
*/
export function buildArtifact(
runResult: QualifyRunResult,
options: {
cwd: string
configPath?: string
profile?: string
preset?: string
env: string
seed: number
},
): Artifact {
const failures: FailureRecord[] = []
const warnings: string[] = []
const replayCompatibleExecutedRoutes = (runResult.executedRoutes || [])
.map(normalizeRouteIdentity)
.filter(isReplayCompatibleRoute)
// Collect scenario failures
for (const scenario of runResult.scenarioResults) {
if (!scenario.ok) {
for (let stepIdx = 0; stepIdx < scenario.steps.length; stepIdx++) {
const step = scenario.steps[stepIdx]!
if (!step.ok && step.diagnostics) {
// Use actual HTTP route from step trace for stable replay identity
const trace = runResult.stepTraces.find(
t => t.name === step.name && t.status === 'failed'
)
const route = normalizeRouteIdentity(trace?.route || `${scenario.name} / ${step.name}`)
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Scenario step "${scenario.name}/${step.name}" did not resolve to METHOD /path route identity.`)
}
failures.push({
route,
contract: step.diagnostics.formula || 'scenario-step',
expected: step.diagnostics.expected || 'success',
observed: step.diagnostics.error || 'failure',
seed: runResult.seed,
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
category: step.diagnostics.error ? classifyError(step.diagnostics.error) : ErrorTaxonomy.RUNTIME,
diff: step.diagnostics.diff ?? undefined,
actual: step.diagnostics.actual ?? undefined,
})
}
}
}
}
// Collect stateful failures
if (runResult.statefulResult) {
let fallbackRouteIdx = 0
for (const test of runResult.statefulResult.tests) {
if (!test.ok) {
let route = normalizeRouteIdentity(test.name)
if (!isReplayCompatibleRoute(route)) {
route = replayCompatibleExecutedRoutes[fallbackRouteIdx] || route
fallbackRouteIdx++
}
if (!isReplayCompatibleRoute(route)) {
warnings.push(`Stateful failure "${test.name}" did not resolve to METHOD /path route identity.`)
}
failures.push({
route,
contract: test.diagnostics?.formula || 'stateful-test',
expected: test.diagnostics?.expected || 'success',
observed: test.diagnostics?.error || 'failure',
seed: runResult.seed,
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
category: test.diagnostics?.error ? classifyError(test.diagnostics.error) : ErrorTaxonomy.RUNTIME,
diff: test.diagnostics?.diff ?? undefined,
actual: test.diagnostics?.actual ?? undefined,
})
}
}
}
const totalTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
(runResult.statefulResult?.tests.length ?? 0)
const passedTests =
runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
(runResult.statefulResult?.summary.passed ?? 0)
if (runResult.cleanupFailures.length > 0) {
warnings.push(
`Cleanup failures: ${runResult.cleanupFailures.map(c => `${c.resource}: ${c.error}`).join(', ')}`
)
}
// Build cleanup outcomes from cleanup failures
const cleanupOutcomes = runResult.cleanupFailures.map(cf => ({
resource: cf.resource,
cleaned: false,
error: cf.error,
}))
// Build execution summary from runner result
const executionSummary = runResult.executionSummary
// Build per-gate coverage breakdown for clear artifact interpretation
const scenarioRoutes = [...new Set(
runResult.stepTraces
.filter(t => t.status === 'passed' || t.status === 'failed')
.map(t => t.route)
.filter(Boolean)
)]
const statefulRoutes = runResult.statefulResult
? [...new Set(runResult.statefulResult.tests.map(t => normalizeRouteIdentity(t.name)))]
: []
const chaosRoutesPlanned: string[] = []
const chaosRoutesExecuted: string[] = []
for (const r of runResult.chaosResults ?? []) {
if (r.applied) chaosRoutesExecuted.push(r.route)
}
// Infer planned from chaos results or executionSummary
if (executionSummary.chaosRoutesPlanned > 0 && chaosRoutesExecuted.length > 0) {
chaosRoutesPlanned.push(...chaosRoutesExecuted)
}
const coverageBreakdown = {
scenario: {
routesCovered: scenarioRoutes,
stepsTotal: executionSummary.totalSteps,
stepsPassed: runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0),
},
stateful: {
routesCovered: statefulRoutes,
testsTotal: executionSummary.statefulTestsRun,
testsPassed: runResult.statefulResult?.summary.passed ?? 0,
},
chaos: {
routesPlanned: chaosRoutesPlanned,
routesExecuted: chaosRoutesExecuted,
runsTotal: executionSummary.chaosRunsRun,
runsPassed: (runResult.chaosResults ?? []).filter(r => r.contractsPassed).length,
},
}
// Build profile gates from the result context
// We need to pass gates through or infer from results
const profileGates = {
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
chaos: executionSummary.chaosRoutesPlanned > 0,
}
// Deterministic parameters for audit
const deterministicParams = {
seed: runResult.seed,
profileGates,
}
return {
version: 'apophis-artifact/1',
command: 'qualify',
mode: 'qualify',
cwd: options.cwd,
configPath: options.configPath,
profile: options.profile,
preset: options.preset,
env: options.env,
seed: options.seed,
startedAt: new Date(Date.now() - runResult.durationMs).toISOString(),
durationMs: runResult.durationMs,
summary: {
total: totalTests,
passed: passedTests,
failed: failures.length,
},
executionSummary,
coverageBreakdown,
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
route: sr.route,
executed: false,
reason: sr.reason,
})),
stepTraces: runResult.stepTraces,
cleanupOutcomes,
profileGates,
deterministicParams,
failures,
artifacts: [],
warnings,
exitReason: runResult.passed ? 'success' : 'behavioral_failure',
}
}
function attachReplayCommands(artifact: Artifact, artifactPath: string): void {
for (const failure of artifact.failures) {
failure.replayCommand = `apophis replay --artifact ${artifactPath}`
}
}
async function emitArtifact(
artifact: Artifact,
options: {
command: 'qualify'
cwd: string
preferredDir?: string
force: boolean
},
): Promise<string | undefined> {
if (!options.force && !options.preferredDir) {
return undefined
}
const defaultDir = resolve(options.cwd, 'reports', 'apophis')
const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[]
const attempted = new Set<string>()
for (const dir of candidateDirs) {
if (attempted.has(dir)) continue
attempted.add(dir)
try {
const { mkdirSync, writeFileSync } = await import('node:fs')
const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`)
mkdirSync(dir, { recursive: true })
attachReplayCommands(artifact, artifactPath)
writeFileSync(artifactPath, JSON.stringify(artifact, null, 2))
if (!artifact.artifacts.includes(artifactPath)) {
artifact.artifacts.push(artifactPath)
}
return artifactPath
} catch {
// Try fallback directory if available.
}
}
artifact.warnings.push('Failed to write artifact to disk')
return undefined
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
// Main command handler
// ---------------------------------------------------------------------------
/**
* Main qualify command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks (block prod by default)
* 3. Generate seed if omitted, always print it
* 4. Resolve profile gates (scenario/stateful/chaos)
* 5. Build scenario configs from profile routes
* 6. Run execution modes
* 7. Build rich artifact with step traces
* 8. Handle cleanup failures separately
* 9. Return appropriate exit code
*/
export async function qualifyCommand(
options: QualifyOptions,
ctx: CliContext,
): Promise<CommandResult> {
const {
profile,
seed: explicitSeed,
config: configPath,
cwd,
artifactDir,
} = options
const workingDir = cwd || ctx.cwd
const format = options.format || ctx.options.format || 'human'
// Detect environment
const env = detectEnvironment()
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
})
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
}
}
const config = loadResult.config
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'qualify',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
})
const policyResult = policyEngine.check()
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => `${e}`),
].join('\n')
return {
exitCode: USAGE_ERROR,
message,
}
}
// 3. Generate seed if omitted
const seed = explicitSeed ?? generateSeed()
if (!ctx.options.quiet && format === 'human') {
console.log(`Seed: ${seed}`)
}
// 4. Resolve profile gates
const profileDef = profile ? config.profiles?.[profile] : undefined
const gates = resolveProfileGates(profileDef?.features)
// 5. Build scenario configs from profile routes and config
const routes = profileDef?.routes ?? []
const fixtureScenarios = buildScenarioConfigs(routes, seed)
const configScenarios: ScenarioConfig[] = (config.scenarios ?? []).map(s => ({
name: s.name,
steps: s.steps.map(step => ({
name: step.name,
request: {
method: step.request.method as ScenarioConfig['steps'][0]['request']['method'],
url: step.request.url,
body: step.request.body,
headers: step.request.headers,
},
expect: step.expect,
capture: step.capture,
})),
}))
const scenarios = [...fixtureScenarios, ...configScenarios]
// 6. Build stateful config
const presetName = profileDef?.preset
const preset = presetName ? config.presets?.[presetName] : undefined
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
const statefulConfig: TestConfig | undefined = gates.stateful
? {
seed,
timeout: presetTimeout,
routes: profileDef?.routes,
}
: undefined
// 7. Build chaos config
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
? {
probability: 0.5,
strategy: preset.chaosStrategy as ChaosConfig['strategy'],
sampleSize: preset.chaosSampleSize,
sampleRoutes: preset.chaosSampleRoutes,
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
error: { probability: 0.2, statusCode: 503 },
dropout: { probability: 0.2, statusCode: 504 },
corruption: { probability: 0.1 },
}
: undefined
// 8. Load the Fastify app for execution
let fastify: FastifyAppLike | undefined
let appEntrypoint: string | undefined
try {
const { loadApp } = await import('../../core/app-loader.js')
const loaded = await loadApp(workingDir)
fastify = loaded.fastify as FastifyAppLike
appEntrypoint = loaded.entrypoint
if (fastify && typeof fastify.ready === 'function') {
await fastify.ready()
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err)
return {
exitCode: USAGE_ERROR,
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
}
}
try {
// 9. Discover routes for chaos (with discovery metadata)
const appRoutes = await discoverAppRoutes(fastify)
// Collect discovery warnings
const { discoverRouteDetails } = await import('../../../domain/discovery.js')
const discoveryResult = discoverRouteDetails(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
const discoveryWarnings: string[] = [
...discoveryResult.warnings,
]
if (discoveryResult.source === 'print-routes' && !discoveryResult.hasSchemaMetadata) {
discoveryWarnings.push(
'Routes were discovered without schema metadata. Behavioral contracts and chaos injection may be limited. ' +
'Register APOPHIS before defining routes or use createFastify().'
)
}
// 10. Run qualify execution
const deps = {
fastify: fastify as any,
seed,
timeout: presetTimeout,
}
const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
// 11. Build artifact first so we can reference it for guardrails
const artifact = buildArtifact(runResult, {
cwd: workingDir,
configPath: loadResult.configPath,
profile: profile || undefined,
preset: presetName,
env,
seed,
})
// 12. Signal quality guardrails — fail if zero checks executed
const execSummary = runResult.executionSummary
const warnings: string[] = [...artifact.warnings, ...discoveryWarnings]
if (execSummary.totalExecuted === 0) {
await emitArtifact(artifact, {
command: 'qualify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: true,
})
return {
exitCode: BEHAVIORAL_FAILURE,
message: 'Qualify failed: zero checks executed. No scenarios, stateful tests, or chaos runs were performed. Verify profile gates and app configuration.',
artifact,
warnings: artifact.warnings,
}
}
// Warn if execution counts are suspiciously low
if (gates.scenario && execSummary.scenariosRun === 0) {
warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
}
if (gates.stateful && execSummary.statefulTestsRun === 0) {
warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
}
if (gates.chaos && execSummary.chaosRunsRun === 0) {
warnings.push('WARNING: chaos gate enabled but zero chaos runs executed. Check chaos config and route availability.')
}
// 12. Write artifact if configured or on failure
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
await emitArtifact(artifact, {
command: 'qualify',
cwd: workingDir,
preferredDir: artifactDir || config.artifactDir,
force: shouldEmitArtifact,
})
// 13. Format output based on format option
const outputCtx: OutputContext = {
isTTY: ctx.isTTY,
isCI: ctx.isCI,
colorMode: ctx.options.color,
}
let message = ''
if (!ctx.options.quiet) {
if (format === 'json') {
message = renderJsonArtifact(artifact)
} else if (format === 'json-summary') {
message = renderJsonSummaryArtifact(artifact)
} else if (format === 'ndjson') {
// For ndjson, we don't return a message string; events are streamed
message = ''
} else if (format === 'ndjson-summary') {
// Concise ndjson: only summary events
message = ''
} else {
// human format
message = renderHumanArtifact(artifact, outputCtx)
}
}
return {
exitCode: runResult.passed ? SUCCESS : BEHAVIORAL_FAILURE,
artifact,
message,
warnings: artifact.warnings,
}
} finally {
if (fastify && typeof fastify.close === 'function') {
try {
await fastify.close()
} catch (closeErr) {
if (process.env.APOPHIS_DEBUG === '1') {
console.error('Failed to close Fastify app after qualify run:', closeErr)
}
}
}
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in qualify command: ${message}`,
}
}
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the qualify command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleQualify(
args: string[],
ctx: CliContext,
): Promise<number> {
const options: QualifyOptions = {
profile: ctx.options.profile || undefined,
seed: undefined,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as QualifyOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
artifactDir: ctx.options.artifactDir || undefined,
}
const seedIdx = args.indexOf('--seed')
if (seedIdx !== -1 && args[seedIdx + 1]) {
const parsed = parseInt(args[seedIdx + 1]!, 10)
if (!isNaN(parsed)) {
options.seed = parsed
}
}
const result = await qualifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
if (!ctx.options.quiet) {
if (format === 'json') {
if (result.artifact) {
console.log(renderJsonArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'json-summary') {
if (result.artifact) {
console.log(renderJsonSummaryArtifact(result.artifact))
} else {
console.log(renderJson({
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}))
}
} else if (format === 'ndjson') {
if (result.artifact) {
renderNdjsonArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'qualify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (format === 'ndjson-summary') {
if (result.artifact) {
renderNdjsonSummaryArtifact(result.artifact)
} else {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'qualify',
exitCode: result.exitCode,
message: result.message,
warnings: result.warnings,
}) + '\n')
}
} else if (result.message) {
console.log(result.message)
}
}
// Print warnings in human mode only
if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
for (const warning of result.warnings) {
console.warn(`Warning: ${warning}`)
}
}
return result.exitCode
}