869 lines
28 KiB
TypeScript
869 lines
28 KiB
TypeScript
|
/**
|
||
|
|
* S6: Qualify thread - Qualify command handler
|
||
|
|
*
|
||
|
|
* Responsibilities:
|
||
|
|
* - Load config and resolve profile
|
||
|
|
* - Block prod runs by default (policy engine)
|
||
|
|
* - Run scenario/stateful/chaos based on profile
|
||
|
|
* - Generate seed if omitted, always print it
|
||
|
|
* - Rich artifact emission with step traces
|
||
|
|
* - Handle cleanup failures separately
|
||
|
|
* - Exit 0 on pass, 1 on qualification failure, 2 on safety violation
|
||
|
|
*
|
||
|
|
* Architecture:
|
||
|
|
* - Dependency injection: all dependencies passed explicitly
|
||
|
|
* - No optional imports — everything is required or injected
|
||
|
|
* - Inline comments for documentation
|
||
|
|
*/
|
||
|
|
|
||
|
|
import type { CliContext } from '../../core/context.js'
|
||
|
|
import { loadConfig } from '../../core/config-loader.js'
|
||
|
|
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
||
|
|
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
|
||
|
|
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
||
|
|
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
|
||
|
|
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
||
|
|
import {
|
||
|
|
runQualify,
|
||
|
|
resolveProfileGates,
|
||
|
|
type QualifyRunResult,
|
||
|
|
type StepTrace,
|
||
|
|
type CleanupFailure,
|
||
|
|
} from './runner.js'
|
||
|
|
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
|
||
|
|
import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
|
||
|
|
import { renderHumanArtifact } from '../../renderers/human.js'
|
||
|
|
import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js'
|
||
|
|
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
|
||
|
|
import type { OutputContext } from '../../renderers/shared.js'
|
||
|
|
import { resolve } from 'node:path'
|
||
|
|
import { pathToFileURL } from 'node:url'
|
||
|
|
|
||
|
|
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
|
||
|
|
|
||
|
|
function normalizeRouteIdentity(route: string): string {
|
||
|
|
const normalized = route.trim().replace(/\s+/g, ' ')
|
||
|
|
const [method, ...pathParts] = normalized.split(' ')
|
||
|
|
if (!method || pathParts.length === 0) {
|
||
|
|
return normalized
|
||
|
|
}
|
||
|
|
return `${method.toUpperCase()} ${pathParts.join(' ')}`
|
||
|
|
}
|
||
|
|
|
||
|
|
function isReplayCompatibleRoute(route: string): boolean {
|
||
|
|
return ROUTE_IDENTITY_PATTERN.test(route)
|
||
|
|
}
|
||
|
|
|
||
|
|
function coerceDepth(value: unknown): TestConfig['depth'] {
|
||
|
|
if (value === 'quick' || value === 'standard' || value === 'thorough') {
|
||
|
|
return value
|
||
|
|
}
|
||
|
|
return 'standard'
|
||
|
|
}
|
||
|
|
|
||
|
|
function coerceTimeout(value: unknown): number | undefined {
|
||
|
|
return typeof value === 'number' ? value : undefined
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// Types
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
export interface QualifyOptions {
|
||
|
|
profile?: string
|
||
|
|
generationProfile?: string
|
||
|
|
seed?: number
|
||
|
|
config?: string
|
||
|
|
cwd?: string
|
||
|
|
format?: 'human' | 'json' | 'ndjson'
|
||
|
|
quiet?: boolean
|
||
|
|
verbose?: boolean
|
||
|
|
artifactDir?: string
|
||
|
|
}
|
||
|
|
|
||
|
|
interface FastifyAppLike {
|
||
|
|
ready?: () => Promise<void>
|
||
|
|
close?: () => Promise<void>
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// Seed generation
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Generate a deterministic seed if none provided.
|
||
|
|
* Uses current time + process pid + counter for uniqueness.
|
||
|
|
*/
|
||
|
|
let seedCounter = 0
|
||
|
|
export function generateSeed(): number {
|
||
|
|
seedCounter++
|
||
|
|
return Date.now() + (process.pid || 0) + seedCounter
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// Route discovery helper
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Discover routes from the Fastify app for chaos execution.
|
||
|
|
* Injected fastify instance must have routes registered.
|
||
|
|
*/
|
||
|
|
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
|
||
|
|
// Cast to access routes
|
||
|
|
const app = fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }> }
|
||
|
|
if (!app.routes) return []
|
||
|
|
|
||
|
|
return app.routes.map(r => ({
|
||
|
|
path: r.url,
|
||
|
|
method: r.method as RouteContract['method'],
|
||
|
|
category: 'observer',
|
||
|
|
requires: [],
|
||
|
|
ensures: [],
|
||
|
|
invariants: [],
|
||
|
|
regexPatterns: {},
|
||
|
|
validateRuntime: false,
|
||
|
|
schema: r.schema,
|
||
|
|
}))
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// Scenario builder from profile
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Build scenario configs from profile routes for protocol-lab fixture.
|
||
|
|
* Creates an OAuth-like multi-step scenario.
|
||
|
|
*/
|
||
|
|
function buildScenarioConfigs(routes: string[], seed: number): ScenarioConfig[] {
|
||
|
|
// For the protocol-lab fixture, build the OAuth scenario
|
||
|
|
const hasOAuth = routes.some(r => r.includes('/oauth/authorize'))
|
||
|
|
if (!hasOAuth) return []
|
||
|
|
|
||
|
|
const rng = new SeededRng(seed)
|
||
|
|
const clientId = `client-${Math.floor(rng.next() * 10000)}`
|
||
|
|
|
||
|
|
return [{
|
||
|
|
name: 'oauth-flow',
|
||
|
|
steps: [
|
||
|
|
{
|
||
|
|
name: 'authorize',
|
||
|
|
request: {
|
||
|
|
method: 'POST',
|
||
|
|
url: '/oauth/authorize',
|
||
|
|
body: {
|
||
|
|
client_id: clientId,
|
||
|
|
redirect_uri: 'http://localhost/callback',
|
||
|
|
scope: 'read',
|
||
|
|
},
|
||
|
|
},
|
||
|
|
expect: ['status:200', 'response_body(this).code != null'],
|
||
|
|
capture: { code: 'response_body(this).code' },
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'token',
|
||
|
|
request: {
|
||
|
|
method: 'POST',
|
||
|
|
url: '/oauth/token',
|
||
|
|
body: {
|
||
|
|
code: '$authorize.code',
|
||
|
|
client_id: clientId,
|
||
|
|
client_secret: 'secret',
|
||
|
|
redirect_uri: 'http://localhost/callback',
|
||
|
|
},
|
||
|
|
},
|
||
|
|
expect: ['status:200', 'response_body(this).access_token != null'],
|
||
|
|
capture: { accessToken: 'response_body(this).access_token' },
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'user',
|
||
|
|
request: {
|
||
|
|
method: 'GET',
|
||
|
|
url: '/api/user',
|
||
|
|
headers: {
|
||
|
|
authorization: 'Bearer $token.accessToken',
|
||
|
|
},
|
||
|
|
},
|
||
|
|
expect: ['status:200', 'response_body(this).id != null'],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
}]
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// Artifact builder
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Build a rich artifact document from qualify results.
|
||
|
|
* Includes step traces, cleanup failures, and replay info.
|
||
|
|
*/
|
||
|
|
export function buildArtifact(
|
||
|
|
runResult: QualifyRunResult,
|
||
|
|
options: {
|
||
|
|
cwd: string
|
||
|
|
configPath?: string
|
||
|
|
profile?: string
|
||
|
|
preset?: string
|
||
|
|
env: string
|
||
|
|
seed: number
|
||
|
|
},
|
||
|
|
): Artifact {
|
||
|
|
const failures: FailureRecord[] = []
|
||
|
|
const warnings: string[] = []
|
||
|
|
const replayCompatibleExecutedRoutes = (runResult.executedRoutes || [])
|
||
|
|
.map(normalizeRouteIdentity)
|
||
|
|
.filter(isReplayCompatibleRoute)
|
||
|
|
|
||
|
|
// Collect scenario failures
|
||
|
|
for (const scenario of runResult.scenarioResults) {
|
||
|
|
if (!scenario.ok) {
|
||
|
|
for (let stepIdx = 0; stepIdx < scenario.steps.length; stepIdx++) {
|
||
|
|
const step = scenario.steps[stepIdx]!
|
||
|
|
if (!step.ok && step.diagnostics) {
|
||
|
|
// Use actual HTTP route from step trace for stable replay identity
|
||
|
|
const trace = runResult.stepTraces.find(
|
||
|
|
t => t.name === step.name && t.status === 'failed'
|
||
|
|
)
|
||
|
|
const route = normalizeRouteIdentity(trace?.route || `${scenario.name} / ${step.name}`)
|
||
|
|
if (!isReplayCompatibleRoute(route)) {
|
||
|
|
warnings.push(`Scenario step "${scenario.name}/${step.name}" did not resolve to METHOD /path route identity.`)
|
||
|
|
}
|
||
|
|
failures.push({
|
||
|
|
route,
|
||
|
|
contract: step.diagnostics.formula || 'scenario-step',
|
||
|
|
expected: step.diagnostics.expected || 'success',
|
||
|
|
observed: step.diagnostics.error || 'failure',
|
||
|
|
seed: runResult.seed,
|
||
|
|
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
|
||
|
|
category: step.diagnostics.error ? classifyError(step.diagnostics.error) : ErrorTaxonomy.RUNTIME,
|
||
|
|
diff: step.diagnostics.diff ?? undefined,
|
||
|
|
actual: step.diagnostics.actual ?? undefined,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Collect stateful failures
|
||
|
|
if (runResult.statefulResult) {
|
||
|
|
let fallbackRouteIdx = 0
|
||
|
|
for (const test of runResult.statefulResult.tests) {
|
||
|
|
if (!test.ok) {
|
||
|
|
let route = normalizeRouteIdentity(test.name)
|
||
|
|
if (!isReplayCompatibleRoute(route)) {
|
||
|
|
route = replayCompatibleExecutedRoutes[fallbackRouteIdx] || route
|
||
|
|
fallbackRouteIdx++
|
||
|
|
}
|
||
|
|
if (!isReplayCompatibleRoute(route)) {
|
||
|
|
warnings.push(`Stateful failure "${test.name}" did not resolve to METHOD /path route identity.`)
|
||
|
|
}
|
||
|
|
failures.push({
|
||
|
|
route,
|
||
|
|
contract: test.diagnostics?.formula || 'stateful-test',
|
||
|
|
expected: test.diagnostics?.expected || 'success',
|
||
|
|
observed: test.diagnostics?.error || 'failure',
|
||
|
|
seed: runResult.seed,
|
||
|
|
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
|
||
|
|
category: test.diagnostics?.error ? classifyError(test.diagnostics.error) : ErrorTaxonomy.RUNTIME,
|
||
|
|
diff: test.diagnostics?.diff ?? undefined,
|
||
|
|
actual: test.diagnostics?.actual ?? undefined,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const totalTests =
|
||
|
|
runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
|
||
|
|
(runResult.statefulResult?.tests.length ?? 0)
|
||
|
|
|
||
|
|
const passedTests =
|
||
|
|
runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
|
||
|
|
(runResult.statefulResult?.summary.passed ?? 0)
|
||
|
|
|
||
|
|
if (runResult.cleanupFailures.length > 0) {
|
||
|
|
warnings.push(
|
||
|
|
`Cleanup failures: ${runResult.cleanupFailures.map(c => `${c.resource}: ${c.error}`).join(', ')}`
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Build cleanup outcomes from cleanup failures
|
||
|
|
const cleanupOutcomes = runResult.cleanupFailures.map(cf => ({
|
||
|
|
resource: cf.resource,
|
||
|
|
cleaned: false,
|
||
|
|
error: cf.error,
|
||
|
|
}))
|
||
|
|
|
||
|
|
// Build execution summary from runner result
|
||
|
|
const executionSummary = runResult.executionSummary
|
||
|
|
|
||
|
|
// Build profile gates from the result context
|
||
|
|
// We need to pass gates through or infer from results
|
||
|
|
const profileGates = {
|
||
|
|
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
|
||
|
|
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
|
||
|
|
chaos: (runResult.chaosResult !== undefined) || executionSummary.chaosRunsRun > 0,
|
||
|
|
}
|
||
|
|
|
||
|
|
// Deterministic parameters for audit
|
||
|
|
const deterministicParams = {
|
||
|
|
seed: runResult.seed,
|
||
|
|
profileGates,
|
||
|
|
}
|
||
|
|
|
||
|
|
return {
|
||
|
|
version: 'apophis-artifact/1',
|
||
|
|
command: 'qualify',
|
||
|
|
mode: 'qualify',
|
||
|
|
cwd: options.cwd,
|
||
|
|
configPath: options.configPath,
|
||
|
|
profile: options.profile,
|
||
|
|
preset: options.preset,
|
||
|
|
env: options.env,
|
||
|
|
seed: options.seed,
|
||
|
|
startedAt: new Date(Date.now() - runResult.durationMs).toISOString(),
|
||
|
|
durationMs: runResult.durationMs,
|
||
|
|
summary: {
|
||
|
|
total: totalTests,
|
||
|
|
passed: passedTests,
|
||
|
|
failed: failures.length,
|
||
|
|
},
|
||
|
|
executionSummary,
|
||
|
|
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
|
||
|
|
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
|
||
|
|
route: sr.route,
|
||
|
|
executed: false,
|
||
|
|
reason: sr.reason,
|
||
|
|
})),
|
||
|
|
stepTraces: runResult.stepTraces,
|
||
|
|
cleanupOutcomes,
|
||
|
|
profileGates,
|
||
|
|
deterministicParams,
|
||
|
|
failures,
|
||
|
|
artifacts: [],
|
||
|
|
warnings,
|
||
|
|
exitReason: runResult.passed ? 'success' : 'behavioral_failure',
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function attachReplayCommands(artifact: Artifact, artifactPath: string): void {
|
||
|
|
for (const failure of artifact.failures) {
|
||
|
|
failure.replayCommand = `apophis replay --artifact ${artifactPath}`
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function emitArtifact(
|
||
|
|
artifact: Artifact,
|
||
|
|
options: {
|
||
|
|
command: 'qualify'
|
||
|
|
cwd: string
|
||
|
|
preferredDir?: string
|
||
|
|
force: boolean
|
||
|
|
},
|
||
|
|
): Promise<string | undefined> {
|
||
|
|
if (!options.force && !options.preferredDir) {
|
||
|
|
return undefined
|
||
|
|
}
|
||
|
|
|
||
|
|
const defaultDir = resolve(options.cwd, 'reports', 'apophis')
|
||
|
|
const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[]
|
||
|
|
const attempted = new Set<string>()
|
||
|
|
|
||
|
|
for (const dir of candidateDirs) {
|
||
|
|
if (attempted.has(dir)) continue
|
||
|
|
attempted.add(dir)
|
||
|
|
try {
|
||
|
|
const { mkdirSync, writeFileSync } = await import('node:fs')
|
||
|
|
const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`)
|
||
|
|
mkdirSync(dir, { recursive: true })
|
||
|
|
attachReplayCommands(artifact, artifactPath)
|
||
|
|
writeFileSync(artifactPath, JSON.stringify(artifact, null, 2))
|
||
|
|
if (!artifact.artifacts.includes(artifactPath)) {
|
||
|
|
artifact.artifacts.push(artifactPath)
|
||
|
|
}
|
||
|
|
return artifactPath
|
||
|
|
} catch {
|
||
|
|
// Try fallback directory if available.
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
artifact.warnings.push('Failed to write artifact to disk')
|
||
|
|
return undefined
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// Output formatting
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
function formatHumanOutput(
|
||
|
|
result: QualifyRunResult,
|
||
|
|
options: { profile?: string; seed: number; env: string },
|
||
|
|
): string {
|
||
|
|
const lines: string[] = []
|
||
|
|
|
||
|
|
lines.push(`Qualify run for environment "${options.env}"`)
|
||
|
|
if (options.profile) {
|
||
|
|
lines.push(`Profile: ${options.profile}`)
|
||
|
|
}
|
||
|
|
lines.push(`Seed: ${options.seed}`)
|
||
|
|
lines.push('')
|
||
|
|
|
||
|
|
// Scenario results
|
||
|
|
for (const scenario of result.scenarioResults) {
|
||
|
|
lines.push(`Scenario: ${scenario.name}`)
|
||
|
|
for (const step of scenario.steps) {
|
||
|
|
const icon = step.ok ? '✓' : '✗'
|
||
|
|
lines.push(` ${icon} ${step.name} (${step.statusCode ?? 'no-status'})`)
|
||
|
|
if (!step.ok && step.diagnostics) {
|
||
|
|
lines.push(` Expected: ${step.diagnostics.expected || 'success'}`)
|
||
|
|
lines.push(` Observed: ${step.diagnostics.error || 'failure'}`)
|
||
|
|
if (step.diagnostics.actual) {
|
||
|
|
lines.push(` Actual: ${step.diagnostics.actual}`)
|
||
|
|
}
|
||
|
|
if (step.diagnostics.diff) {
|
||
|
|
lines.push(` Diff:`)
|
||
|
|
for (const line of String(step.diagnostics.diff).split('\n')) {
|
||
|
|
lines.push(` ${line}`)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
lines.push('')
|
||
|
|
}
|
||
|
|
|
||
|
|
// Stateful results
|
||
|
|
if (result.statefulResult) {
|
||
|
|
lines.push(`Stateful: ${result.statefulResult.summary.passed} passed, ${result.statefulResult.summary.failed} failed`)
|
||
|
|
lines.push('')
|
||
|
|
}
|
||
|
|
|
||
|
|
// Chaos results
|
||
|
|
if (result.chaosResult) {
|
||
|
|
lines.push(`Chaos: ${result.chaosResult.applied ? 'applied' : 'none'}`)
|
||
|
|
if (result.chaosResult.events.length > 0) {
|
||
|
|
for (const event of result.chaosResult.events) {
|
||
|
|
lines.push(` ${event}`)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
lines.push('')
|
||
|
|
}
|
||
|
|
|
||
|
|
// Step traces
|
||
|
|
if (result.stepTraces.length > 0) {
|
||
|
|
lines.push('Step traces:')
|
||
|
|
for (const trace of result.stepTraces.slice(0, 20)) {
|
||
|
|
const icon = trace.status === 'passed' ? '✓' : trace.status === 'skipped' ? '⊘' : '✗'
|
||
|
|
lines.push(` ${icon} ${trace.name} (${trace.durationMs}ms)`)
|
||
|
|
}
|
||
|
|
if (result.stepTraces.length > 20) {
|
||
|
|
lines.push(` ... and ${result.stepTraces.length - 20} more`)
|
||
|
|
}
|
||
|
|
lines.push('')
|
||
|
|
}
|
||
|
|
|
||
|
|
// Cleanup failures
|
||
|
|
if (result.cleanupFailures.length > 0) {
|
||
|
|
lines.push('Cleanup failures (reported separately):')
|
||
|
|
for (const cf of result.cleanupFailures) {
|
||
|
|
lines.push(` ⚠ ${cf.resource}: ${cf.error}`)
|
||
|
|
}
|
||
|
|
lines.push('')
|
||
|
|
}
|
||
|
|
|
||
|
|
// Per-profile gate execution counts
|
||
|
|
lines.push('Profile gate execution counts:')
|
||
|
|
lines.push(` Scenario: ${result.executionSummary.scenariosRun} run`)
|
||
|
|
lines.push(` Stateful: ${result.executionSummary.statefulTestsRun} tests run`)
|
||
|
|
lines.push(` Chaos: ${result.executionSummary.chaosRunsRun} runs run`)
|
||
|
|
lines.push('')
|
||
|
|
|
||
|
|
// Executed routes
|
||
|
|
if (result.executedRoutes.length > 0) {
|
||
|
|
lines.push(`Executed routes (${result.executedRoutes.length}):`)
|
||
|
|
for (const route of result.executedRoutes) {
|
||
|
|
lines.push(` ${route}`)
|
||
|
|
}
|
||
|
|
lines.push('')
|
||
|
|
}
|
||
|
|
|
||
|
|
// Skipped routes
|
||
|
|
if (result.skippedRoutes.length > 0) {
|
||
|
|
lines.push(`Skipped routes (${result.skippedRoutes.length}):`)
|
||
|
|
for (const sr of result.skippedRoutes) {
|
||
|
|
lines.push(` ${sr.route}: ${sr.reason}`)
|
||
|
|
}
|
||
|
|
lines.push('')
|
||
|
|
}
|
||
|
|
|
||
|
|
// Summary
|
||
|
|
if (result.passed) {
|
||
|
|
lines.push('All qualifications passed.')
|
||
|
|
} else {
|
||
|
|
lines.push('Qualification failed.')
|
||
|
|
lines.push(`Replay: apophis replay --artifact <artifact-path>`)
|
||
|
|
}
|
||
|
|
|
||
|
|
return lines.join('\n')
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// Main command handler
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Main qualify command handler.
|
||
|
|
*
|
||
|
|
* Flow:
|
||
|
|
* 1. Load and resolve config
|
||
|
|
* 2. Run policy engine checks (block prod by default)
|
||
|
|
* 3. Generate seed if omitted, always print it
|
||
|
|
* 4. Resolve profile gates (scenario/stateful/chaos)
|
||
|
|
* 5. Build scenario configs from profile routes
|
||
|
|
* 6. Run execution modes
|
||
|
|
* 7. Build rich artifact with step traces
|
||
|
|
* 8. Handle cleanup failures separately
|
||
|
|
* 9. Return appropriate exit code
|
||
|
|
*/
|
||
|
|
export async function qualifyCommand(
|
||
|
|
options: QualifyOptions,
|
||
|
|
ctx: CliContext,
|
||
|
|
): Promise<CommandResult> {
|
||
|
|
const {
|
||
|
|
profile,
|
||
|
|
generationProfile,
|
||
|
|
seed: explicitSeed,
|
||
|
|
config: configPath,
|
||
|
|
cwd,
|
||
|
|
artifactDir,
|
||
|
|
} = options
|
||
|
|
const workingDir = cwd || ctx.cwd
|
||
|
|
const format = options.format || ctx.options.format || 'human'
|
||
|
|
|
||
|
|
// Detect environment
|
||
|
|
const env = detectEnvironment()
|
||
|
|
|
||
|
|
try {
|
||
|
|
// 1. Load config
|
||
|
|
const loadResult = await loadConfig({
|
||
|
|
cwd: workingDir,
|
||
|
|
configPath,
|
||
|
|
profileName: profile,
|
||
|
|
env,
|
||
|
|
})
|
||
|
|
|
||
|
|
if (!loadResult.configPath) {
|
||
|
|
return {
|
||
|
|
exitCode: USAGE_ERROR,
|
||
|
|
message: 'No config found. Run "apophis init" to create one.',
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const config = loadResult.config
|
||
|
|
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
|
||
|
|
|
||
|
|
// 2. Run policy engine checks
|
||
|
|
const policyEngine = new PolicyEngine({
|
||
|
|
config,
|
||
|
|
env,
|
||
|
|
mode: 'qualify',
|
||
|
|
profileName: profile || undefined,
|
||
|
|
presetName: loadResult.presetName || undefined,
|
||
|
|
})
|
||
|
|
|
||
|
|
const policyResult = policyEngine.check()
|
||
|
|
|
||
|
|
if (!policyResult.allowed) {
|
||
|
|
const message = [
|
||
|
|
'Policy check failed:',
|
||
|
|
...policyResult.errors.map(e => ` ✗ ${e}`),
|
||
|
|
].join('\n')
|
||
|
|
|
||
|
|
return {
|
||
|
|
exitCode: USAGE_ERROR,
|
||
|
|
message,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// 3. Generate seed if omitted
|
||
|
|
const seed = explicitSeed ?? generateSeed()
|
||
|
|
if (!ctx.options.quiet && format === 'human') {
|
||
|
|
console.log(`Seed: ${seed}`)
|
||
|
|
}
|
||
|
|
|
||
|
|
// 4. Resolve profile gates
|
||
|
|
const profileDef = profile ? config.profiles?.[profile] : undefined
|
||
|
|
const gates = resolveProfileGates(profileDef?.features)
|
||
|
|
|
||
|
|
// 5. Build scenario configs from profile routes
|
||
|
|
const routes = profileDef?.routes ?? []
|
||
|
|
const scenarios = buildScenarioConfigs(routes, seed)
|
||
|
|
|
||
|
|
// 6. Build stateful config
|
||
|
|
const presetName = profileDef?.preset
|
||
|
|
const preset = presetName ? config.presets?.[presetName] : undefined
|
||
|
|
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
|
||
|
|
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
|
||
|
|
const statefulConfig: TestConfig | undefined = gates.stateful
|
||
|
|
? {
|
||
|
|
depth: presetDepth,
|
||
|
|
generationProfile: resolvedGenerationProfile,
|
||
|
|
seed,
|
||
|
|
timeout: presetTimeout,
|
||
|
|
routes: profileDef?.routes,
|
||
|
|
}
|
||
|
|
: undefined
|
||
|
|
|
||
|
|
// 7. Build chaos config
|
||
|
|
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
|
||
|
|
? {
|
||
|
|
probability: 0.5,
|
||
|
|
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
|
||
|
|
error: { probability: 0.2, statusCode: 503 },
|
||
|
|
dropout: { probability: 0.2, statusCode: 504 },
|
||
|
|
corruption: { probability: 0.1 },
|
||
|
|
}
|
||
|
|
: undefined
|
||
|
|
|
||
|
|
// 8. Load the Fastify app for execution
|
||
|
|
// Try to import the app from the fixture
|
||
|
|
let fastify: FastifyAppLike | undefined
|
||
|
|
try {
|
||
|
|
const appPath = resolve(workingDir, 'app.js')
|
||
|
|
const appUrl = pathToFileURL(appPath)
|
||
|
|
appUrl.searchParams.set('apophisRun', String(Date.now()))
|
||
|
|
const appModule = await import(appUrl.href)
|
||
|
|
fastify = (appModule.default || appModule) as FastifyAppLike
|
||
|
|
if (fastify && typeof fastify.ready === 'function') {
|
||
|
|
await fastify.ready()
|
||
|
|
}
|
||
|
|
} catch (err) {
|
||
|
|
// App not available — return a result indicating no app to test
|
||
|
|
if (process.env.APOPHIS_DEBUG === '1') {
|
||
|
|
console.error('Failed to load app:', err)
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
exitCode: USAGE_ERROR,
|
||
|
|
message: 'No Fastify app found. Ensure app.js exports a Fastify instance.',
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
// 9. Discover routes for chaos
|
||
|
|
const appRoutes = await discoverAppRoutes(fastify)
|
||
|
|
|
||
|
|
// 10. Run qualify execution
|
||
|
|
const deps = {
|
||
|
|
fastify: fastify as any,
|
||
|
|
seed,
|
||
|
|
timeout: presetTimeout,
|
||
|
|
}
|
||
|
|
|
||
|
|
const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
|
||
|
|
|
||
|
|
// 11. Build artifact first so we can reference it for guardrails
|
||
|
|
const artifact = buildArtifact(runResult, {
|
||
|
|
cwd: workingDir,
|
||
|
|
configPath: loadResult.configPath,
|
||
|
|
profile: profile || undefined,
|
||
|
|
preset: presetName,
|
||
|
|
env,
|
||
|
|
seed,
|
||
|
|
})
|
||
|
|
|
||
|
|
// 12. Signal quality guardrails — fail if zero checks executed
|
||
|
|
const execSummary = runResult.executionSummary
|
||
|
|
const warnings: string[] = [...artifact.warnings]
|
||
|
|
|
||
|
|
if (execSummary.totalExecuted === 0) {
|
||
|
|
await emitArtifact(artifact, {
|
||
|
|
command: 'qualify',
|
||
|
|
cwd: workingDir,
|
||
|
|
preferredDir: artifactDir || config.artifactDir,
|
||
|
|
force: true,
|
||
|
|
})
|
||
|
|
|
||
|
|
return {
|
||
|
|
exitCode: BEHAVIORAL_FAILURE,
|
||
|
|
message: 'Qualify failed: zero checks executed. No scenarios, stateful tests, or chaos runs were performed. Verify profile gates and app configuration.',
|
||
|
|
artifact,
|
||
|
|
warnings: artifact.warnings,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Warn if execution counts are suspiciously low
|
||
|
|
if (gates.scenario && execSummary.scenariosRun === 0) {
|
||
|
|
warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
|
||
|
|
}
|
||
|
|
if (gates.stateful && execSummary.statefulTestsRun === 0) {
|
||
|
|
warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
|
||
|
|
}
|
||
|
|
if (gates.chaos && execSummary.chaosRunsRun === 0) {
|
||
|
|
warnings.push('WARNING: chaos gate enabled but zero chaos runs executed. Check chaos config and route availability.')
|
||
|
|
}
|
||
|
|
|
||
|
|
// 12. Write artifact if configured or on failure
|
||
|
|
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
|
||
|
|
await emitArtifact(artifact, {
|
||
|
|
command: 'qualify',
|
||
|
|
cwd: workingDir,
|
||
|
|
preferredDir: artifactDir || config.artifactDir,
|
||
|
|
force: shouldEmitArtifact,
|
||
|
|
})
|
||
|
|
|
||
|
|
// 13. Format output based on format option
|
||
|
|
const outputCtx: OutputContext = {
|
||
|
|
isTTY: ctx.isTTY,
|
||
|
|
isCI: ctx.isCI,
|
||
|
|
colorMode: ctx.options.color,
|
||
|
|
}
|
||
|
|
|
||
|
|
let message = ''
|
||
|
|
if (!ctx.options.quiet) {
|
||
|
|
if (format === 'json') {
|
||
|
|
message = renderJsonArtifact(artifact)
|
||
|
|
} else if (format === 'json-summary') {
|
||
|
|
message = renderJsonSummaryArtifact(artifact)
|
||
|
|
} else if (format === 'ndjson') {
|
||
|
|
// For ndjson, we don't return a message string; events are streamed
|
||
|
|
message = ''
|
||
|
|
} else if (format === 'ndjson-summary') {
|
||
|
|
// Concise ndjson: only summary events
|
||
|
|
message = ''
|
||
|
|
} else {
|
||
|
|
// human format
|
||
|
|
message = renderHumanArtifact(artifact, outputCtx)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return {
|
||
|
|
exitCode: runResult.passed ? SUCCESS : BEHAVIORAL_FAILURE,
|
||
|
|
artifact,
|
||
|
|
message,
|
||
|
|
warnings: artifact.warnings,
|
||
|
|
}
|
||
|
|
} finally {
|
||
|
|
if (fastify && typeof fastify.close === 'function') {
|
||
|
|
try {
|
||
|
|
await fastify.close()
|
||
|
|
} catch (closeErr) {
|
||
|
|
if (process.env.APOPHIS_DEBUG === '1') {
|
||
|
|
console.error('Failed to close Fastify app after qualify run:', closeErr)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} catch (error) {
|
||
|
|
if (error instanceof GenerationProfileResolutionError) {
|
||
|
|
return {
|
||
|
|
exitCode: USAGE_ERROR,
|
||
|
|
message: error.message,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
const message = error instanceof Error ? error.message : String(error)
|
||
|
|
return {
|
||
|
|
exitCode: INTERNAL_ERROR,
|
||
|
|
message: `Internal error in qualify command: ${message}`,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// CLI adapter
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Adapter that bridges the CLI framework (cac) to the qualify command handler.
|
||
|
|
* This function signature matches what the CLI core expects.
|
||
|
|
*/
|
||
|
|
export async function handleQualify(
|
||
|
|
args: string[],
|
||
|
|
ctx: CliContext,
|
||
|
|
): Promise<number> {
|
||
|
|
const options: QualifyOptions = {
|
||
|
|
profile: ctx.options.profile || undefined,
|
||
|
|
generationProfile: ctx.options.generationProfile,
|
||
|
|
seed: undefined,
|
||
|
|
config: ctx.options.config || undefined,
|
||
|
|
cwd: ctx.cwd,
|
||
|
|
format: ctx.options.format as QualifyOptions['format'],
|
||
|
|
quiet: ctx.options.quiet,
|
||
|
|
verbose: ctx.options.verbose,
|
||
|
|
artifactDir: ctx.options.artifactDir || undefined,
|
||
|
|
}
|
||
|
|
|
||
|
|
const seedIdx = args.indexOf('--seed')
|
||
|
|
if (seedIdx !== -1 && args[seedIdx + 1]) {
|
||
|
|
const parsed = parseInt(args[seedIdx + 1]!, 10)
|
||
|
|
if (!isNaN(parsed)) {
|
||
|
|
options.seed = parsed
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const generationProfileIdx = args.indexOf('--generation-profile')
|
||
|
|
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
|
||
|
|
options.generationProfile = args[generationProfileIdx + 1]
|
||
|
|
}
|
||
|
|
|
||
|
|
const result = await qualifyCommand(options, ctx)
|
||
|
|
const format = options.format || ctx.options.format || 'human'
|
||
|
|
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
|
||
|
|
|
||
|
|
if (!ctx.options.quiet) {
|
||
|
|
if (format === 'json') {
|
||
|
|
if (result.artifact) {
|
||
|
|
console.log(renderJsonArtifact(result.artifact))
|
||
|
|
} else {
|
||
|
|
console.log(renderJson({
|
||
|
|
exitCode: result.exitCode,
|
||
|
|
message: result.message,
|
||
|
|
warnings: result.warnings,
|
||
|
|
}))
|
||
|
|
}
|
||
|
|
} else if (format === 'json-summary') {
|
||
|
|
if (result.artifact) {
|
||
|
|
console.log(renderJsonSummaryArtifact(result.artifact))
|
||
|
|
} else {
|
||
|
|
console.log(renderJson({
|
||
|
|
exitCode: result.exitCode,
|
||
|
|
message: result.message,
|
||
|
|
warnings: result.warnings,
|
||
|
|
}))
|
||
|
|
}
|
||
|
|
} else if (format === 'ndjson') {
|
||
|
|
if (result.artifact) {
|
||
|
|
renderNdjsonArtifact(result.artifact)
|
||
|
|
} else {
|
||
|
|
process.stdout.write(JSON.stringify({
|
||
|
|
type: 'run.completed',
|
||
|
|
command: 'qualify',
|
||
|
|
exitCode: result.exitCode,
|
||
|
|
message: result.message,
|
||
|
|
warnings: result.warnings,
|
||
|
|
}) + '\n')
|
||
|
|
}
|
||
|
|
} else if (format === 'ndjson-summary') {
|
||
|
|
if (result.artifact) {
|
||
|
|
renderNdjsonSummaryArtifact(result.artifact)
|
||
|
|
} else {
|
||
|
|
process.stdout.write(JSON.stringify({
|
||
|
|
type: 'run.completed',
|
||
|
|
command: 'qualify',
|
||
|
|
exitCode: result.exitCode,
|
||
|
|
message: result.message,
|
||
|
|
warnings: result.warnings,
|
||
|
|
}) + '\n')
|
||
|
|
}
|
||
|
|
} else if (result.message) {
|
||
|
|
console.log(result.message)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Print warnings in human mode only
|
||
|
|
if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
|
||
|
|
for (const warning of result.warnings) {
|
||
|
|
console.warn(`Warning: ${warning}`)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return result.exitCode
|
||
|
|
}
|