John Dvorak
839 lines
28 KiB
TypeScript
839 lines
28 KiB
TypeScript
/**
|
|
* S6: Qualify thread - Qualify command handler
|
|
*
|
|
* Responsibilities:
|
|
* - Load config and resolve profile
|
|
* - Block prod runs by default (policy engine)
|
|
* - Run scenario/stateful/chaos based on profile
|
|
* - Generate seed if omitted, always print it
|
|
* - Rich artifact emission with step traces
|
|
* - Handle cleanup failures separately
|
|
* - Exit 0 on pass, 1 on qualification failure, 2 on safety violation
|
|
*
|
|
* Architecture:
|
|
* - Dependency injection: all dependencies passed explicitly
|
|
* - No optional imports — everything is required or injected
|
|
* - Inline comments for documentation
|
|
*/
|
|
|
|
import type { CliContext } from '../../core/context.js'
|
|
import { loadConfig } from '../../core/config-loader.js'
|
|
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
|
|
|
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
|
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
|
|
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
|
import {
|
|
runQualify,
|
|
resolveProfileGates,
|
|
type QualifyRunResult,
|
|
type StepTrace,
|
|
type CleanupFailure,
|
|
} from './runner.js'
|
|
import { isGitRepo, filterChangedRoutes } from '../verify/runner.js'
|
|
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
|
|
import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
|
|
import { renderHumanArtifact } from '../../renderers/human.js'
|
|
import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../../renderers/json.js'
|
|
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
|
|
import type { OutputContext } from '../../renderers/shared.js'
|
|
import { resolve } from 'node:path'
|
|
|
|
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
|
|
|
|
function normalizeRouteIdentity(route: string): string {
|
|
const normalized = route.trim().replace(/\s+/g, ' ')
|
|
const [method, ...pathParts] = normalized.split(' ')
|
|
if (!method || pathParts.length === 0) {
|
|
return normalized
|
|
}
|
|
return `${method.toUpperCase()} ${pathParts.join(' ')}`
|
|
}
|
|
|
|
function isReplayCompatibleRoute(route: string): boolean {
|
|
return ROUTE_IDENTITY_PATTERN.test(route)
|
|
}
|
|
|
|
function coerceTimeout(value: unknown): number | undefined {
|
|
return typeof value === 'number' ? value : undefined
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface QualifyOptions {
|
|
profile?: string
|
|
seed?: number
|
|
changed?: boolean
|
|
config?: string
|
|
cwd?: string
|
|
format?: 'human' | 'json' | 'ndjson'
|
|
quiet?: boolean
|
|
verbose?: boolean
|
|
artifactDir?: string
|
|
}
|
|
|
|
interface FastifyAppLike {
|
|
ready?: () => Promise<void>
|
|
close?: () => Promise<void>
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Seed generation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Generate a deterministic seed if none provided.
|
|
* Uses current time + process pid + counter for uniqueness.
|
|
*/
|
|
let seedCounter = 0
|
|
export function generateSeed(): number {
|
|
seedCounter++
|
|
return Date.now() + (process.pid || 0) + seedCounter
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Route discovery helper
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Discover routes from the Fastify app for chaos execution.
|
|
* Uses the shared discovery module for capture + fallback support.
|
|
*/
|
|
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
|
|
const { discoverRoutes } = await import('../../../domain/discovery.js')
|
|
return discoverRoutes(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario builder from profile
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build scenario configs from profile routes for protocol-lab fixture.
|
|
* Creates an OAuth-like multi-step scenario.
|
|
*/
|
|
function buildScenarioConfigs(routes: string[], seed: number): ScenarioConfig[] {
|
|
// For the protocol-lab fixture, build the OAuth scenario
|
|
const hasOAuth = routes.some(r => r.includes('/oauth/authorize'))
|
|
if (!hasOAuth) return []
|
|
|
|
const rng = new SeededRng(seed)
|
|
const clientId = `client-${Math.floor(rng.next() * 10000)}`
|
|
|
|
return [{
|
|
name: 'oauth-flow',
|
|
steps: [
|
|
{
|
|
name: 'authorize',
|
|
request: {
|
|
method: 'POST',
|
|
url: '/oauth/authorize',
|
|
body: {
|
|
client_id: clientId,
|
|
redirect_uri: 'http://localhost/callback',
|
|
scope: 'read',
|
|
},
|
|
},
|
|
expect: ['status:200', 'response_body(this).code != null'],
|
|
capture: { code: 'response_body(this).code' },
|
|
},
|
|
{
|
|
name: 'token',
|
|
request: {
|
|
method: 'POST',
|
|
url: '/oauth/token',
|
|
body: {
|
|
code: '$authorize.code',
|
|
client_id: clientId,
|
|
client_secret: 'secret',
|
|
redirect_uri: 'http://localhost/callback',
|
|
},
|
|
},
|
|
expect: ['status:200', 'response_body(this).access_token != null'],
|
|
capture: { accessToken: 'response_body(this).access_token' },
|
|
},
|
|
{
|
|
name: 'user',
|
|
request: {
|
|
method: 'GET',
|
|
url: '/api/user',
|
|
headers: {
|
|
authorization: 'Bearer $token.accessToken',
|
|
},
|
|
},
|
|
expect: ['status:200', 'response_body(this).id != null'],
|
|
},
|
|
],
|
|
}]
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Artifact builder
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build a rich artifact document from qualify results.
|
|
* Includes step traces, cleanup failures, and replay info.
|
|
*/
|
|
export function buildArtifact(
|
|
runResult: QualifyRunResult,
|
|
options: {
|
|
cwd: string
|
|
configPath?: string
|
|
profile?: string
|
|
preset?: string
|
|
env: string
|
|
seed: number
|
|
},
|
|
): Artifact {
|
|
const failures: FailureRecord[] = []
|
|
const warnings: string[] = []
|
|
const replayCompatibleExecutedRoutes = (runResult.executedRoutes || [])
|
|
.map(normalizeRouteIdentity)
|
|
.filter(isReplayCompatibleRoute)
|
|
|
|
// Collect scenario failures
|
|
for (const scenario of runResult.scenarioResults) {
|
|
if (!scenario.ok) {
|
|
for (let stepIdx = 0; stepIdx < scenario.steps.length; stepIdx++) {
|
|
const step = scenario.steps[stepIdx]!
|
|
if (!step.ok && step.diagnostics) {
|
|
// Use actual HTTP route from step trace for stable replay identity
|
|
const trace = runResult.stepTraces.find(
|
|
t => t.name === step.name && t.status === 'failed'
|
|
)
|
|
const route = normalizeRouteIdentity(trace?.route || `${scenario.name} / ${step.name}`)
|
|
if (!isReplayCompatibleRoute(route)) {
|
|
warnings.push(`Scenario step "${scenario.name}/${step.name}" did not resolve to METHOD /path route identity.`)
|
|
}
|
|
failures.push({
|
|
route,
|
|
contract: step.diagnostics.formula || 'scenario-step',
|
|
expected: step.diagnostics.expected || 'success',
|
|
observed: step.diagnostics.error || 'failure',
|
|
seed: runResult.seed,
|
|
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
|
|
category: step.diagnostics.error ? classifyError(step.diagnostics.error) : ErrorTaxonomy.RUNTIME,
|
|
diff: step.diagnostics.diff ?? undefined,
|
|
actual: step.diagnostics.actual ?? undefined,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Collect stateful failures
|
|
if (runResult.statefulResult) {
|
|
let fallbackRouteIdx = 0
|
|
for (const test of runResult.statefulResult.tests) {
|
|
if (!test.ok) {
|
|
let route = normalizeRouteIdentity(test.name)
|
|
if (!isReplayCompatibleRoute(route)) {
|
|
route = replayCompatibleExecutedRoutes[fallbackRouteIdx] || route
|
|
fallbackRouteIdx++
|
|
}
|
|
if (!isReplayCompatibleRoute(route)) {
|
|
warnings.push(`Stateful failure "${test.name}" did not resolve to METHOD /path route identity.`)
|
|
}
|
|
failures.push({
|
|
route,
|
|
contract: test.diagnostics?.formula || 'stateful-test',
|
|
expected: test.diagnostics?.expected || 'success',
|
|
observed: test.diagnostics?.error || 'failure',
|
|
seed: runResult.seed,
|
|
replayCommand: `apophis replay --artifact <artifact-path-unavailable>`,
|
|
category: test.diagnostics?.error ? classifyError(test.diagnostics.error) : ErrorTaxonomy.RUNTIME,
|
|
diff: test.diagnostics?.diff ?? undefined,
|
|
actual: test.diagnostics?.actual ?? undefined,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
const totalTests =
|
|
runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
|
|
(runResult.statefulResult?.summary.passed ?? 0) +
|
|
(runResult.statefulResult?.summary.failed ?? 0)
|
|
|
|
const passedTests =
|
|
runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
|
|
(runResult.statefulResult?.summary.passed ?? 0)
|
|
|
|
if (runResult.cleanupFailures.length > 0) {
|
|
warnings.push(
|
|
`Cleanup failures: ${runResult.cleanupFailures.map(c => `${c.resource}: ${c.error}`).join(', ')}`
|
|
)
|
|
}
|
|
|
|
// Build cleanup outcomes from cleanup failures
|
|
const cleanupOutcomes = runResult.cleanupFailures.map(cf => ({
|
|
resource: cf.resource,
|
|
cleaned: false,
|
|
error: cf.error,
|
|
}))
|
|
|
|
// Build execution summary from runner result
|
|
const executionSummary = runResult.executionSummary
|
|
|
|
// Build per-gate coverage breakdown for clear artifact interpretation
|
|
const scenarioRoutes = [...new Set(
|
|
runResult.stepTraces
|
|
.filter(t => t.status === 'passed' || t.status === 'failed')
|
|
.map(t => t.route)
|
|
.filter(Boolean)
|
|
)]
|
|
const statefulRoutes = runResult.statefulResult
|
|
? [...new Set(
|
|
runResult.statefulResult.tests
|
|
.filter(t => !t.directive)
|
|
.map(t => normalizeRouteIdentity(t.name))
|
|
)]
|
|
: []
|
|
const chaosRoutesPlanned: string[] = []
|
|
const chaosRoutesExecuted: string[] = []
|
|
for (const r of runResult.chaosResults ?? []) {
|
|
if (r.applied) chaosRoutesExecuted.push(r.route)
|
|
}
|
|
// Infer planned from chaos results or executionSummary
|
|
if (executionSummary.chaosRoutesPlanned > 0 && chaosRoutesExecuted.length > 0) {
|
|
chaosRoutesPlanned.push(...chaosRoutesExecuted)
|
|
}
|
|
|
|
const coverageBreakdown = {
|
|
scenario: {
|
|
routesCovered: scenarioRoutes,
|
|
stepsTotal: executionSummary.totalSteps,
|
|
stepsPassed: runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0),
|
|
},
|
|
stateful: {
|
|
routesCovered: statefulRoutes,
|
|
testsTotal: executionSummary.statefulTestsRun,
|
|
testsPassed: runResult.statefulResult?.summary.passed ?? 0,
|
|
},
|
|
chaos: {
|
|
routesPlanned: chaosRoutesPlanned,
|
|
routesExecuted: chaosRoutesExecuted,
|
|
runsTotal: executionSummary.chaosRunsRun,
|
|
runsPassed: (runResult.chaosResults ?? []).filter(r => r.contractsPassed).length,
|
|
},
|
|
}
|
|
|
|
// Build profile gates from the result context
|
|
// We need to pass gates through or infer from results
|
|
const profileGates = {
|
|
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
|
|
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
|
|
chaos: executionSummary.chaosRoutesPlanned > 0,
|
|
}
|
|
|
|
// Deterministic parameters for audit
|
|
const deterministicParams = {
|
|
seed: runResult.seed,
|
|
profileGates,
|
|
}
|
|
|
|
return {
|
|
version: 'apophis-artifact/1',
|
|
command: 'qualify',
|
|
mode: 'qualify',
|
|
cwd: options.cwd,
|
|
configPath: options.configPath,
|
|
profile: options.profile,
|
|
preset: options.preset,
|
|
env: options.env,
|
|
seed: options.seed,
|
|
startedAt: new Date(Date.now() - runResult.durationMs).toISOString(),
|
|
durationMs: runResult.durationMs,
|
|
summary: {
|
|
total: totalTests,
|
|
passed: passedTests,
|
|
failed: failures.length,
|
|
skipped: (runResult.statefulResult?.summary.skipped ?? 0),
|
|
},
|
|
executionSummary,
|
|
coverageBreakdown,
|
|
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
|
|
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
|
|
route: sr.route,
|
|
executed: false,
|
|
reason: sr.reason,
|
|
})),
|
|
stepTraces: runResult.stepTraces,
|
|
cleanupOutcomes,
|
|
profileGates,
|
|
deterministicParams,
|
|
failures,
|
|
artifacts: [],
|
|
warnings,
|
|
exitReason: runResult.passed ? 'success' : 'behavioral_failure',
|
|
}
|
|
}
|
|
|
|
function attachReplayCommands(artifact: Artifact, artifactPath: string): void {
|
|
for (const failure of artifact.failures) {
|
|
failure.replayCommand = `apophis replay --artifact ${artifactPath}`
|
|
}
|
|
}
|
|
|
|
async function emitArtifact(
|
|
artifact: Artifact,
|
|
options: {
|
|
command: 'qualify'
|
|
cwd: string
|
|
preferredDir?: string
|
|
force: boolean
|
|
},
|
|
): Promise<string | undefined> {
|
|
if (!options.force && !options.preferredDir) {
|
|
return undefined
|
|
}
|
|
|
|
const defaultDir = resolve(options.cwd, 'reports', 'apophis')
|
|
const candidateDirs = [options.preferredDir, defaultDir].filter(Boolean) as string[]
|
|
const attempted = new Set<string>()
|
|
|
|
for (const dir of candidateDirs) {
|
|
if (attempted.has(dir)) continue
|
|
attempted.add(dir)
|
|
try {
|
|
const { mkdirSync, writeFileSync } = await import('node:fs')
|
|
const artifactPath = resolve(dir, `${options.command}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`)
|
|
mkdirSync(dir, { recursive: true })
|
|
attachReplayCommands(artifact, artifactPath)
|
|
writeFileSync(artifactPath, JSON.stringify(artifact, null, 2))
|
|
if (!artifact.artifacts.includes(artifactPath)) {
|
|
artifact.artifacts.push(artifactPath)
|
|
}
|
|
return artifactPath
|
|
} catch {
|
|
// Try fallback directory if available.
|
|
}
|
|
}
|
|
|
|
artifact.warnings.push('Failed to write artifact to disk')
|
|
return undefined
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Output formatting
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main command handler
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Main qualify command handler.
|
|
*
|
|
* Flow:
|
|
* 1. Load and resolve config
|
|
* 2. Run policy engine checks (block prod by default)
|
|
* 3. Generate seed if omitted, always print it
|
|
* 4. Resolve profile gates (scenario/stateful/chaos)
|
|
* 5. Build scenario configs from profile routes
|
|
* 6. Run execution modes
|
|
* 7. Build rich artifact with step traces
|
|
* 8. Handle cleanup failures separately
|
|
* 9. Return appropriate exit code
|
|
*/
|
|
export async function qualifyCommand(
|
|
options: QualifyOptions,
|
|
ctx: CliContext,
|
|
): Promise<CommandResult> {
|
|
const {
|
|
profile,
|
|
seed: explicitSeed,
|
|
changed,
|
|
config: configPath,
|
|
cwd,
|
|
artifactDir,
|
|
} = options
|
|
const workingDir = cwd || ctx.cwd
|
|
const format = options.format || ctx.options.format || 'human'
|
|
|
|
// Detect environment
|
|
const env = detectEnvironment()
|
|
|
|
try {
|
|
// 1. Load config
|
|
const loadResult = await loadConfig({
|
|
cwd: workingDir,
|
|
configPath,
|
|
profileName: profile,
|
|
env,
|
|
})
|
|
|
|
if (!loadResult.configPath) {
|
|
return {
|
|
exitCode: USAGE_ERROR,
|
|
message: 'No config found. Run "apophis init" to create one.',
|
|
}
|
|
}
|
|
|
|
const config = loadResult.config
|
|
|
|
// 2. Run policy engine checks
|
|
const policyEngine = new PolicyEngine({
|
|
config,
|
|
env,
|
|
mode: 'qualify',
|
|
profileName: profile || undefined,
|
|
presetName: loadResult.presetName || undefined,
|
|
})
|
|
|
|
const policyResult = policyEngine.check()
|
|
|
|
if (!policyResult.allowed) {
|
|
const message = [
|
|
'Policy check failed:',
|
|
...policyResult.errors.map(e => ` ✗ ${e}`),
|
|
].join('\n')
|
|
|
|
return {
|
|
exitCode: USAGE_ERROR,
|
|
message,
|
|
}
|
|
}
|
|
|
|
// 3. Generate seed if omitted
|
|
const seed = explicitSeed ?? generateSeed()
|
|
if (!ctx.options.quiet && format === 'human') {
|
|
console.log(`Seed: ${seed}`)
|
|
}
|
|
|
|
// 4. Resolve profile gates
|
|
const profileDef = profile ? config.profiles?.[profile] : undefined
|
|
const gates = resolveProfileGates(profileDef?.features)
|
|
|
|
// 5. Build scenario configs from profile routes and config
|
|
const routes = profileDef?.routes ?? []
|
|
const fixtureScenarios = buildScenarioConfigs(routes, seed)
|
|
|
|
const configScenarios: ScenarioConfig[] = (config.scenarios ?? []).map(s => ({
|
|
name: s.name,
|
|
steps: s.steps.map(step => ({
|
|
name: step.name,
|
|
request: {
|
|
method: step.request.method as ScenarioConfig['steps'][0]['request']['method'],
|
|
url: step.request.url,
|
|
body: step.request.body,
|
|
headers: step.request.headers,
|
|
},
|
|
expect: step.expect,
|
|
capture: step.capture,
|
|
})),
|
|
}))
|
|
|
|
const scenarios = [...fixtureScenarios, ...configScenarios]
|
|
|
|
// 6. Build stateful config
|
|
const presetName = profileDef?.preset
|
|
const preset = presetName ? config.presets?.[presetName] : undefined
|
|
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
|
|
const statefulConfig: TestConfig | undefined = gates.stateful
|
|
? {
|
|
seed,
|
|
timeout: presetTimeout,
|
|
routes: profileDef?.routes,
|
|
}
|
|
: undefined
|
|
|
|
// 7. Build chaos config
|
|
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
|
|
? {
|
|
probability: 0.5,
|
|
strategy: preset.chaosStrategy as ChaosConfig['strategy'],
|
|
sampleSize: preset.chaosSampleSize,
|
|
sampleRoutes: preset.chaosSampleRoutes,
|
|
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
|
|
error: { probability: 0.2, statusCode: 503 },
|
|
dropout: { probability: 0.2, statusCode: 504 },
|
|
corruption: { probability: 0.1 },
|
|
}
|
|
: undefined
|
|
|
|
// 8. Load the Fastify app for execution
|
|
let fastify: FastifyAppLike | undefined
|
|
let appEntrypoint: string | undefined
|
|
try {
|
|
const { loadApp } = await import('../../core/app-loader.js')
|
|
const loaded = await loadApp(workingDir)
|
|
fastify = loaded.fastify as FastifyAppLike
|
|
appEntrypoint = loaded.entrypoint
|
|
if (fastify && typeof fastify.ready === 'function') {
|
|
await fastify.ready()
|
|
}
|
|
} catch (err) {
|
|
const errorMessage = err instanceof Error ? err.message : String(err)
|
|
return {
|
|
exitCode: USAGE_ERROR,
|
|
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
|
}
|
|
}
|
|
|
|
try {
|
|
// 9. Discover routes for chaos (with discovery metadata)
|
|
const allRoutes = await discoverAppRoutes(fastify)
|
|
let appRoutes = allRoutes
|
|
|
|
// 9a. Apply --changed filter
|
|
if (changed) {
|
|
const inGit = await isGitRepo(workingDir)
|
|
if (!inGit) {
|
|
return {
|
|
exitCode: USAGE_ERROR,
|
|
message: '--changed requires a git repository. Current directory is not inside a git repo.',
|
|
}
|
|
}
|
|
appRoutes = await filterChangedRoutes(allRoutes, workingDir)
|
|
const matched = appRoutes.length
|
|
const total = allRoutes.length
|
|
if (!ctx.options.quiet && format === 'human') {
|
|
console.log(`${matched}/${total} routes match --changed filter`)
|
|
}
|
|
if (matched === 0) {
|
|
return {
|
|
exitCode: SUCCESS,
|
|
message: `No routes changed since HEAD. All ${total} routes skipped.`,
|
|
}
|
|
}
|
|
}
|
|
|
|
// Collect discovery warnings
|
|
const { discoverRouteDetails } = await import('../../../domain/discovery.js')
|
|
const discoveryResult = discoverRouteDetails(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
|
|
const discoveryWarnings: string[] = [
|
|
...discoveryResult.warnings,
|
|
]
|
|
if (discoveryResult.source === 'print-routes' && !discoveryResult.hasSchemaMetadata) {
|
|
discoveryWarnings.push(
|
|
'Routes were discovered without schema metadata. Behavioral contracts and chaos injection may be limited. ' +
|
|
'Register APOPHIS before defining routes or use createFastify().'
|
|
)
|
|
}
|
|
|
|
// 10. Run qualify execution
|
|
const deps = {
|
|
fastify: fastify as any,
|
|
seed,
|
|
timeout: presetTimeout,
|
|
pluginContractRegistry: (fastify as unknown as Record<string, unknown>)?._apophisPluginContractRegistry as
|
|
import('../../../domain/plugin-contracts.js').PluginContractRegistry | undefined,
|
|
}
|
|
|
|
const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
|
|
|
|
// 11. Build artifact first so we can reference it for guardrails
|
|
const artifact = buildArtifact(runResult, {
|
|
cwd: workingDir,
|
|
configPath: loadResult.configPath,
|
|
profile: profile || undefined,
|
|
preset: presetName,
|
|
env,
|
|
seed,
|
|
})
|
|
|
|
// Drain plugin contract registry warnings if app registered APOPHIS
|
|
const pcr = (fastify as any)?._apophisPluginContractRegistry
|
|
if (pcr?.drainWarnings) {
|
|
const pcrWarnings = pcr.drainWarnings()
|
|
if (pcrWarnings.length > 0) {
|
|
artifact.warnings.push(...pcrWarnings)
|
|
}
|
|
}
|
|
|
|
// 12. Signal quality guardrails — fail if zero checks executed
|
|
const execSummary = runResult.executionSummary
|
|
const warnings: string[] = [...artifact.warnings, ...discoveryWarnings]
|
|
|
|
if (execSummary.totalExecuted === 0) {
|
|
await emitArtifact(artifact, {
|
|
command: 'qualify',
|
|
cwd: workingDir,
|
|
preferredDir: artifactDir || config.artifactDir,
|
|
force: true,
|
|
})
|
|
|
|
return {
|
|
exitCode: BEHAVIORAL_FAILURE,
|
|
message: 'Qualify failed: zero checks executed. No scenarios, stateful tests, or chaos runs were performed. Verify profile gates and app configuration.',
|
|
artifact,
|
|
warnings: artifact.warnings,
|
|
}
|
|
}
|
|
|
|
// Warn if execution counts are suspiciously low
|
|
if (gates.scenario && execSummary.scenariosRun === 0) {
|
|
warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
|
|
}
|
|
if (gates.stateful && execSummary.statefulTestsRun === 0 && execSummary.statefulSkipped > 0) {
|
|
warnings.push(`WARNING: stateful gate enabled but all ${execSummary.statefulSkipped} commands were skipped (preconditions not met). Check required headers, scopes, or plugin contract settings.`)
|
|
} else if (gates.stateful && execSummary.statefulTestsRun === 0) {
|
|
warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
|
|
}
|
|
if (gates.chaos && execSummary.chaosRunsRun === 0) {
|
|
warnings.push('WARNING: chaos gate enabled but zero chaos runs executed. Check chaos config and route availability.')
|
|
}
|
|
|
|
// 12. Write artifact if configured or on failure
|
|
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
|
|
await emitArtifact(artifact, {
|
|
command: 'qualify',
|
|
cwd: workingDir,
|
|
preferredDir: artifactDir || config.artifactDir,
|
|
force: shouldEmitArtifact,
|
|
})
|
|
|
|
// 13. Format output based on format option
|
|
const outputCtx: OutputContext = {
|
|
isTTY: ctx.isTTY,
|
|
isCI: ctx.isCI,
|
|
colorMode: ctx.options.color,
|
|
}
|
|
|
|
let message = ''
|
|
if (!ctx.options.quiet) {
|
|
if (format === 'json') {
|
|
message = renderJsonArtifact(artifact)
|
|
} else if (format === 'json-summary') {
|
|
message = renderJsonSummaryArtifact(artifact)
|
|
} else if (format === 'ndjson') {
|
|
// For ndjson, we don't return a message string; events are streamed
|
|
message = ''
|
|
} else if (format === 'ndjson-summary') {
|
|
// Concise ndjson: only summary events
|
|
message = ''
|
|
} else {
|
|
// human format
|
|
message = renderHumanArtifact(artifact, outputCtx)
|
|
}
|
|
}
|
|
|
|
return {
|
|
exitCode: runResult.passed ? SUCCESS : BEHAVIORAL_FAILURE,
|
|
artifact,
|
|
message,
|
|
warnings: artifact.warnings,
|
|
}
|
|
} finally {
|
|
if (fastify && typeof fastify.close === 'function') {
|
|
try {
|
|
await fastify.close()
|
|
} catch (closeErr) {
|
|
if (process.env.APOPHIS_DEBUG === '1') {
|
|
console.error('Failed to close Fastify app after qualify run:', closeErr)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error)
|
|
return {
|
|
exitCode: INTERNAL_ERROR,
|
|
message: `Internal error in qualify command: ${message}`,
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CLI adapter
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Adapter that bridges the CLI framework (cac) to the qualify command handler.
|
|
* This function signature matches what the CLI core expects.
|
|
*/
|
|
export async function handleQualify(
|
|
args: string[],
|
|
ctx: CliContext,
|
|
): Promise<number> {
|
|
const options: QualifyOptions = {
|
|
profile: ctx.options.profile || undefined,
|
|
seed: undefined,
|
|
changed: false,
|
|
config: ctx.options.config || undefined,
|
|
cwd: ctx.cwd,
|
|
format: ctx.options.format as QualifyOptions['format'],
|
|
quiet: ctx.options.quiet,
|
|
verbose: ctx.options.verbose,
|
|
artifactDir: ctx.options.artifactDir || undefined,
|
|
}
|
|
|
|
const seedIdx = args.indexOf('--seed')
|
|
if (seedIdx !== -1 && args[seedIdx + 1]) {
|
|
const parsed = parseInt(args[seedIdx + 1]!, 10)
|
|
if (!isNaN(parsed)) {
|
|
options.seed = parsed
|
|
}
|
|
}
|
|
|
|
if (args.includes('--changed')) {
|
|
options.changed = true
|
|
}
|
|
|
|
const result = await qualifyCommand(options, ctx)
|
|
const format = options.format || ctx.options.format || 'human'
|
|
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
|
|
|
|
if (!ctx.options.quiet) {
|
|
if (format === 'json') {
|
|
if (result.artifact) {
|
|
console.log(renderJsonArtifact(result.artifact))
|
|
} else {
|
|
console.log(renderJson({
|
|
exitCode: result.exitCode,
|
|
message: result.message,
|
|
warnings: result.warnings,
|
|
}))
|
|
}
|
|
} else if (format === 'json-summary') {
|
|
if (result.artifact) {
|
|
console.log(renderJsonSummaryArtifact(result.artifact))
|
|
} else {
|
|
console.log(renderJson({
|
|
exitCode: result.exitCode,
|
|
message: result.message,
|
|
warnings: result.warnings,
|
|
}))
|
|
}
|
|
} else if (format === 'ndjson') {
|
|
if (result.artifact) {
|
|
renderNdjsonArtifact(result.artifact)
|
|
} else {
|
|
process.stdout.write(JSON.stringify({
|
|
type: 'run.completed',
|
|
command: 'qualify',
|
|
exitCode: result.exitCode,
|
|
message: result.message,
|
|
warnings: result.warnings,
|
|
}) + '\n')
|
|
}
|
|
} else if (format === 'ndjson-summary') {
|
|
if (result.artifact) {
|
|
renderNdjsonSummaryArtifact(result.artifact)
|
|
} else {
|
|
process.stdout.write(JSON.stringify({
|
|
type: 'run.completed',
|
|
command: 'qualify',
|
|
exitCode: result.exitCode,
|
|
message: result.message,
|
|
warnings: result.warnings,
|
|
}) + '\n')
|
|
}
|
|
} else if (result.message) {
|
|
console.log(result.message)
|
|
}
|
|
}
|
|
|
|
// Print warnings in human mode only
|
|
if (!machineMode && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
|
|
for (const warning of result.warnings) {
|
|
console.warn(`Warning: ${warning}`)
|
|
}
|
|
}
|
|
|
|
return result.exitCode
|
|
}
|