John Dvorak
570 lines
18 KiB
TypeScript
570 lines
18 KiB
TypeScript
/**
|
|
* S7: Replay thread - Replay command handler
|
|
*
|
|
* Responsibilities:
|
|
* - Load artifact from --artifact path
|
|
* - Validate artifact schema version
|
|
* - Check CLI version compatibility
|
|
* - Re-run the failing route/contract with the same seed
|
|
* - Handle source code changes since artifact (warn but attempt)
|
|
* - Handle missing/corrupted artifacts
|
|
* - Handle route no longer existing
|
|
* - Fast startup (must feel instant)
|
|
* - Exit 0 if replay reproduces same failure, 1 if different, 2 on error
|
|
*
|
|
* Architecture:
|
|
* - Dependency injection: all dependencies passed explicitly
|
|
* - No optional imports — everything is required or injected
|
|
* - Inline comments for documentation
|
|
* - Reuses verify runner for actual replay execution
|
|
*/
|
|
|
|
import type { CliContext } from '../../core/context.js'
|
|
import { loadConfig } from '../../core/config-loader.js'
|
|
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
|
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
|
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
|
|
import { runVerify } from '../verify/runner.js'
|
|
import { loadArtifact, type ArtifactLoadResult } from './loader.js'
|
|
import { renderJson } from '../../renderers/json.js'
|
|
import type { OutputContext } from '../../renderers/shared.js'
|
|
import { executeHttp } from '../../../infrastructure/http-executor.js'
|
|
import { parse } from '../../../formula/parser.js'
|
|
import { evaluateAsync } from '../../../formula/evaluator.js'
|
|
import { createOperationResolver } from '../../../formula/runtime.js'
|
|
import type { EvalContext, RouteContract } from '../../../types.js'
|
|
import { resolve } from 'node:path'
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface ReplayOptions {
|
|
artifact: string
|
|
config?: string
|
|
cwd?: string
|
|
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary'
|
|
quiet?: boolean
|
|
verbose?: boolean
|
|
route?: string
|
|
}
|
|
|
|
export interface ReplayResult {
|
|
exitCode: number
|
|
message?: string
|
|
warnings?: string[]
|
|
reproduced: boolean
|
|
originalFailure?: FailureRecord
|
|
newFailure?: FailureRecord
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Human output formatting
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Format human-readable output for replay results.
|
|
*/
|
|
function formatHumanOutput(result: ReplayResult, artifact: Artifact): string {
|
|
const lines: string[] = []
|
|
const sourceDriftDetected = (result.warnings || []).some(w =>
|
|
w.includes('Source code has changed since artifact was created') ||
|
|
w.includes('modified since artifact was created') ||
|
|
w.includes('Artifact cwd no longer exists')
|
|
)
|
|
|
|
if (result.reproduced) {
|
|
lines.push('Replay reproduced the original failure.')
|
|
lines.push('')
|
|
lines.push('Original failure')
|
|
lines.push(` Route: ${result.originalFailure?.route}`)
|
|
lines.push(` Contract: ${result.originalFailure?.contract}`)
|
|
lines.push(` Expected: ${result.originalFailure?.expected}`)
|
|
lines.push(` Observed: ${result.originalFailure?.observed}`)
|
|
lines.push(` Seed: ${artifact.seed}`)
|
|
} else if (result.newFailure) {
|
|
lines.push('Replay produced a different result.')
|
|
lines.push('')
|
|
lines.push('Original failure')
|
|
lines.push(` Route: ${result.originalFailure?.route}`)
|
|
lines.push(` Contract: ${result.originalFailure?.contract}`)
|
|
lines.push('')
|
|
lines.push('New result')
|
|
lines.push(` Route: ${result.newFailure.route}`)
|
|
lines.push(` Contract: ${result.newFailure.contract}`)
|
|
lines.push(` Expected: ${result.newFailure.expected}`)
|
|
lines.push(` Observed: ${result.newFailure.observed}`)
|
|
lines.push(` Seed: ${artifact.seed}`)
|
|
} else {
|
|
lines.push('Replay passed — failure no longer reproduces.')
|
|
lines.push('')
|
|
lines.push('Original failure')
|
|
lines.push(` Route: ${result.originalFailure?.route}`)
|
|
lines.push(` Contract: ${result.originalFailure?.contract}`)
|
|
lines.push(` Seed: ${artifact.seed}`)
|
|
}
|
|
|
|
// Add trust labeling and stabilization guidance when replay does not exactly match.
|
|
if (!result.reproduced) {
|
|
lines.push('')
|
|
lines.push('Replay confidence')
|
|
if (sourceDriftDetected) {
|
|
lines.push(' Degraded: source drift detected since artifact creation; exact reproduction is not guaranteed.')
|
|
} else {
|
|
lines.push(' Degraded: same-seed replay diverged without source drift; likely runtime/data nondeterminism.')
|
|
}
|
|
lines.push('')
|
|
lines.push('Stabilization guidance:')
|
|
lines.push(' 1. Ensure the app database/state is reset to a known baseline')
|
|
lines.push(' 2. Run with --seed for explicit control')
|
|
lines.push(' 3. Freeze time/randomness in app code and isolate external dependencies')
|
|
lines.push(' 4. Disable chaos/stateful gates in profile if not needed for this failure')
|
|
}
|
|
|
|
if (result.warnings && result.warnings.length > 0) {
|
|
lines.push('')
|
|
lines.push('Warnings')
|
|
for (const warning of result.warnings) {
|
|
lines.push(` ⚠ ${warning}`)
|
|
}
|
|
}
|
|
|
|
return lines.join('\n')
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Direct contract execution (bypasses route discovery)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Execute a contract directly against a Fastify instance without route discovery.
|
|
* Used by replay when the app doesn't have APOPHIS plugin pre-registered.
|
|
*/
|
|
async function executeContractDirect(
|
|
fastify: any,
|
|
route: string,
|
|
contract: string,
|
|
seed: number,
|
|
): Promise<{ success: boolean; observed?: string }> {
|
|
// Parse route into method and path
|
|
const parts = route.split(' ')
|
|
const method = parts[0] || 'GET'
|
|
const path = parts.slice(1).join(' ')
|
|
|
|
// Check if route exists using hasRoute
|
|
const hasRoute = typeof fastify.hasRoute === 'function' &&
|
|
fastify.hasRoute({ url: path, method })
|
|
|
|
if (!hasRoute) {
|
|
return { success: false, observed: `Route "${route}" no longer exists` }
|
|
}
|
|
|
|
// Build a minimal route contract
|
|
const routeContract: RouteContract = {
|
|
method: method as RouteContract['method'],
|
|
path,
|
|
category: 'observer',
|
|
schema: {},
|
|
requires: [],
|
|
ensures: [contract],
|
|
invariants: [],
|
|
regexPatterns: {},
|
|
validateRuntime: false,
|
|
}
|
|
|
|
// Build request
|
|
const headers: Record<string, string> = {}
|
|
|
|
// Execute request
|
|
try {
|
|
const ctx = await executeHttp(fastify, routeContract, {
|
|
method,
|
|
url: path,
|
|
headers,
|
|
query: {},
|
|
})
|
|
|
|
// Build eval context
|
|
const evalCtx: EvalContext = {
|
|
...ctx,
|
|
operationResolver: createOperationResolver(fastify, headers, ctx),
|
|
}
|
|
|
|
// Parse and evaluate contract
|
|
const parsed = parse(contract)
|
|
const result = await evaluateAsync(parsed.ast, evalCtx)
|
|
|
|
if (!result.success || !result.value) {
|
|
return {
|
|
success: false,
|
|
observed: result.success ? String(result.value) : result.error,
|
|
}
|
|
}
|
|
|
|
return { success: true }
|
|
} catch (error) {
|
|
return {
|
|
success: false,
|
|
observed: error instanceof Error ? error.message : String(error),
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Replay execution
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Run the replay by re-executing verify with the same seed and route filter.
|
|
*
|
|
* Flow:
|
|
* 1. Load the Fastify app from artifact.cwd
|
|
* 2. Run verify with the artifact's seed and route filter
|
|
* 3. Compare results to the original failure
|
|
* 4. Return whether the failure was reproduced
|
|
*/
|
|
async function executeReplay(
|
|
artifact: Artifact,
|
|
failure: FailureRecord,
|
|
artifactPath: string,
|
|
ctx: CliContext,
|
|
options?: { sourceChanged?: boolean },
|
|
): Promise<ReplayResult> {
|
|
const workingDir = artifact.cwd
|
|
const warnings: string[] = []
|
|
|
|
// Load the Fastify app
|
|
let fastify: unknown
|
|
try {
|
|
const { loadApp } = await import('../../core/app-loader.js')
|
|
const loaded = await loadApp(workingDir)
|
|
fastify = loaded.fastify
|
|
if (fastify && typeof (fastify as any).ready === 'function') {
|
|
// Only register APOPHIS plugin if not already registered
|
|
// The fixture apps already register it, so re-registering throws
|
|
const hasApophis = (fastify as any).apophis !== undefined
|
|
const canRegister = typeof (fastify as any).register === 'function'
|
|
if (!hasApophis && canRegister) {
|
|
const { apophisPlugin } = await import('../../../plugin/index.js')
|
|
if (typeof apophisPlugin === 'function') {
|
|
await (fastify as any).register(apophisPlugin, { runtime: 'off' })
|
|
}
|
|
}
|
|
await (fastify as any).ready()
|
|
}
|
|
} catch (err) {
|
|
const errorMessage = err instanceof Error ? err.message : String(err)
|
|
return {
|
|
exitCode: USAGE_ERROR,
|
|
message: `Cannot load Fastify app from ${workingDir}/app.js: ${errorMessage}`,
|
|
warnings,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
}
|
|
}
|
|
|
|
// Try to run verify first (works if app has APOPHIS plugin)
|
|
let runResult = await runVerify({
|
|
fastify: fastify as any,
|
|
seed: artifact.seed || 42,
|
|
routeFilters: [failure.route],
|
|
})
|
|
|
|
// If no routes matched, or route found but no contracts (plugin not registered before routes),
|
|
// try direct contract execution
|
|
if (runResult.noRoutesMatched || runResult.noContractsFound) {
|
|
const directResult = await executeContractDirect(
|
|
fastify as any,
|
|
failure.route,
|
|
failure.contract,
|
|
artifact.seed || 42,
|
|
)
|
|
|
|
if (!directResult.success) {
|
|
// Check if it's a route-not-found error
|
|
if (directResult.observed?.includes('no longer exists')) {
|
|
return {
|
|
exitCode: USAGE_ERROR,
|
|
message: `Route "${failure.route}" no longer exists in the application.\n` +
|
|
`The source code has drifted since the artifact was created.`,
|
|
warnings: [...warnings, `Route "${failure.route}" no longer exists`],
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
}
|
|
}
|
|
|
|
// Same failure reproduced via direct execution
|
|
return {
|
|
exitCode: BEHAVIORAL_FAILURE,
|
|
message: formatHumanOutput({
|
|
exitCode: BEHAVIORAL_FAILURE,
|
|
reproduced: true,
|
|
originalFailure: failure,
|
|
warnings,
|
|
}, artifact),
|
|
warnings,
|
|
reproduced: true,
|
|
originalFailure: failure,
|
|
}
|
|
}
|
|
|
|
// Direct execution passed — failure no longer reproduces
|
|
return {
|
|
exitCode: SUCCESS,
|
|
message: formatHumanOutput({
|
|
exitCode: SUCCESS,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
warnings,
|
|
}, artifact),
|
|
warnings,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
}
|
|
}
|
|
|
|
// Check if the same failure was reproduced
|
|
const reproducedFailure = runResult.failures.find(f =>
|
|
f.route === failure.route && f.contract === failure.contract
|
|
)
|
|
|
|
if (reproducedFailure) {
|
|
// Same failure reproduced
|
|
return {
|
|
exitCode: BEHAVIORAL_FAILURE,
|
|
message: formatHumanOutput({
|
|
exitCode: BEHAVIORAL_FAILURE,
|
|
reproduced: true,
|
|
originalFailure: failure,
|
|
warnings,
|
|
}, artifact),
|
|
warnings,
|
|
reproduced: true,
|
|
originalFailure: failure,
|
|
}
|
|
}
|
|
|
|
// Check if there are different failures
|
|
if (runResult.failures.length > 0) {
|
|
const newFailure = runResult.failures[0]
|
|
if (!newFailure) {
|
|
return {
|
|
exitCode: SUCCESS,
|
|
message: formatHumanOutput({
|
|
exitCode: SUCCESS,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
warnings,
|
|
}, artifact),
|
|
warnings,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
}
|
|
}
|
|
return {
|
|
exitCode: BEHAVIORAL_FAILURE,
|
|
message: formatHumanOutput({
|
|
exitCode: BEHAVIORAL_FAILURE,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
newFailure: {
|
|
route: newFailure.route,
|
|
contract: newFailure.contract,
|
|
expected: newFailure.expected,
|
|
observed: newFailure.observed,
|
|
seed: artifact.seed || 42,
|
|
replayCommand: `apophis replay --artifact ${artifactPath}`,
|
|
},
|
|
warnings,
|
|
}, artifact),
|
|
warnings,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
newFailure: {
|
|
route: newFailure.route,
|
|
contract: newFailure.contract,
|
|
expected: newFailure.expected,
|
|
observed: newFailure.observed,
|
|
seed: artifact.seed || 42,
|
|
replayCommand: `apophis replay --artifact ${artifactPath}`,
|
|
},
|
|
}
|
|
}
|
|
|
|
// No failures — the bug was fixed
|
|
if (!options?.sourceChanged) {
|
|
warnings.push('Replay diverged with same seed and no source drift detected. Likely runtime/data nondeterminism.')
|
|
}
|
|
return {
|
|
exitCode: SUCCESS,
|
|
message: formatHumanOutput({
|
|
exitCode: SUCCESS,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
warnings,
|
|
}, artifact),
|
|
warnings,
|
|
reproduced: false,
|
|
originalFailure: failure,
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main command handler
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Main replay command handler.
|
|
*
|
|
* Flow:
|
|
* 1. Load and validate artifact
|
|
* 2. Check CLI version compatibility
|
|
* 3. Detect source code changes (warn but continue)
|
|
* 4. Load Fastify app and re-run verify with same seed
|
|
* 5. Compare results to original failure
|
|
* 6. Return appropriate exit code
|
|
*
|
|
* Exit codes:
|
|
* - 0: Replay passed (failure no longer reproduces)
|
|
* - 1: Same failure reproduced OR different failure found
|
|
* - 2: Error (missing artifact, corrupted, route no longer exists, etc.)
|
|
*/
|
|
export async function replayCommand(
|
|
options: ReplayOptions,
|
|
ctx: CliContext,
|
|
): Promise<CommandResult> {
|
|
const { artifact: artifactPath, config: configPath, cwd } = options
|
|
const workingDir = cwd || ctx.cwd
|
|
const resolvedArtifactPath = resolve(workingDir, artifactPath)
|
|
|
|
try {
|
|
// 1. Load and validate artifact
|
|
const loadResult = loadArtifact({
|
|
artifactPath,
|
|
cwd: workingDir,
|
|
routeFilter: options.route,
|
|
})
|
|
|
|
if (!loadResult.success) {
|
|
return {
|
|
exitCode: USAGE_ERROR,
|
|
message: loadResult.message,
|
|
warnings: loadResult.warnings,
|
|
}
|
|
}
|
|
|
|
const artifact = loadResult.artifact!
|
|
const failure = loadResult.failure!
|
|
const warnings = [...loadResult.warnings]
|
|
|
|
// 2. Execute replay
|
|
const replayResult = await executeReplay(artifact, failure, resolvedArtifactPath, ctx, {
|
|
sourceChanged: loadResult.sourceChanged,
|
|
})
|
|
|
|
// Merge warnings
|
|
if (replayResult.warnings) {
|
|
warnings.push(...replayResult.warnings)
|
|
}
|
|
|
|
return {
|
|
exitCode: replayResult.exitCode as import('../../core/types.js').ExitCode,
|
|
message: replayResult.message,
|
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
}
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error)
|
|
return {
|
|
exitCode: INTERNAL_ERROR,
|
|
message: `Internal error in replay command: ${message}`,
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CLI adapter
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Adapter that bridges the CLI framework (cac) to the replay command handler.
|
|
* This function signature matches what the CLI core expects.
|
|
*/
|
|
export async function handleReplay(
|
|
args: string[],
|
|
ctx: CliContext,
|
|
): Promise<number> {
|
|
const options: ReplayOptions = {
|
|
artifact: '',
|
|
config: ctx.options.config || undefined,
|
|
cwd: ctx.cwd,
|
|
format: ctx.options.format as ReplayOptions['format'],
|
|
quiet: ctx.options.quiet,
|
|
verbose: ctx.options.verbose,
|
|
}
|
|
|
|
// Parse command-specific flags from args (passed by CLI dispatcher)
|
|
const artifactIdx = args.indexOf('--artifact')
|
|
if (artifactIdx !== -1 && args[artifactIdx + 1]) {
|
|
options.artifact = args[artifactIdx + 1]!
|
|
}
|
|
|
|
const routeIdx = args.indexOf('--route')
|
|
if (routeIdx !== -1 && args[routeIdx + 1]) {
|
|
options.route = args[routeIdx + 1]!
|
|
}
|
|
|
|
if (!options.artifact) {
|
|
const format = options.format || ctx.options.format || 'human'
|
|
if (format === 'json') {
|
|
console.log(renderJson({
|
|
exitCode: USAGE_ERROR,
|
|
error: 'Error: --artifact is required',
|
|
}))
|
|
} else if (format === 'ndjson') {
|
|
process.stdout.write(JSON.stringify({
|
|
type: 'run.completed',
|
|
command: 'replay',
|
|
exitCode: USAGE_ERROR,
|
|
error: 'Error: --artifact is required',
|
|
}) + '\n')
|
|
} else {
|
|
console.error('Error: --artifact is required')
|
|
}
|
|
return USAGE_ERROR
|
|
}
|
|
|
|
const result = await replayCommand(options, ctx)
|
|
|
|
// Output result based on format
|
|
if (!ctx.options.quiet && result.message) {
|
|
const format = options.format || ctx.options.format || 'human'
|
|
if (format === 'json') {
|
|
console.log(renderJson({
|
|
exitCode: result.exitCode,
|
|
message: result.message,
|
|
warnings: result.warnings,
|
|
}))
|
|
} else if (format === 'ndjson') {
|
|
process.stdout.write(JSON.stringify({
|
|
type: 'run.completed',
|
|
command: 'replay',
|
|
exitCode: result.exitCode,
|
|
message: result.message,
|
|
warnings: result.warnings,
|
|
}) + '\n')
|
|
} else {
|
|
console.log(result.message)
|
|
}
|
|
}
|
|
|
|
// Print warnings in human mode only
|
|
const format = options.format || ctx.options.format || 'human'
|
|
if (format === 'human' && result.warnings && result.warnings.length > 0 && !ctx.options.quiet) {
|
|
for (const warning of result.warnings) {
|
|
console.warn(`Warning: ${warning}`)
|
|
}
|
|
}
|
|
|
|
return result.exitCode
|
|
}
|