/** * S4: Verify thread - Runner for deterministic contract verification * * Responsibilities: * - Route discovery from Fastify app * - Route filtering by patterns and git changes * - Contract execution using existing plugin/evaluator code * - Deterministic execution with seed * - Result aggregation * * Architecture: * - Pure execution functions that accept injected dependencies * - Reuses existing APOPHIS plugin and formula code * - No reimplementation of parser/evaluator */ import { discoverRoutes } from '../../../domain/discovery.js' import { extractContract } from '../../../domain/contract.js' import { executeHttp } from '../../../infrastructure/http-executor.js' import { parse } from '../../../formula/parser.js' import { evaluateAsync } from '../../../formula/evaluator.js' import { createOperationResolver } from '../../../formula/runtime.js' import { buildRequest } from '../../../domain/request-builder.js' import { convertSchema } from '../../../domain/schema-to-arbitrary.js' import * as fc from 'fast-check' import { resolveRuns } from '../../../types.js' import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js' import type { RouteResult } from '../../core/types.js' import type { PluginContractRegistry } from '../../../domain/plugin-contracts.js' // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- export interface VerifyFailure { route: string contract: string expected: string observed: string artifactPath?: string formula?: string category?: string /** Source of the contract: 'route' or 'plugin:name' */ source?: string } export interface VerifyRunResult { passed: boolean total: number passedCount: number failed: number skipped: number failures: VerifyFailure[] durationMs: number noRoutesMatched: boolean noContractsFound: boolean notGitRepo?: boolean availableRoutes?: string[] artifactPaths: string[] discoveryWarnings?: string[] runs: number } export interface VerifyRunnerDeps { fastify: FastifyInjectInstance seed: number timeout?: number routeFilters?: string[] changed?: boolean profileRoutes?: string[] runs?: number pluginContractRegistry?: PluginContractRegistry } // --------------------------------------------------------------------------- // Route discovery // --------------------------------------------------------------------------- /** * Discover routes from a Fastify instance. * Uses the existing discovery module. */ export async function discoverAppRoutes(fastify: FastifyInjectInstance): Promise { return discoverRoutes(fastify) } /** * Check if specific routes exist in a Fastify instance using hasRoute. * Used when the APOPHIS plugin wasn't registered before routes. */ export async function discoverSpecificRoutes( fastify: FastifyInjectInstance, routePatterns: string[], ): Promise { if (typeof fastify.hasRoute !== 'function') { return [] } const routes: RouteContract[] = [] const seen = new Set() for (const pattern of routePatterns) { // Parse pattern like "GET /users" or "POST /api/*" const parts = pattern.split(' ') const method = parts[0] || 'GET' const path = parts.slice(1).join(' ') // For exact routes (no wildcards), check if route exists if (!pattern.includes('*') && !pattern.includes('?')) { if (fastify.hasRoute({ url: path, method })) { const key = `${method} ${path}` if (!seen.has(key)) { seen.add(key) routes.push({ method: method as RouteContract['method'], path, category: 'observer', schema: {}, requires: [], ensures: [], invariants: [], regexPatterns: {}, validateRuntime: false, }) } } } } return routes } // --------------------------------------------------------------------------- // Route filtering // --------------------------------------------------------------------------- /** * Check if a route matches a filter pattern. * Supports wildcards: * matches any characters. */ function matchRoutePattern(route: string, pattern: string): boolean { // Convert pattern to regex const regexPattern = pattern .replace(/\*/g, '.*') .replace(/\?/g, '.') const regex = new RegExp(`^${regexPattern}$`, 'i') return regex.test(route) } /** * Filter routes by patterns. */ function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): RouteContract[] { return routes.filter(route => { const routeStr = `${route.method} ${route.path}` return patterns.some(pattern => matchRoutePattern(routeStr, pattern)) }) } /** * Check if cwd is inside a git repository. */ export async function isGitRepo(cwd: string): Promise { try { const { execSync } = await import('node:child_process') execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' }) return true } catch { return false } } /** * Get git-modified files for --changed filtering. */ export async function getGitChangedFiles(cwd: string): Promise { try { const { execSync } = await import('node:child_process') const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' }) return output.split('\n').filter(Boolean) } catch { return [] } } /** * Filter routes to only those modified in git. */ export async function filterChangedRoutes( routes: RouteContract[], cwd: string, ): Promise { const changedFiles = await getGitChangedFiles(cwd) // Map route paths to potential file paths (heuristic) return routes.filter(route => { const routePath = route.path // Check if any changed file might contain this route return changedFiles.some(file => { // Simple heuristic: check if route path segments appear in file path const segments = routePath.split('/').filter(Boolean) return segments.some(segment => file.includes(segment)) }) }) } // --------------------------------------------------------------------------- // Contract execution // --------------------------------------------------------------------------- /** * Build a request for a route. */ function buildRouteRequest( route: RouteContract, seed?: number, runIndex?: number, variant?: { name: string; headers?: Record }, ): { method: string url: string body?: unknown query?: Record headers: Record } { let generatedData: Record = {} const bodySchema = route.schema?.body as Record | undefined if (bodySchema && seed !== undefined) { try { const bodyArb = convertSchema(bodySchema, { context: 'request' }) const bodySeed = seed + (runIndex ?? 0) * 31 const samples = fc.sample(bodyArb, { numRuns: 1, seed: bodySeed }) const bodySample = samples[0] if (bodySample !== null && typeof bodySample === 'object') { generatedData = bodySample as Record } } catch { // fall through to example-based generation } } if (!generatedData || Object.keys(generatedData).length === 0) { generatedData = buildExampleBody(bodySchema ?? {}) as Record ?? {} } const request = buildRequest( route, generatedData, variant?.headers ?? ({} as Record), { resources: new Map(), counters: new Map() }, ) const headers: Record = { 'content-type': 'application/json', ...request.headers, ...(variant?.headers ?? {}), } return { method: request.method, url: request.url, body: request.body, headers, query: request.query, } } /** * Build an example body from JSON Schema. */ function buildExampleBody(schema: Record): unknown { if (schema.type === 'object' && schema.properties) { const obj: Record = {} const properties = schema.properties as Record> for (const [key, propSchema] of Object.entries(properties)) { obj[key] = buildExampleValue(propSchema) } return obj } return undefined } /** * Build an example value from a property schema. */ function buildExampleValue(schema: Record): unknown { if (schema.type === 'string') { if (schema.enum && Array.isArray(schema.enum) && schema.enum.length > 0) { return schema.enum[0] } return 'test' } if (schema.type === 'number' || schema.type === 'integer') { return 1 } if (schema.type === 'boolean') { return true } if (schema.type === 'array') { return [] } if (schema.type === 'object' && schema.properties) { return buildExampleBody(schema) } return undefined } function buildFailureDiagnostic( contract: string, evalCtx: EvalContext, evalError?: string, ): Pick { if (evalError) { return { expected: 'true', observed: evalError, formula: contract, category: 'runtime' } } const status = evalCtx.response.statusCode const statusMatch = contract.match(/^status:(\d+)$/i) if (statusMatch) { return { expected: `HTTP ${statusMatch[1]}`, observed: `HTTP ${status}`, formula: contract, category: 'runtime' } } const fieldMatch = contract.match(/response_body\(this\)\.([\w.]+)\s*(!=|==)\s*null/i) if (fieldMatch) { const path = fieldMatch[1]! const negated = fieldMatch[2] === '!=' const parts = path.split('.') let value: unknown = evalCtx.response.body for (const p of parts) { if (value != null && typeof value === 'object') { value = (value as Record)[p] } else { value = undefined; break } } const actual = value === undefined ? 'undefined' : value === null ? 'null' : `"${String(value)}"` return { expected: negated ? `${path} != null` : `${path} == null`, observed: `${path} is ${actual}`, formula: contract, category: 'runtime', } } const codeMatch = contract.match(/response_code\(this\)\s*(==|!=|<|>|<=|>=)\s*(\d+)/i) if (codeMatch) { return { expected: `response_code == ${codeMatch[2]}`, observed: `response_code is ${status}`, formula: contract, category: 'runtime', } } return { expected: contract, observed: `false`, formula: contract, category: 'runtime' } } async function executeContract( fastify: FastifyInjectInstance, route: RouteContract, contract: string, timeout?: number, variant?: { name: string; headers?: Record }, seed?: number, runIndex?: number, ): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> { const request = buildRouteRequest(route, seed, runIndex, variant) // Merge variant headers if provided const headers = variant?.headers ? { ...request.headers, ...variant.headers } : request.headers // Execute the primary request const ctx = await executeHttp(fastify, route, { method: request.method, url: request.url, body: request.body, headers, query: {}, }, undefined, timeout) // Build eval context with operation resolver for cross-operation calls const evalCtx: EvalContext = { ...ctx, operationResolver: createOperationResolver(fastify, headers, ctx), } // Parse and evaluate the contract try { const parsed = parse(contract) const result = await evaluateAsync(parsed.ast, evalCtx) if (!result.success || !result.value) { const diagnostic = buildFailureDiagnostic(contract, evalCtx, result.success ? undefined : result.error) return { ctx: evalCtx, failure: { route: variant && variant.name !== 'default' ? `[variant:${variant.name}] ${route.method} ${route.path}` : `${route.method} ${route.path}`, contract, expected: diagnostic.expected, observed: diagnostic.observed, formula: diagnostic.formula, category: diagnostic.category, }, } } return { ctx: evalCtx } } catch (error) { const diagnostic = buildFailureDiagnostic(contract, evalCtx, error instanceof Error ? error.message : String(error)) return { ctx: evalCtx, failure: { route: variant && variant.name !== 'default' ? `[variant:${variant.name}] ${route.method} ${route.path}` : `${route.method} ${route.path}`, contract, expected: diagnostic.expected, observed: diagnostic.observed, formula: diagnostic.formula, category: diagnostic.category, }, } } } // --------------------------------------------------------------------------- // Main verify runner // --------------------------------------------------------------------------- /** * Run deterministic contract verification. * * Flow: * 1. Discover routes from Fastify app * 2. Apply route filters (patterns, changed, profile routes) * 3. Check for behavioral contracts * 4. Execute each contract deterministically * 5. Aggregate results */ export async function runVerify(deps: VerifyRunnerDeps): Promise { const started = Date.now() const { fastify, routeFilters, changed, profileRoutes } = deps // 1. Discover routes let allRoutes = await discoverAppRoutes(fastify) // If no routes discovered (plugin not registered before routes), // try to discover specific routes from filters if (allRoutes.length === 0 && (routeFilters?.length || profileRoutes?.length)) { const patternsToCheck = [ ...(routeFilters || []), ...(profileRoutes || []), ] allRoutes = await discoverSpecificRoutes(fastify, patternsToCheck) } const availableRoutes = allRoutes.map(r => `${r.method} ${r.path}`) // 2. Apply filters let routes = allRoutes // Apply profile routes filter first if (profileRoutes && profileRoutes.length > 0) { routes = filterRoutesByPatterns(routes, profileRoutes) } // Apply --routes flag filter if (routeFilters && routeFilters.length > 0) { routes = filterRoutesByPatterns(routes, routeFilters) } // Apply --changed filter if (changed) { const cwd = process.cwd() const inGit = await isGitRepo(cwd) if (!inGit) { return { passed: false, total: 0, passedCount: 0, failed: 0, skipped: 0, failures: [], durationMs: Date.now() - started, noRoutesMatched: false, noContractsFound: false, availableRoutes, artifactPaths: [], notGitRepo: true, runs: 0, } } routes = await filterChangedRoutes(routes, cwd) } // Check if any routes matched if (routes.length === 0) { return { passed: false, total: 0, passedCount: 0, failed: 0, skipped: 0, failures: [], durationMs: Date.now() - started, noRoutesMatched: true, noContractsFound: false, availableRoutes, artifactPaths: [], runs: 0, } } // 3a. Compose plugin contracts into route contracts if (deps.pluginContractRegistry) { for (const route of routes) { const composed = deps.pluginContractRegistry.composeContracts(route) if (!route.formulaSources) route.formulaSources = {} for (const phase of Object.values(composed.phases)) { for (const req of phase.requires) { if (!route.requires.includes(req.formula)) { route.requires.push(req.formula) route.formulaSources[req.formula] = req.source } } for (const ens of phase.ensures) { if (!route.ensures.includes(ens.formula)) { route.ensures.push(ens.formula) route.formulaSources[ens.formula] = ens.source } } } } } // 3. Check for behavioral contracts const routesWithContracts = routes.filter(route => route.ensures.length > 0 || route.requires.length > 0 ) if (routesWithContracts.length === 0) { return { passed: false, total: 0, passedCount: 0, failed: 0, skipped: 0, failures: [], durationMs: Date.now() - started, noRoutesMatched: false, noContractsFound: true, availableRoutes, artifactPaths: [], runs: 0, } } // 4. Execute contracts (with variant expansion) // Execute one HTTP request per (route, variant, runIndex), then validate // all contracts against the same request context. // Requires (preconditions) that fail → SKIP the ensures and count as skipped. // Ensures (postconditions) that fail → count as failures. const runConfig = resolveRuns(deps.runs) const runs = runConfig.contractRuns const failures: VerifyFailure[] = [] let total = 0 let passedCount = 0 let skippedCount = 0 // Extract required headers from precondition formulas and auto-inject them // so that plugin contracts (e.g., x-tenant-id != null) are satisfied. function extractRequiredHeaders(formulas: string[]): Record { const headers: Record = {} for (const f of formulas) { const nullCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*!=\s*null/i) if (nullCheck) { headers[nullCheck[1]!] = 'test-value'; continue } const eqCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*==\s*"([^"]+)"/i) if (eqCheck) { headers[eqCheck[1]!] = eqCheck[2]!; continue } } return headers } for (const route of routesWithContracts) { const defaultHeaders = extractRequiredHeaders(route.requires) const variants = route.variants && route.variants.length > 0 ? route.variants : [{ name: 'default' }] for (const variant of variants) { for (let runIndex = 0; runIndex < runs; runIndex++) { // Execute one HTTP request per run const request = buildRouteRequest(route, deps.seed, runIndex, variant) const headers = { ...defaultHeaders, ...request.headers, ...(variant.headers ?? {}), } const ctx = await executeHttp(fastify, route, { method: request.method, url: request.url, body: request.body, headers, query: {}, }, undefined, deps.timeout) const evalCtx: EvalContext = { ...ctx, operationResolver: createOperationResolver(fastify, headers, ctx), } // Evaluate requires first. If any fail, skip ensures for this run. let requiresPassed = true for (const formula of route.requires) { total++ try { const parsed = parse(formula) const result = await evaluateAsync(parsed.ast, evalCtx) if (!result.success || !result.value) { requiresPassed = false skippedCount++ } else { passedCount++ } } catch { requiresPassed = false skippedCount++ } } // Only evaluate ensures if all requires passed if (requiresPassed) { for (const formula of route.ensures) { total++ try { const parsed = parse(formula) const result = await evaluateAsync(parsed.ast, evalCtx) if (!result.success || !result.value) { const diagnostic = buildFailureDiagnostic(formula, evalCtx, result.success ? undefined : result.error) failures.push({ route: variant.name && variant.name !== 'default' ? `[variant:${variant.name}] ${route.method} ${route.path}` : `${route.method} ${route.path}`, contract: formula, expected: diagnostic.expected, observed: diagnostic.observed, formula: diagnostic.formula, category: diagnostic.category, source: route.formulaSources?.[formula], }) } else { passedCount++ } } catch (error) { const diagnostic = buildFailureDiagnostic(formula, evalCtx, error instanceof Error ? error.message : String(error)) failures.push({ route: variant.name && variant.name !== 'default' ? `[variant:${variant.name}] ${route.method} ${route.path}` : `${route.method} ${route.path}`, contract: formula, expected: diagnostic.expected, observed: diagnostic.observed, formula: diagnostic.formula, category: diagnostic.category, source: route.formulaSources?.[formula], }) } } } else { // Preconditions failed: ensures are skipped but not evaluated. // We still count each ensure as "total" and "skipped" for transparency. total += route.ensures.length skippedCount += route.ensures.length } } } } const durationMs = Date.now() - started // Sort failures deterministically by route then contract for stable output const sortedFailures = failures.sort((a, b) => { const routeCmp = a.route.localeCompare(b.route) if (routeCmp !== 0) return routeCmp return a.contract.localeCompare(b.contract) }) return { passed: failures.length === 0, total, passedCount, failed: failures.length, skipped: skippedCount, failures: sortedFailures, durationMs, noRoutesMatched: false, noContractsFound: false, availableRoutes, artifactPaths: [], runs, } }