John Dvorak
709 lines
22 KiB
TypeScript
709 lines
22 KiB
TypeScript
/**
|
|
* S4: Verify thread - Runner for deterministic contract verification
|
|
*
|
|
* Responsibilities:
|
|
* - Route discovery from Fastify app
|
|
* - Route filtering by patterns and git changes
|
|
* - Contract execution using existing plugin/evaluator code
|
|
* - Deterministic execution with seed
|
|
* - Result aggregation
|
|
*
|
|
* Architecture:
|
|
* - Pure execution functions that accept injected dependencies
|
|
* - Reuses existing APOPHIS plugin and formula code
|
|
* - No reimplementation of parser/evaluator
|
|
*/
|
|
|
|
import { discoverRoutes } from '../../../domain/discovery.js'
|
|
import { extractContract } from '../../../domain/contract.js'
|
|
import { executeHttp } from '../../../infrastructure/http-executor.js'
|
|
import { parse } from '../../../formula/parser.js'
|
|
import { evaluateAsync } from '../../../formula/evaluator.js'
|
|
import { createOperationResolver } from '../../../formula/runtime.js'
|
|
import { buildRequest } from '../../../domain/request-builder.js'
|
|
import { convertSchema } from '../../../domain/schema-to-arbitrary.js'
|
|
import * as fc from 'fast-check'
|
|
import { resolveRuns } from '../../../types.js'
|
|
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
|
|
import type { RouteResult } from '../../core/types.js'
|
|
import type { PluginContractRegistry } from '../../../domain/plugin-contracts.js'
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface VerifyFailure {
|
|
route: string
|
|
contract: string
|
|
expected: string
|
|
observed: string
|
|
artifactPath?: string
|
|
formula?: string
|
|
category?: string
|
|
/** Source of the contract: 'route' or 'plugin:name' */
|
|
source?: string
|
|
}
|
|
|
|
export interface VerifyRunResult {
|
|
passed: boolean
|
|
total: number
|
|
passedCount: number
|
|
failed: number
|
|
skipped: number
|
|
failures: VerifyFailure[]
|
|
durationMs: number
|
|
noRoutesMatched: boolean
|
|
noContractsFound: boolean
|
|
notGitRepo?: boolean
|
|
availableRoutes?: string[]
|
|
artifactPaths: string[]
|
|
discoveryWarnings?: string[]
|
|
runs: number
|
|
}
|
|
|
|
export interface VerifyRunnerDeps {
|
|
fastify: FastifyInjectInstance
|
|
seed: number
|
|
timeout?: number
|
|
routeFilters?: string[]
|
|
changed?: boolean
|
|
profileRoutes?: string[]
|
|
runs?: number
|
|
pluginContractRegistry?: PluginContractRegistry
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Route discovery
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Discover routes from a Fastify instance.
|
|
* Uses the existing discovery module.
|
|
*/
|
|
export async function discoverAppRoutes(fastify: FastifyInjectInstance): Promise<RouteContract[]> {
|
|
return discoverRoutes(fastify)
|
|
}
|
|
|
|
/**
|
|
* Check if specific routes exist in a Fastify instance using hasRoute.
|
|
* Used when the APOPHIS plugin wasn't registered before routes.
|
|
*/
|
|
export async function discoverSpecificRoutes(
|
|
fastify: FastifyInjectInstance,
|
|
routePatterns: string[],
|
|
): Promise<RouteContract[]> {
|
|
if (typeof fastify.hasRoute !== 'function') {
|
|
return []
|
|
}
|
|
|
|
const routes: RouteContract[] = []
|
|
const seen = new Set<string>()
|
|
|
|
for (const pattern of routePatterns) {
|
|
// Parse pattern like "GET /users" or "POST /api/*"
|
|
const parts = pattern.split(' ')
|
|
const method = parts[0] || 'GET'
|
|
const path = parts.slice(1).join(' ')
|
|
|
|
// For exact routes (no wildcards), check if route exists
|
|
if (!pattern.includes('*') && !pattern.includes('?')) {
|
|
if (fastify.hasRoute({ url: path, method })) {
|
|
const key = `${method} ${path}`
|
|
if (!seen.has(key)) {
|
|
seen.add(key)
|
|
routes.push({
|
|
method: method as RouteContract['method'],
|
|
path,
|
|
category: 'observer',
|
|
schema: {},
|
|
requires: [],
|
|
ensures: [],
|
|
invariants: [],
|
|
regexPatterns: {},
|
|
validateRuntime: false,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return routes
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Route filtering
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Check if a route matches a filter pattern.
|
|
* Supports wildcards: * matches any characters.
|
|
*/
|
|
function matchRoutePattern(route: string, pattern: string): boolean {
|
|
// Convert pattern to regex
|
|
const regexPattern = pattern
|
|
.replace(/\*/g, '.*')
|
|
.replace(/\?/g, '.')
|
|
|
|
const regex = new RegExp(`^${regexPattern}$`, 'i')
|
|
return regex.test(route)
|
|
}
|
|
|
|
/**
|
|
* Filter routes by patterns.
|
|
*/
|
|
function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): RouteContract[] {
|
|
return routes.filter(route => {
|
|
const routeStr = `${route.method} ${route.path}`
|
|
return patterns.some(pattern => matchRoutePattern(routeStr, pattern))
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Check if cwd is inside a git repository.
|
|
*/
|
|
export async function isGitRepo(cwd: string): Promise<boolean> {
|
|
try {
|
|
const { execSync } = await import('node:child_process')
|
|
execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' })
|
|
return true
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get git-modified files for --changed filtering.
|
|
*/
|
|
export async function getGitChangedFiles(cwd: string): Promise<string[]> {
|
|
try {
|
|
const { execSync } = await import('node:child_process')
|
|
const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' })
|
|
return output.split('\n').filter(Boolean)
|
|
} catch {
|
|
return []
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Filter routes to only those whose source file was modified in git.
|
|
* Uses sourceFile captured from stack traces during route registration.
|
|
* Falls back to path-segment heuristic when sourceFile is unavailable.
|
|
*/
|
|
export async function filterChangedRoutes(
|
|
routes: RouteContract[],
|
|
cwd: string,
|
|
): Promise<RouteContract[]> {
|
|
const changedFiles = await getGitChangedFiles(cwd)
|
|
|
|
return routes.filter(route => {
|
|
// Prefer precise source file match
|
|
if (route.sourceFile) {
|
|
return changedFiles.some(file =>
|
|
route.sourceFile!.endsWith(file) || file.endsWith(route.sourceFile!)
|
|
)
|
|
}
|
|
// Fallback: heuristic path-segment match
|
|
const routePath = route.path
|
|
const segments = routePath.split('/').filter(Boolean)
|
|
return changedFiles.some(file =>
|
|
segments.some(segment => file.includes(segment))
|
|
)
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Contract execution
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build a request for a route.
|
|
*/
|
|
function buildRouteRequest(
|
|
route: RouteContract,
|
|
seed?: number,
|
|
runIndex?: number,
|
|
variant?: { name: string; headers?: Record<string, string> },
|
|
): {
|
|
method: string
|
|
url: string
|
|
body?: unknown
|
|
query?: Record<string, string>
|
|
headers: Record<string, string>
|
|
} {
|
|
let generatedData: Record<string, unknown> = {}
|
|
|
|
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
|
if (bodySchema && seed !== undefined) {
|
|
try {
|
|
const bodyArb = convertSchema(bodySchema, { context: 'request' })
|
|
const bodySeed = seed + (runIndex ?? 0) * 31
|
|
const samples = fc.sample(bodyArb, { numRuns: 1, seed: bodySeed })
|
|
const bodySample = samples[0]
|
|
if (bodySample !== null && typeof bodySample === 'object') {
|
|
generatedData = bodySample as Record<string, unknown>
|
|
}
|
|
} catch {
|
|
// fall through to example-based generation
|
|
}
|
|
}
|
|
|
|
if (!generatedData || Object.keys(generatedData).length === 0) {
|
|
generatedData = buildExampleBody(bodySchema ?? {}) as Record<string, unknown> ?? {}
|
|
}
|
|
|
|
const request = buildRequest(
|
|
route,
|
|
generatedData,
|
|
variant?.headers ?? ({} as Record<string, string>),
|
|
{ resources: new Map(), counters: new Map() },
|
|
)
|
|
|
|
const headers: Record<string, string> = {
|
|
'content-type': 'application/json',
|
|
...request.headers,
|
|
...(variant?.headers ?? {}),
|
|
}
|
|
|
|
return {
|
|
method: request.method,
|
|
url: request.url,
|
|
body: request.body,
|
|
headers,
|
|
query: request.query,
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Build an example body from JSON Schema.
|
|
*/
|
|
function buildExampleBody(schema: Record<string, unknown>): unknown {
|
|
if (schema.type === 'object' && schema.properties) {
|
|
const obj: Record<string, unknown> = {}
|
|
const properties = schema.properties as Record<string, Record<string, unknown>>
|
|
for (const [key, propSchema] of Object.entries(properties)) {
|
|
obj[key] = buildExampleValue(propSchema)
|
|
}
|
|
return obj
|
|
}
|
|
return undefined
|
|
}
|
|
|
|
/**
|
|
* Build an example value from a property schema.
|
|
*/
|
|
function buildExampleValue(schema: Record<string, unknown>): unknown {
|
|
if (schema.type === 'string') {
|
|
if (schema.enum && Array.isArray(schema.enum) && schema.enum.length > 0) {
|
|
return schema.enum[0]
|
|
}
|
|
return 'test'
|
|
}
|
|
if (schema.type === 'number' || schema.type === 'integer') {
|
|
return 1
|
|
}
|
|
if (schema.type === 'boolean') {
|
|
return true
|
|
}
|
|
if (schema.type === 'array') {
|
|
return []
|
|
}
|
|
if (schema.type === 'object' && schema.properties) {
|
|
return buildExampleBody(schema)
|
|
}
|
|
return undefined
|
|
}
|
|
|
|
function buildFailureDiagnostic(
|
|
contract: string,
|
|
evalCtx: EvalContext,
|
|
evalError?: string,
|
|
): Pick<VerifyFailure, 'expected' | 'observed' | 'formula' | 'category'> {
|
|
if (evalError) {
|
|
return { expected: 'true', observed: evalError, formula: contract, category: 'runtime' }
|
|
}
|
|
const status = evalCtx.response.statusCode
|
|
|
|
const statusMatch = contract.match(/^status:(\d+)$/i)
|
|
if (statusMatch) {
|
|
return { expected: `HTTP ${statusMatch[1]}`, observed: `HTTP ${status}`, formula: contract, category: 'runtime' }
|
|
}
|
|
|
|
const fieldMatch = contract.match(/response_body\(this\)\.([\w.]+)\s*(!=|==)\s*null/i)
|
|
if (fieldMatch) {
|
|
const path = fieldMatch[1]!
|
|
const negated = fieldMatch[2] === '!='
|
|
const parts = path.split('.')
|
|
let value: unknown = evalCtx.response.body
|
|
for (const p of parts) {
|
|
if (value != null && typeof value === 'object') {
|
|
value = (value as Record<string, unknown>)[p]
|
|
} else { value = undefined; break }
|
|
}
|
|
const actual = value === undefined ? 'undefined' : value === null ? 'null' : `"${String(value)}"`
|
|
return {
|
|
expected: negated ? `${path} != null` : `${path} == null`,
|
|
observed: `${path} is ${actual}`,
|
|
formula: contract, category: 'runtime',
|
|
}
|
|
}
|
|
|
|
const codeMatch = contract.match(/response_code\(this\)\s*(==|!=|<|>|<=|>=)\s*(\d+)/i)
|
|
if (codeMatch) {
|
|
return {
|
|
expected: `response_code == ${codeMatch[2]}`,
|
|
observed: `response_code is ${status}`,
|
|
formula: contract, category: 'runtime',
|
|
}
|
|
}
|
|
|
|
return { expected: contract, observed: `false`, formula: contract, category: 'runtime' }
|
|
}
|
|
|
|
async function executeContract(
|
|
fastify: FastifyInjectInstance,
|
|
route: RouteContract,
|
|
contract: string,
|
|
timeout?: number,
|
|
variant?: { name: string; headers?: Record<string, string> },
|
|
seed?: number,
|
|
runIndex?: number,
|
|
): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> {
|
|
const request = buildRouteRequest(route, seed, runIndex, variant)
|
|
|
|
// Merge variant headers if provided
|
|
const headers = variant?.headers
|
|
? { ...request.headers, ...variant.headers }
|
|
: request.headers
|
|
|
|
// Execute the primary request
|
|
const ctx = await executeHttp(fastify, route, {
|
|
method: request.method,
|
|
url: request.url,
|
|
body: request.body,
|
|
headers,
|
|
query: {},
|
|
}, undefined, timeout)
|
|
|
|
// Build eval context with operation resolver for cross-operation calls
|
|
const evalCtx: EvalContext = {
|
|
...ctx,
|
|
operationResolver: createOperationResolver(fastify, headers, ctx),
|
|
}
|
|
|
|
// Parse and evaluate the contract
|
|
try {
|
|
const parsed = parse(contract)
|
|
const result = await evaluateAsync(parsed.ast, evalCtx)
|
|
|
|
if (!result.success || !result.value) {
|
|
const diagnostic = buildFailureDiagnostic(contract, evalCtx, result.success ? undefined : result.error)
|
|
return {
|
|
ctx: evalCtx,
|
|
failure: {
|
|
route: variant && variant.name !== 'default'
|
|
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
|
: `${route.method} ${route.path}`,
|
|
contract,
|
|
expected: diagnostic.expected,
|
|
observed: diagnostic.observed,
|
|
formula: diagnostic.formula,
|
|
category: diagnostic.category,
|
|
},
|
|
}
|
|
}
|
|
|
|
return { ctx: evalCtx }
|
|
} catch (error) {
|
|
const diagnostic = buildFailureDiagnostic(contract, evalCtx, error instanceof Error ? error.message : String(error))
|
|
return {
|
|
ctx: evalCtx,
|
|
failure: {
|
|
route: variant && variant.name !== 'default'
|
|
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
|
: `${route.method} ${route.path}`,
|
|
contract,
|
|
expected: diagnostic.expected,
|
|
observed: diagnostic.observed,
|
|
formula: diagnostic.formula,
|
|
category: diagnostic.category,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main verify runner
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Run deterministic contract verification.
|
|
*
|
|
* Flow:
|
|
* 1. Discover routes from Fastify app
|
|
* 2. Apply route filters (patterns, changed, profile routes)
|
|
* 3. Check for behavioral contracts
|
|
* 4. Execute each contract deterministically
|
|
* 5. Aggregate results
|
|
*/
|
|
export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult> {
|
|
const started = Date.now()
|
|
const { fastify, routeFilters, changed, profileRoutes } = deps
|
|
|
|
// 1. Discover routes
|
|
let allRoutes = await discoverAppRoutes(fastify)
|
|
|
|
// If no routes discovered (plugin not registered before routes),
|
|
// try to discover specific routes from filters
|
|
if (allRoutes.length === 0 && (routeFilters?.length || profileRoutes?.length)) {
|
|
const patternsToCheck = [
|
|
...(routeFilters || []),
|
|
...(profileRoutes || []),
|
|
]
|
|
allRoutes = await discoverSpecificRoutes(fastify, patternsToCheck)
|
|
}
|
|
|
|
const availableRoutes = allRoutes.map(r => `${r.method} ${r.path}`)
|
|
|
|
// 2. Apply filters
|
|
let routes = allRoutes
|
|
|
|
// Apply profile routes filter first
|
|
if (profileRoutes && profileRoutes.length > 0) {
|
|
routes = filterRoutesByPatterns(routes, profileRoutes)
|
|
}
|
|
|
|
// Apply --routes flag filter
|
|
if (routeFilters && routeFilters.length > 0) {
|
|
routes = filterRoutesByPatterns(routes, routeFilters)
|
|
}
|
|
|
|
// Apply --changed filter
|
|
if (changed) {
|
|
const cwd = process.cwd()
|
|
const inGit = await isGitRepo(cwd)
|
|
if (!inGit) {
|
|
return {
|
|
passed: false,
|
|
total: 0,
|
|
passedCount: 0,
|
|
failed: 0,
|
|
skipped: 0,
|
|
failures: [],
|
|
durationMs: Date.now() - started,
|
|
noRoutesMatched: false,
|
|
noContractsFound: false,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
notGitRepo: true,
|
|
runs: 0,
|
|
}
|
|
}
|
|
routes = await filterChangedRoutes(routes, cwd)
|
|
}
|
|
|
|
// Check if any routes matched
|
|
if (routes.length === 0) {
|
|
return {
|
|
passed: false,
|
|
total: 0,
|
|
passedCount: 0,
|
|
failed: 0,
|
|
skipped: 0,
|
|
failures: [],
|
|
durationMs: Date.now() - started,
|
|
noRoutesMatched: true,
|
|
noContractsFound: false,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
runs: 0,
|
|
}
|
|
}
|
|
|
|
// 3a. Compose plugin contracts into route contracts
|
|
if (deps.pluginContractRegistry) {
|
|
for (const route of routes) {
|
|
const composed = deps.pluginContractRegistry.composeContracts(route)
|
|
if (!route.formulaSources) route.formulaSources = {}
|
|
for (const phase of Object.values(composed.phases)) {
|
|
for (const req of phase.requires) {
|
|
if (!route.requires.includes(req.formula)) {
|
|
route.requires.push(req.formula)
|
|
route.formulaSources[req.formula] = req.source
|
|
}
|
|
}
|
|
for (const ens of phase.ensures) {
|
|
if (!route.ensures.includes(ens.formula)) {
|
|
route.ensures.push(ens.formula)
|
|
route.formulaSources[ens.formula] = ens.source
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Check for behavioral contracts
|
|
const routesWithContracts = routes.filter(route =>
|
|
route.ensures.length > 0 || route.requires.length > 0
|
|
)
|
|
|
|
if (routesWithContracts.length === 0) {
|
|
return {
|
|
passed: false,
|
|
total: 0,
|
|
passedCount: 0,
|
|
failed: 0,
|
|
skipped: 0,
|
|
failures: [],
|
|
durationMs: Date.now() - started,
|
|
noRoutesMatched: false,
|
|
noContractsFound: true,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
runs: 0,
|
|
}
|
|
}
|
|
|
|
// 4. Execute contracts (with variant expansion)
|
|
// Execute one HTTP request per (route, variant, runIndex), then validate
|
|
// all contracts against the same request context.
|
|
// Requires (preconditions) that fail → SKIP the ensures and count as skipped.
|
|
// Ensures (postconditions) that fail → count as failures.
|
|
const runConfig = resolveRuns(deps.runs)
|
|
const runs = runConfig.contractRuns
|
|
const failures: VerifyFailure[] = []
|
|
let total = 0
|
|
let passedCount = 0
|
|
let skippedCount = 0
|
|
|
|
// Extract required headers from precondition formulas and auto-inject them
|
|
// so that plugin contracts (e.g., x-tenant-id != null) are satisfied.
|
|
function extractRequiredHeaders(formulas: string[]): Record<string, string> {
|
|
const headers: Record<string, string> = {}
|
|
for (const f of formulas) {
|
|
const nullCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*!=\s*null/i)
|
|
if (nullCheck) { headers[nullCheck[1]!] = 'test-value'; continue }
|
|
const eqCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*==\s*"([^"]+)"/i)
|
|
if (eqCheck) { headers[eqCheck[1]!] = eqCheck[2]!; continue }
|
|
}
|
|
return headers
|
|
}
|
|
|
|
for (const route of routesWithContracts) {
|
|
const defaultHeaders = extractRequiredHeaders(route.requires)
|
|
const variants = route.variants && route.variants.length > 0
|
|
? route.variants
|
|
: [{ name: 'default' }]
|
|
|
|
for (const variant of variants) {
|
|
for (let runIndex = 0; runIndex < runs; runIndex++) {
|
|
// Execute one HTTP request per run
|
|
const request = buildRouteRequest(route, deps.seed, runIndex, variant)
|
|
const headers = {
|
|
...defaultHeaders,
|
|
...request.headers,
|
|
...(variant.headers ?? {}),
|
|
}
|
|
const ctx = await executeHttp(fastify, route, {
|
|
method: request.method,
|
|
url: request.url,
|
|
body: request.body,
|
|
headers,
|
|
query: {},
|
|
}, undefined, deps.timeout)
|
|
const evalCtx: EvalContext = {
|
|
...ctx,
|
|
operationResolver: createOperationResolver(fastify, headers, ctx),
|
|
}
|
|
|
|
// Evaluate requires first. If any fail, skip ensures for this run.
|
|
let requiresPassed = true
|
|
for (const formula of route.requires) {
|
|
total++
|
|
try {
|
|
const parsed = parse(formula)
|
|
const result = await evaluateAsync(parsed.ast, evalCtx)
|
|
if (!result.success || !result.value) {
|
|
requiresPassed = false
|
|
skippedCount++
|
|
} else {
|
|
passedCount++
|
|
}
|
|
} catch {
|
|
requiresPassed = false
|
|
skippedCount++
|
|
}
|
|
}
|
|
|
|
// Only evaluate ensures if all requires passed
|
|
if (requiresPassed) {
|
|
for (const formula of route.ensures) {
|
|
total++
|
|
try {
|
|
const parsed = parse(formula)
|
|
const result = await evaluateAsync(parsed.ast, evalCtx)
|
|
if (!result.success || !result.value) {
|
|
const diagnostic = buildFailureDiagnostic(formula, evalCtx, result.success ? undefined : result.error)
|
|
failures.push({
|
|
route: variant.name && variant.name !== 'default'
|
|
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
|
: `${route.method} ${route.path}`,
|
|
contract: formula,
|
|
expected: diagnostic.expected,
|
|
observed: diagnostic.observed,
|
|
formula: diagnostic.formula,
|
|
category: diagnostic.category,
|
|
source: route.formulaSources?.[formula],
|
|
})
|
|
} else {
|
|
passedCount++
|
|
}
|
|
} catch (error) {
|
|
const diagnostic = buildFailureDiagnostic(formula, evalCtx, error instanceof Error ? error.message : String(error))
|
|
failures.push({
|
|
route: variant.name && variant.name !== 'default'
|
|
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
|
: `${route.method} ${route.path}`,
|
|
contract: formula,
|
|
expected: diagnostic.expected,
|
|
observed: diagnostic.observed,
|
|
formula: diagnostic.formula,
|
|
category: diagnostic.category,
|
|
source: route.formulaSources?.[formula],
|
|
})
|
|
}
|
|
}
|
|
} else {
|
|
// Preconditions failed: ensures are skipped but not evaluated.
|
|
// We still count each ensure as "total" and "skipped" for transparency.
|
|
total += route.ensures.length
|
|
skippedCount += route.ensures.length
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const durationMs = Date.now() - started
|
|
|
|
// Sort failures deterministically by route then contract for stable output
|
|
const sortedFailures = failures.sort((a, b) => {
|
|
const routeCmp = a.route.localeCompare(b.route)
|
|
if (routeCmp !== 0) return routeCmp
|
|
return a.contract.localeCompare(b.contract)
|
|
})
|
|
|
|
return {
|
|
passed: failures.length === 0,
|
|
total,
|
|
passedCount,
|
|
failed: failures.length,
|
|
skipped: skippedCount,
|
|
failures: sortedFailures,
|
|
durationMs,
|
|
noRoutesMatched: false,
|
|
noContractsFound: false,
|
|
availableRoutes,
|
|
artifactPaths: [],
|
|
runs,
|
|
}
|
|
}
|