apophis/apophis-fastify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: plugin contract e2e, qualify --changed, production observe, regressions

Browse Source
This commit is contained in:
John Dvorak
2026-05-22 11:05:52 -07:00
parent d0523fcc2d
commit 1de735ee08
34 changed files with 1392 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/commands/verify/runner.ts
+129 -15
View File
@@ -26,6 +26,7 @@ import * as fc from 'fast-check'
import { resolveRuns } from '../../../types.js'
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
import type { RouteResult } from '../../core/types.js'
import type { PluginContractRegistry } from '../../../domain/plugin-contracts.js'
// ---------------------------------------------------------------------------
// Types
@@ -46,6 +47,7 @@ export interface VerifyRunResult {
total: number
passedCount: number
failed: number
skipped: number
failures: VerifyFailure[]
durationMs: number
noRoutesMatched: boolean
@@ -66,6 +68,7 @@ export interface VerifyRunnerDeps {
changed?: boolean
profileRoutes?: string[]
runs?: number
pluginContractRegistry?: PluginContractRegistry
}
// ---------------------------------------------------------------------------
@@ -161,7 +164,7 @@ function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): Ro
/**
* Check if cwd is inside a git repository.
*/
async function isGitRepo(cwd: string): Promise<boolean> {
export async function isGitRepo(cwd: string): Promise<boolean> {
try {
const { execSync } = await import('node:child_process')
execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' })
@@ -174,7 +177,7 @@ async function isGitRepo(cwd: string): Promise<boolean> {
/**
* Get git-modified files for --changed filtering.
*/
async function getGitChangedFiles(cwd: string): Promise<string[]> {
export async function getGitChangedFiles(cwd: string): Promise<string[]> {
try {
const { execSync } = await import('node:child_process')
const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' })
@@ -187,7 +190,7 @@ async function getGitChangedFiles(cwd: string): Promise<string[]> {
/**
* Filter routes to only those modified in git.
*/
async function filterChangedRoutes(
export async function filterChangedRoutes(
routes: RouteContract[],
cwd: string,
): Promise<RouteContract[]> {
@@ -481,6 +484,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total: 0,
passedCount: 0,
failed: 0,
skipped: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
@@ -501,6 +505,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total: 0,
passedCount: 0,
failed: 0,
skipped: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: true,
@@ -511,6 +516,28 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
}
}
// 3a. Compose plugin contracts into route contracts
if (deps.pluginContractRegistry) {
for (const route of routes) {
const composed = deps.pluginContractRegistry.composeContracts(route)
if (!route.formulaSources) route.formulaSources = {}
for (const phase of Object.values(composed.phases)) {
for (const req of phase.requires) {
if (!route.requires.includes(req.formula)) {
route.requires.push(req.formula)
route.formulaSources[req.formula] = req.source
}
}
for (const ens of phase.ensures) {
if (!route.ensures.includes(ens.formula)) {
route.ensures.push(ens.formula)
route.formulaSources[ens.formula] = ens.source
}
}
}
}
}
// 3. Check for behavioral contracts
const routesWithContracts = routes.filter(route =>
route.ensures.length > 0 || route.requires.length > 0
@@ -522,6 +549,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total: 0,
passedCount: 0,
failed: 0,
skipped: 0,
failures: [],
durationMs: Date.now() - started,
noRoutesMatched: false,
@@ -533,33 +561,118 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
}
// 4. Execute contracts (with variant expansion)
// Execute one HTTP request per (route, variant, runIndex), then validate
// all contracts against the same request context.
// Requires (preconditions) that fail → SKIP the ensures and count as skipped.
// Ensures (postconditions) that fail → count as failures.
const runConfig = resolveRuns(deps.runs)
const runs = runConfig.contractRuns
const failures: VerifyFailure[] = []
let total = 0
let passedCount = 0
let skippedCount = 0
// Extract required headers from precondition formulas and auto-inject them
// so that plugin contracts (e.g., x-tenant-id != null) are satisfied.
function extractRequiredHeaders(formulas: string[]): Record<string, string> {
const headers: Record<string, string> = {}
for (const f of formulas) {
const nullCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*!=\s*null/i)
if (nullCheck) { headers[nullCheck[1]!] = 'test-value'; continue }
const eqCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*==\s*"([^"]+)"/i)
if (eqCheck) { headers[eqCheck[1]!] = eqCheck[2]!; continue }
}
return headers
}
for (const route of routesWithContracts) {
const contracts = [...route.requires, ...route.ensures]
const defaultHeaders = extractRequiredHeaders(route.requires)
const variants = route.variants && route.variants.length > 0
? route.variants
: [{ name: 'default' }]
for (const variant of variants) {
for (const contract of contracts) {
for (let runIndex = 0; runIndex < runs; runIndex++) {
total++
const result = await executeContract(
fastify, route, contract, deps.timeout,
variant, deps.seed, runIndex,
)
for (let runIndex = 0; runIndex < runs; runIndex++) {
// Execute one HTTP request per run
const request = buildRouteRequest(route, deps.seed, runIndex, variant)
const headers = {
...defaultHeaders,
...request.headers,
...(variant.headers ?? {}),
}
const ctx = await executeHttp(fastify, route, {
method: request.method,
url: request.url,
body: request.body,
headers,
query: {},
}, undefined, deps.timeout)
const evalCtx: EvalContext = {
...ctx,
operationResolver: createOperationResolver(fastify, headers, ctx),
}
if (result.failure) {
failures.push(result.failure)
} else {
passedCount++
// Evaluate requires first. If any fail, skip ensures for this run.
let requiresPassed = true
for (const formula of route.requires) {
total++
try {
const parsed = parse(formula)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
requiresPassed = false
skippedCount++
} else {
passedCount++
}
} catch {
requiresPassed = false
skippedCount++
}
}
// Only evaluate ensures if all requires passed
if (requiresPassed) {
for (const formula of route.ensures) {
total++
try {
const parsed = parse(formula)
const result = await evaluateAsync(parsed.ast, evalCtx)
if (!result.success || !result.value) {
const diagnostic = buildFailureDiagnostic(formula, evalCtx, result.success ? undefined : result.error)
failures.push({
route: variant.name && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract: formula,
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
})
} else {
passedCount++
}
} catch (error) {
const diagnostic = buildFailureDiagnostic(formula, evalCtx, error instanceof Error ? error.message : String(error))
failures.push({
route: variant.name && variant.name !== 'default'
? `[variant:${variant.name}] ${route.method} ${route.path}`
: `${route.method} ${route.path}`,
contract: formula,
expected: diagnostic.expected,
observed: diagnostic.observed,
formula: diagnostic.formula,
category: diagnostic.category,
})
}
}
} else {
// Preconditions failed: ensures are skipped but not evaluated.
// We still count each ensure as "total" and "skipped" for transparency.
total += route.ensures.length
skippedCount += route.ensures.length
}
}
}
}
@@ -578,6 +691,7 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
total,
passedCount,
failed: failures.length,
skipped: skippedCount,
failures: sortedFailures,
durationMs,
noRoutesMatched: false,