apophis/apophis-fastify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden engine, enrich failure diagnostics, close adoption gaps

Browse Source 
- P0: CLI verify now honors  test budget with seeded multi-sample
- P0: Observe sampling enforced via Math.random() gate in hook-validator
- P1: Remove misleading undici-mock-agent isolation option
- P1: Qualify reuses shared discoverRouteDetails() with warnings
- P1: Chaos/scenario config exposed via preset schema
- P1: README/docs limitations updated to current state
- P2: Nested response annotations prefer 2xx deterministically
- P2: --changed documented as heuristic in verify.md

- Add observe sink tests (sampling 0/1, sink failure non-interference)
- Add verify runs regression tests (scale, determinism, variants)
- Add configured-scenario qualify test (independent of OAuth fixture)
- Add coverageBreakdown to qualify artifacts (per-gate route coverage)
- Add production-style observe example with real sink in docs/observe.md
- Add nightly/staging vs PR gating guidance to docs/qualify.md

- Enrich VerifyFailure with formula-aware diagnostics:
  status:201 => 'HTTP 200', body field checks => actual values
- Remove stale observe CLI activation message
- Document outbound mocks as process-global in getting-started.md
- Refresh APOPHIS_ADOPTION_AUDIT.md with current state

903 tests pass, build clean, typecheck clean.
This commit is contained in:
John Dvorak
2026-05-21 20:39:36 -07:00
parent 55b0262799
commit d0523fcc2d
128 changed files with 4004 additions and 3631 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/core/app-loader.ts
+157 -45
View File
@@ -1,99 +1,211 @@
/**
* App loader utility for CLI commands.
* Handles various app export patterns and module systems.
*
* Supports auto-detection of entrypoint files:
* app.js, app.ts, server.js, server.ts, index.js, index.ts,
* src/app.js, src/app.ts, src/server.js, src/server.ts,
* src/index.js, src/index.ts
*/
import { resolve } from 'node:path'
import { existsSync } from 'node:fs'
import { resolve, extname } from 'node:path'
import { pathToFileURL } from 'node:url'
function isTsxAvailable(): boolean {
return process.execArgv.some(arg => arg.includes('tsx') || arg.includes('ts-node')) ||
process.argv[0]?.includes('tsx') ||
!!process.env.TSX_TSCONFIG_PATH
}
export const APP_CANDIDATES = [
'app.js',
'app.ts',
'server.js',
'server.ts',
'index.js',
'index.ts',
'src/app.js',
'src/app.ts',
'src/server.js',
'src/server.ts',
'src/index.js',
'src/index.ts',
]
export interface LoadedApp {
fastify: unknown
source: 'default' | 'named' | 'commonjs'
entrypoint: string
}
/**
* Load a Fastify app from app.js in the given directory.
* Supports:
* Find the first existing app entrypoint file in the given directory.
* Returns the resolved absolute path, or null if no candidate exists.
*/
export function findAppFile(cwd: string): string | null {
for (const candidate of APP_CANDIDATES) {
const fullPath = resolve(cwd, candidate)
if (existsSync(fullPath)) {
return fullPath
}
}
return null
}
/**
* Load a Fastify app from the given directory.
*
* Auto-detects the entrypoint by searching APP_CANDIDATES for the first
* existing file. Supports:
* - ESM default export: export default fastifyInstance
* - ESM named export: export const createApp = () => fastifyInstance
* - CommonJS: module.exports = fastifyInstance
* - CommonJS named: exports.createApp = () => fastifyInstance
*
* If an explicit entrypoint is provided, it takes precedence over auto-detection.
* For .ts entrypoints, a clear error message suggests installing tsx.
*
* After loading the app, this function ensures route discovery works even if the
* APOPHIS plugin is not registered or was registered after routes.
*/
export async function loadApp(cwd: string): Promise<LoadedApp> {
const appPath = resolve(cwd, 'app.js')
const appUrl = pathToFileURL(appPath).href + '?t=' + Date.now()
export async function loadApp(cwd: string, entrypoint?: string): Promise<LoadedApp> {
const resolvedEntrypoint = entrypoint
? resolve(cwd, entrypoint)
: findAppFile(cwd)
if (!resolvedEntrypoint) {
throw new AppLoadError(
`No Fastify app entrypoint found. Searched for: ${APP_CANDIDATES.join(', ')}.\n` +
'Create an app.js, server.js, or similar that exports a Fastify instance.',
'not_found',
)
}
const ext = extname(resolvedEntrypoint)
const isTypeScript = ext === '.ts'
if (isTypeScript && !isTsxAvailable()) {
throw new AppLoadError(
`TypeScript entrypoint ${resolvedEntrypoint} requires a TS loader.\n` +
'Install tsx and run:\n' +
' npm install -D tsx\n' +
' npx tsx ./node_modules/.bin/apophis verify\n\n' +
'Or convert your entrypoint to JavaScript.',
'import_failed',
)
}
const appUrl = pathToFileURL(resolvedEntrypoint).href + '?t=' + Date.now()
let appModule: Record<string, unknown>
try {
appModule = await import(appUrl) as Record<string, unknown>
} catch (err) {
const errMessage = err instanceof Error ? err.message : String(err)
if (isTypeScript) {
throw new AppLoadError(
`Cannot load TypeScript entrypoint ${resolvedEntrypoint}: ${errMessage}\n\n` +
'TypeScript entrypoints require tsx. Install tsx and run:\n' +
' npm install -D tsx\n' +
' npx tsx ./node_modules/.bin/apophis verify\n\n' +
'Or convert your entrypoint to JavaScript.',
'import_failed',
)
}
throw new AppLoadError(
`Cannot load app.js: ${err instanceof Error ? err.message : String(err)}`,
`Cannot load ${resolvedEntrypoint}: ${errMessage}`,
'import_failed',
)
}
// Try default export first
let fastify: unknown
let source: 'default' | 'named' | 'commonjs' = 'default'
if (appModule.default && isFastifyInstance(appModule.default)) {
return { fastify: appModule.default, source: 'default' }
}
fastify = appModule.default
source = 'default'
} else {
let found = false
for (const [key, value] of Object.entries(appModule)) {
if (key === 'default') continue
// Try named exports that look like Fastify instances or factory functions
for (const [key, value] of Object.entries(appModule)) {
if (key === 'default') continue
if (isFastifyInstance(value)) {
fastify = value
source = 'named'
found = true
break
}
if (isFastifyInstance(value)) {
return { fastify: value, source: 'named' }
}
// Try calling factory functions
if (typeof value === 'function' && !isClass(value)) {
try {
const result = await value()
if (isFastifyInstance(result)) {
return { fastify: result, source: 'named' }
if (typeof value === 'function' && !isClass(value)) {
try {
const result = await value()
if (isFastifyInstance(result)) {
fastify = result
source = 'named'
found = true
break
}
} catch {
// Factory function failed, try next
}
} catch {
// Factory function failed, try next
}
}
if (!found && isFastifyInstance(appModule)) {
fastify = appModule
source = 'commonjs'
found = true
}
if (!found) {
throw new AppLoadError(
`No Fastify instance found in ${resolvedEntrypoint}. ` +
'Ensure the file exports a Fastify instance or a factory function.\n\n' +
'Supported patterns:\n' +
' export default app\n' +
' export { app }\n' +
' export const createApp = () => app\n' +
' module.exports = app',
'no_fastify',
)
}
}
// If module itself is a Fastify instance (CommonJS)
if (isFastifyInstance(appModule)) {
return { fastify: appModule, source: 'commonjs' }
}
await ensureRouteDiscovery(fastify as Record<string, unknown>)
throw new AppLoadError(
'No Fastify instance found in app.js. Ensure app.js exports a Fastify instance or a factory function.',
'no_fastify',
)
return { fastify, source, entrypoint: resolvedEntrypoint }
}
/**
* Check if a value looks like a Fastify instance.
* Install a lightweight onRoute hook for route discovery.
* This runs before avvio's boot phase, so routes registered
* inside deferred plugins will be captured even if apophis
* is not registered first.
*/
async function ensureRouteDiscovery(f: Record<string, unknown>): Promise<void> {
if (typeof f.addHook !== 'function') return
try {
const { installRouteDiscovery } = await import('../../infrastructure/discovery-hook.js')
installRouteDiscovery(f as { addHook: Function })
} catch {
// discovery module not importable, skip auto-capture
}
}
function isFastifyInstance(value: unknown): boolean {
return value !== null &&
typeof value === 'object' &&
typeof (value as Record<string, unknown>).ready === 'function'
if (value === null || typeof value !== 'object') return false
const obj = value as Record<string, unknown>
return typeof obj.ready === 'function'
}
/**
* Check if a function is a class constructor.
*/
function isClass(fn: unknown): boolean {
return typeof fn === 'function' &&
fn.toString().startsWith('class ')
}
/**
* Error type for app loading failures.
*/
export class AppLoadError extends Error {
constructor(
message: string,
public readonly code: 'import_failed' | 'no_fastify',
public readonly code: 'import_failed' | 'no_fastify' | 'not_found',
) {
super(message)
this.name = 'AppLoadError'
src/cli/core/config-loader.ts
+63 -3
View File
@@ -30,6 +30,7 @@ export interface Config {
environments?: Record<string, EnvironmentPolicy>;
profiles?: Record<string, ProfileDefinition>;
presets?: Record<string, PresetDefinition>;
scenarios?: ScenarioConfigDef[];
[key: string]: unknown;
}
@@ -53,9 +54,28 @@ export interface PresetDefinition {
routes?: string[];
seed?: number;
features?: string[];
chaos?: boolean;
chaosStrategy?: 'one' | 'all' | 'sample' | 'routes';
chaosSampleSize?: number;
chaosSampleRoutes?: string[];
[key: string]: unknown;
}
export interface ScenarioConfigDef {
name: string;
steps: Array<{
name: string;
request: {
method: string;
url: string;
body?: unknown;
headers?: Record<string, string>;
};
expect: string[];
capture?: Record<string, string>;
}>;
}
export interface LoadConfigOptions {
cwd: string;
configPath?: string;
@@ -111,6 +131,20 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
optional: true,
items: { type: 'string' },
},
metadata: {
type: 'object',
optional: true,
},
scenarios: {
type: 'array',
optional: true,
items: { type: 'object' },
},
chaos: {
type: 'object',
optional: true,
properties: {},
},
};
// Schema for EnvironmentPolicy values (inside environments.<name>)
@@ -140,6 +174,10 @@ const PROFILE_SCHEMA: Record<string, SchemaField> = {
sampling: { type: 'number', optional: true },
blocking: { type: 'boolean', optional: true },
sinks: { type: 'object', optional: true },
chaos: { type: 'object', optional: true },
chaosStrategy: { type: 'string', optional: true, enumValues: ['one', 'all', 'sample', 'routes'] },
chaosSampleSize: { type: 'number', optional: true, min: 1 },
chaosSampleRoutes: { type: 'array', optional: true, items: { type: 'string' } },
};
// Schema for PresetDefinition values (inside presets.<name>)
@@ -148,6 +186,9 @@ const PRESET_SCHEMA: Record<string, SchemaField> = {
timeout: { type: 'number', optional: true, min: 0 },
parallel: { type: 'boolean', optional: true },
chaos: { type: 'boolean', optional: true },
chaosStrategy: { type: 'string', optional: true, enumValues: ['one', 'all', 'sample', 'routes'] },
chaosSampleSize: { type: 'number', optional: true, min: 1 },
chaosSampleRoutes: { type: 'array', optional: true, items: { type: 'string' } },
observe: { type: 'boolean', optional: true },
features: { type: 'array', optional: true, items: { type: 'string' } },
sampling: { type: 'number', optional: true },
@@ -201,7 +242,7 @@ export function loadPackageJsonConfig(cwd: string): { config: Config | null; pat
/**
* Load a config file by path.
* Supports .js, .ts (via dynamic import, assumes tsx available), and .json.
* Supports .js, .ts (via dynamic import when tsx is available), and .json.
*/
export async function loadConfigFile(configPath: string): Promise<Config> {
if (configPath.endsWith('.json')) {
@@ -209,6 +250,14 @@ export async function loadConfigFile(configPath: string): Promise<Config> {
return JSON.parse(content) as Config;
}
if (configPath.endsWith('.ts') && !process.execArgv.some(a => a.includes('tsx') || a.includes('ts-node'))) {
throw new Error(
`TypeScript config file ${configPath} requires a TS loader.\n` +
'Convert to apophis.config.js or apophis.config.json, or run through tsx:\n' +
' npx tsx ./node_modules/.bin/apophis verify'
);
}
// For .js and .ts, use dynamic import.
// tsx handles .ts files in dev environments.
const fileUrl = pathToFileURL(configPath).href;
@@ -259,6 +308,15 @@ function isInsideDynamicContainer(path: string): boolean {
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.');
}
/**
* User-managed extension keys that pass through schema validation.
* APOPHIS will never interpret these fields; they are reserved for team metadata.
* Keys starting with 'x-' are always allowed at any level.
*/
function isUserExtensionKey(key: string): boolean {
return key.startsWith('x-')
}
/**
* Validate that a value matches the expected type for a schema field.
* Throws ConfigValidationError on type mismatch.
@@ -406,13 +464,14 @@ export function validateConfigAgainstSchema(
}
}
} else {
if (isUserExtensionKey(key)) continue
// Unknown key inside a profile/preset/environment object
throw new ConfigValidationError(
`Unknown config key at ${currentPath}`,
currentPath,
key,
obj[key],
`Valid keys for ${parentContainer} entries: ${Object.keys(childSchema || {}).join(', ')}.`,
`Valid keys for ${parentContainer} entries: ${Object.keys(childSchema || {}).join(', ')}. Use x- prefixed keys for team metadata.`,
);
}
}
@@ -420,12 +479,13 @@ export function validateConfigAgainstSchema(
}
if (!fieldSchema) {
if (isUserExtensionKey(key)) continue
throw new ConfigValidationError(
`Unknown config key at ${currentPath}`,
currentPath,
key,
obj[key],
`Valid top-level keys: ${Object.keys(CONFIG_SCHEMA).join(', ')}.`,
`Valid top-level keys: ${Object.keys(CONFIG_SCHEMA).join(', ')}. Use the metadata field or x- prefixed keys for team data.`,
);
}
src/cli/core/index.ts
+5 -5
View File
@@ -2,7 +2,7 @@ import { cac } from 'cac';
import pc from 'picocolors';
import { createContext, type CliContext } from './context.js';
const CLI_VERSION = '2.0.0';
const CLI_VERSION = '2.7.0';
const HELP_HEADER = `
${pc.bold('apophis')} — Contract-driven API testing for Fastify
@@ -23,7 +23,7 @@ const HELP_HEADER = `
--config <path> Config file path
--profile <name> Profile name from config
--cwd <path> Working directory override
--format <mode> Output format: human | json | ndjson (default: human)
--format <mode> Output format: human | json | ndjson | json-summary | ndjson-summary (default: human)
--color <mode> Color mode: auto | always | never (default: auto)
--quiet Suppress non-error output
--verbose Enable verbose logging
@@ -223,7 +223,7 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
cli.option('--config <path>', 'Config file path');
cli.option('--profile <name>', 'Profile name from config');
cli.option('--cwd <path>', 'Working directory override');
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
cli.option('--format <mode>', 'Output format: human | json | ndjson | json-summary | ndjson-summary', { default: 'human' });
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
cli.option('--quiet', 'Suppress non-error output');
cli.option('--verbose', 'Enable verbose logging');
@@ -372,11 +372,11 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
]);
const commandSpecificFlags: Record<string, Set<string>> = {
init: new Set(['--preset', '--force', '--noninteractive']),
init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']),
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config', '--workspace']),
qualify: new Set(['--profile', '--seed', '--workspace']),
replay: new Set(['--artifact']),
replay: new Set(['--artifact', '--route']),
doctor: new Set(['--mode', '--strict', '--workspace']),
migrate: new Set(['--check', '--dry-run', '--write']),
};
src/cli/core/types.ts
+5
View File
@@ -299,6 +299,11 @@ export interface Artifact {
failed: number;
};
executionSummary?: ExecutionSummary;
coverageBreakdown?: {
scenario: { routesCovered: string[]; stepsTotal: number; stepsPassed: number }
stateful: { routesCovered: string[]; testsTotal: number; testsPassed: number }
chaos: { routesPlanned: string[]; routesExecuted: string[]; runsTotal: number; runsPassed: number }
}
executedRoutes?: string[];
skippedRoutes?: RouteExecutionInfo[];
stepTraces?: StepTrace[];