fix: harden engine, enrich failure diagnostics, close adoption gaps

- P0: CLI verify now honors test budget with seeded multi-sample - P0: Observe sampling enforced via Math.random() gate in hook-validator - P1: Remove misleading undici-mock-agent isolation option - P1: Qualify reuses shared discoverRouteDetails() with warnings - P1: Chaos/scenario config exposed via preset schema - P1: README/docs limitations updated to current state - P2: Nested response annotations prefer 2xx deterministically - P2: --changed documented as heuristic in verify.md - Add observe sink tests (sampling 0/1, sink failure non-interference) - Add verify runs regression tests (scale, determinism, variants) - Add configured-scenario qualify test (independent of OAuth fixture) - Add coverageBreakdown to qualify artifacts (per-gate route coverage) - Add production-style observe example with real sink in docs/observe.md - Add nightly/staging vs PR gating guidance to docs/qualify.md - Enrich VerifyFailure with formula-aware diagnostics: status:201 => 'HTTP 200', body field checks => actual values - Remove stale observe CLI activation message - Document outbound mocks as process-global in getting-started.md - Refresh APOPHIS_ADOPTION_AUDIT.md with current state 903 tests pass, build clean, typecheck clean.
2026-05-21 20:39:36 -07:00
parent 55b0262799
commit d0523fcc2d
128 changed files with 4004 additions and 3631 deletions
@@ -14,6 +14,14 @@ interface CapturedRoute {
  schema?: Record<string, unknown>
  prefix?: string
 }
+
+export interface DiscoveryResult {
+  routes: RouteContract[]
+  source: 'captured' | 'legacy-routes-array' | 'print-routes' | 'none'
+  hasSchemaMetadata: boolean
+  warnings: string[]
+}
+
 // WeakMap to store captured routes per Fastify instance (no memory leaks)
 const capturedRoutes = new WeakMap<object, CapturedRoute[]>()
 /**
@@ -32,36 +40,127 @@ export const captureRoute = (
 /**
 * Fallback route discovery for Fastify 5 when routes were registered before
 * the APOPHIS plugin (e.g., external apps loaded by CLI).
- * Uses hasRoute to test known route patterns.
+ * Parses printRoutes() output to discover route paths and methods.
+ *
+ * printRoutes() output looks like:
+ *   └── /
+ *       ├── users (GET, HEAD)
+ *       └── items/
+ *           └── :id (POST)
+ */
+function parsePrintRoutesOutput(output: string): Array<{ method: string; path: string }> {
+  const routes: Array<{ method: string; path: string }> = []
+  const lines = output.split('\n')
+
+  const depthPrefixes: string[] = []
+
+  for (const line of lines) {
+    const trimmed = line.trimStart()
+    if (!trimmed) continue
+
+    const indent = line.length - line.trimStart().length
+    const depth = Math.floor(indent / 2)
+
+    depthPrefixes.length = depth
+
+    const branchMatch = trimmed.match(/^(├──|└──)\s+(.+)$/)
+    if (!branchMatch) continue
+
+    let segment = branchMatch[2]!
+
+    const isIntermediate = segment.endsWith('/')
+    if (isIntermediate) {
+      segment = segment.slice(0, -1)
+    }
+
+    const methodMatch = segment.match(/^(.+?)\s+\(([^)]+)\)$/)
+    if (methodMatch) {
+      const rawPath = methodMatch[1]!
+      const methodList = methodMatch[2]!.split(',').map(m => m.trim())
+      const fullPath = '/' + [...depthPrefixes, rawPath].filter(Boolean).join('/')
+
+      for (const method of methodList) {
+        if (method === 'HEAD') continue
+        routes.push({ method, path: fullPath })
+      }
+    } else {
+      depthPrefixes.push(segment)
+    }
+  }
+
+  return routes
+}
+
+/**
+ * Fallback route discovery for Fastify 5 when onRoute hook wasn't installed
+ * before route registration.
 */
 function discoverRoutesFallback(
-  instance: { hasRoute?: (opts: { method: string; url: string }) => boolean }
-): RouteContract[] {
-  if (typeof instance.hasRoute !== 'function') {
-    return []
+  instance: {
+    hasRoute?: (opts: { method: string; url: string }) => boolean
+    printRoutes?: () => string
+  }
+): DiscoveryResult {
+  let routes: RouteContract[] = []
+  let source: DiscoveryResult['source'] = 'none'
+  let warnings: string[] = []
+
+  if (typeof instance.printRoutes === 'function') {
+    const output = instance.printRoutes()
+    const parsed = parsePrintRoutesOutput(output)
+    if (parsed.length > 0) {
+      routes = parsed.map(r => extractContract(r.path, r.method, undefined))
+      source = 'print-routes'
+      warnings.push(
+        'Route schemas were not available during discovery. ' +
+        'Behavioral contracts (x-ensures, x-requires, x-outbound, x-variants, x-timeout) ' +
+        'will not be detected. Register APOPHIS or install route discovery before defining routes, ' +
+        'or use createFastify() from @apophis/fastify.'
+      )
+    }
  }

-  // Common HTTP methods to test
-  const methods = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS']
+  return { routes, source, hasSchemaMetadata: false, warnings }
+}

-  // We can't enumerate all possible routes, but we can check if the instance
-  // has any routes at all by testing a few common patterns
-  // This is a best-effort fallback
-  const routes: RouteContract[] = []
+export interface DiscoverRouteDetailsInput {
+  routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }>
+  hasRoute?: (opts: { method: string; url: string }) => boolean
+  printRoutes?: () => string
+}

-  // Try to extract routes from the instance's internal state
-  // Fastify stores routes in find-my-way router, but it's not directly accessible
-  // We'll use a heuristic: check if the instance responds to common route methods
-
-  // Check if instance has any routes by looking at prototype methods
-  const hasRouting = typeof (instance as any).routing === 'function'
-  if (!hasRouting) {
-    return []
+/**
+ * Discover routes from a Fastify instance with full metadata about discovery quality.
+ *
+ * Use this function when you need to know whether routes were discovered
+ * with full schema metadata (captured / legacy-routes-array) or as
+ * schema-less fallback paths (print-routes / none).
+ */
+export const discoverRouteDetails = (instance: DiscoverRouteDetailsInput): DiscoveryResult => {
+  const captured = capturedRoutes.get(instance)
+  if (captured && captured.length > 0) {
+    return {
+      routes: captured.map((route) =>
+        extractContract(route.url, route.method, route.schema)
+      ),
+      source: 'captured',
+      hasSchemaMetadata: true,
+      warnings: [],
+    }
  }

-  // Since we can't enumerate routes in Fastify 5 without the onRoute hook,
-  // we return empty and let the caller handle the "no routes" case
-  return []
+  if (Array.isArray(instance.routes) && instance.routes.length > 0) {
+    return {
+      routes: instance.routes.map((route) =>
+        extractContract(route.url, route.method, route.schema)
+      ),
+      source: 'legacy-routes-array',
+      hasSchemaMetadata: true,
+      warnings: [],
+    }
+  }
+
+  return discoverRoutesFallback(instance)
 }

 /**
@@ -70,20 +169,6 @@ function discoverRoutesFallback(
 * First checks captured routes (from onRoute hook), then falls back to
 * the legacy `routes` array for Fastify 4 compatibility.
 */
-export const discoverRoutes = (instance: { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }>; hasRoute?: (opts: { method: string; url: string }) => boolean }): RouteContract[] => {
-  // Fastify 5: routes captured via onRoute hook
-  const captured = capturedRoutes.get(instance)
-  if (captured && captured.length > 0) {
-    return captured.map((route) =>
-      extractContract(route.url, route.method, route.schema)
-    )
-  }
-  // Fastify 4 fallback
-  if (Array.isArray(instance.routes) && instance.routes.length > 0) {
-    return instance.routes.map((route) =>
-      extractContract(route.url, route.method, route.schema)
-    )
-  }
-  // Fastify 5 fallback: routes registered before plugin
-  return discoverRoutesFallback(instance)
+export const discoverRoutes = (instance: DiscoverRouteDetailsInput): RouteContract[] => {
+  return discoverRouteDetails(instance).routes
 }