feat: plugin contract e2e, qualify --changed, production observe, regressions

src/cli/commands/qualify/index.ts

@@ -30,6 +30,7 @@ import {
   type StepTrace,
   type CleanupFailure,
 } from './runner.js'
+import { isGitRepo, filterChangedRoutes } from '../verify/runner.js'
 import { SeededRng } from '../../../infrastructure/seeded-rng.js'
 import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
 import { renderHumanArtifact } from '../../renderers/human.js'
@@ -64,6 +65,7 @@ function coerceTimeout(value: unknown): number | undefined {
 export interface QualifyOptions {
   profile?: string
   seed?: number
+  changed?: boolean
   config?: string
   cwd?: string
   format?: 'human' | 'json' | 'ndjson'
@@ -252,7 +254,8 @@ export function buildArtifact(
 
   const totalTests =
     runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
-    (runResult.statefulResult?.tests.length ?? 0)
+    (runResult.statefulResult?.summary.passed ?? 0) +
+    (runResult.statefulResult?.summary.failed ?? 0)
 
   const passedTests =
     runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
@@ -282,7 +285,11 @@ export function buildArtifact(
       .filter(Boolean)
   )]
   const statefulRoutes = runResult.statefulResult
-    ? [...new Set(runResult.statefulResult.tests.map(t => normalizeRouteIdentity(t.name)))]
+    ? [...new Set(
+        runResult.statefulResult.tests
+          .filter(t => !t.directive)
+          .map(t => normalizeRouteIdentity(t.name))
+      )]
     : []
   const chaosRoutesPlanned: string[] = []
   const chaosRoutesExecuted: string[] = []
@@ -343,6 +350,7 @@ export function buildArtifact(
       total: totalTests,
       passed: passedTests,
       failed: failures.length,
+      skipped: (runResult.statefulResult?.summary.skipped ?? 0),
     },
     executionSummary,
     coverageBreakdown,
@@ -437,6 +445,7 @@ export async function qualifyCommand(
   const {
     profile,
     seed: explicitSeed,
+    changed,
     config: configPath,
     cwd,
     artifactDir,
@@ -566,7 +575,31 @@ export async function qualifyCommand(
 
     try {
       // 9. Discover routes for chaos (with discovery metadata)
-      const appRoutes = await discoverAppRoutes(fastify)
+      const allRoutes = await discoverAppRoutes(fastify)
+      let appRoutes = allRoutes
+
+      // 9a. Apply --changed filter
+      if (changed) {
+        const inGit = await isGitRepo(workingDir)
+        if (!inGit) {
+          return {
+            exitCode: USAGE_ERROR,
+            message: '--changed requires a git repository. Current directory is not inside a git repo.',
+          }
+        }
+        appRoutes = await filterChangedRoutes(allRoutes, workingDir)
+        const matched = appRoutes.length
+        const total = allRoutes.length
+        if (!ctx.options.quiet && format === 'human') {
+          console.log(`${matched}/${total} routes match --changed filter`)
+        }
+        if (matched === 0) {
+          return {
+            exitCode: SUCCESS,
+            message: `No routes changed since HEAD. All ${total} routes skipped.`,
+          }
+        }
+      }
 
       // Collect discovery warnings
       const { discoverRouteDetails } = await import('../../../domain/discovery.js')
@@ -586,6 +619,8 @@ export async function qualifyCommand(
         fastify: fastify as any,
         seed,
         timeout: presetTimeout,
+        pluginContractRegistry: (fastify as unknown as Record<string, unknown>)?._apophisPluginContractRegistry as
+          import('../../../domain/plugin-contracts.js').PluginContractRegistry | undefined,
       }
 
       const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
@@ -600,6 +635,15 @@ export async function qualifyCommand(
         seed,
       })
 
+      // Drain plugin contract registry warnings if app registered APOPHIS
+      const pcr = (fastify as any)?._apophisPluginContractRegistry
+      if (pcr?.drainWarnings) {
+        const pcrWarnings = pcr.drainWarnings()
+        if (pcrWarnings.length > 0) {
+          artifact.warnings.push(...pcrWarnings)
+        }
+      }
+
       // 12. Signal quality guardrails — fail if zero checks executed
       const execSummary = runResult.executionSummary
       const warnings: string[] = [...artifact.warnings, ...discoveryWarnings]
@@ -624,7 +668,9 @@ export async function qualifyCommand(
       if (gates.scenario && execSummary.scenariosRun === 0) {
         warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
       }
-      if (gates.stateful && execSummary.statefulTestsRun === 0) {
+      if (gates.stateful && execSummary.statefulTestsRun === 0 && execSummary.statefulSkipped > 0) {
+        warnings.push(`WARNING: stateful gate enabled but all ${execSummary.statefulSkipped} commands were skipped (preconditions not met). Check required headers, scopes, or plugin contract settings.`)
+      } else if (gates.stateful && execSummary.statefulTestsRun === 0) {
         warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
       }
       if (gates.chaos && execSummary.chaosRunsRun === 0) {
@@ -706,6 +752,7 @@ export async function handleQualify(
   const options: QualifyOptions = {
     profile: ctx.options.profile || undefined,
     seed: undefined,
+    changed: false,
     config: ctx.options.config || undefined,
     cwd: ctx.cwd,
     format: ctx.options.format as QualifyOptions['format'],
@@ -722,6 +769,10 @@ export async function handleQualify(
     }
   }
 
+  if (args.includes('--changed')) {
+    options.changed = true
+  }
+
   const result = await qualifyCommand(options, ctx)
   const format = options.format || ctx.options.format || 'human'
   const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'