Compare commits
23 Commits
bf7376b5ad
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| fa0f6e1fe5 | |||
| 4be5fd74cb | |||
| 9474e2d321 | |||
| 67d0d02c12 | |||
| 79d6cd130d | |||
| 6331933388 | |||
| dab14ef77d | |||
| 26f0e6ad7e | |||
| fca585f1e7 | |||
| fe8e94fbaa | |||
| 04359613b5 | |||
| 60e710370a | |||
| 4891b0883e | |||
| a186146c69 | |||
| edc2989900 | |||
| 6e4656add5 | |||
| 1de735ee08 | |||
| d0523fcc2d | |||
| 55b0262799 | |||
| 7ca78b74b8 | |||
| db1c3f3663 | |||
| 687321d2cf | |||
| 3ac1daf7e9 |
@@ -0,0 +1,28 @@
|
||||
name: Publish npm Package
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: node-22
|
||||
steps:
|
||||
- name: Route Git to external Gitea URL
|
||||
run: git config --global url."https://hub.kl1.tenere.ai/".insteadOf "http://gitea:3000/"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install
|
||||
run: npm install --registry https://packages.tenere.ai
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Publish to Gitea npm
|
||||
env:
|
||||
NPM_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
||||
run: |
|
||||
echo "//hub.kl1.tenere.ai/api/packages/apophis/npm/:_authToken=${NPM_TOKEN}" > .npmrc
|
||||
npm publish --registry https://hub.kl1.tenere.ai/api/packages/apophis/npm/
|
||||
@@ -1,44 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node-version: [20.x, 22.x]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Run source tests
|
||||
run: npm run test:src
|
||||
|
||||
- name: Run CLI tests
|
||||
run: npm run test:cli
|
||||
|
||||
- name: Determinism smoke (fixed seed)
|
||||
run: npx tsx --test src/test/cli/verify-ux.test.ts --test-name-pattern "verify repeated runs with fixed seed produce identical artifacts"
|
||||
|
||||
- name: Run packaging tests
|
||||
run: npx tsx --test src/test/cli/packaging.test.ts
|
||||
+22
-11
@@ -1,22 +1,33 @@
|
||||
# APOPHIS Setup — safe-ci preset
|
||||
# APOPHIS Setup — llm-safe preset
|
||||
|
||||
This project was scaffolded with `apophis init --preset safe-ci`.
|
||||
This project was scaffolded with `apophis init --preset llm-safe`.
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Confirm the Fastify app registers `@fastify/swagger`.
|
||||
1. Ensure you have a Fastify app with @fastify/swagger registered.
|
||||
2. Add behavioral contracts to your route schemas using `x-ensures`.
|
||||
3. Run: apophis verify --profile quick
|
||||
3. Run: apophis verify --profile llm-check
|
||||
|
||||
## What This Preset Does
|
||||
|
||||
- Runs only behavioral contracts (not schema-only routes).
|
||||
- No chaos, no observe, no stateful testing.
|
||||
- Safe for CI pipelines.
|
||||
- Timeout: 5s per route.
|
||||
- Ultra-minimal preset for LLM-generated codebases.
|
||||
- 3s timeout per route (fast feedback).
|
||||
- No observe, no qualify, no chaos — verify only.
|
||||
- Conservative defaults to avoid surprising failures.
|
||||
|
||||
## Example Behavioral Contract
|
||||
|
||||
Add this inside your route schema to check that a created resource is retrievable:
|
||||
|
||||
```javascript
|
||||
"x-ensures": [
|
||||
"response_code(GET /users/{response_body(this).id}) == 200"
|
||||
]
|
||||
```
|
||||
|
||||
If `apophis verify` says "No behavioral contracts found", it means your routes have schemas but no `x-ensures` or `x-requires` clauses. Add at least one clause per route you want to verify.
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Add more routes to the `routes` array in your profile.
|
||||
- Try `apophis init --preset platform-observe` to configure observe-mode policy and runtime drift reporting.
|
||||
- Try `apophis init --preset protocol-lab` for multi-step flows.
|
||||
- Add routes to the `routes` array once you have behavioral contracts.
|
||||
- Run `apophis doctor` to check for missing dependencies.
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
# APOPHIS Adoption Audit
|
||||
|
||||
Date: 2026-05-21
|
||||
|
||||
Scope: current working tree for `@apophis/fastify` v2.7.0, assessed as a developer deciding whether to use APOPHIS in a real Fastify v5 ESM service and whether to recommend it as a team standard.
|
||||
|
||||
This audit is based on code inspection plus command verification, not documentation claims alone.
|
||||
|
||||
## Executive Summary
|
||||
|
||||
APOPHIS has real product value. It is not just a schema wrapper: it gives Fastify teams a way to express, verify, and observe behavioral API promises that OpenAPI/JSON Schema cannot cover, especially cross-route invariants such as create/read consistency, delete semantics, auth/session flows, state transitions, idempotency, outbound dependency expectations, and replayable counterexamples.
|
||||
|
||||
I would adopt APOPHIS today as a focused behavioral verification tool for Fastify v5 ESM services. I would start with CI `verify` and a small number of high-value contracts, then expand into `qualify` and runtime observation once the team has clear operating guidance.
|
||||
|
||||
I would not yet treat it as a complete production observability platform or a turnkey organization-wide release gate. The core implementation is strong, but the remaining value gap is mostly around operational maturity: standalone observe process management, richer scenario authoring, and organization-specific release-gate policy.
|
||||
|
||||
Adoption verdict: strong team pilot candidate, credible standardization candidate after the remaining gaps below are addressed.
|
||||
|
||||
## Verification Performed
|
||||
|
||||
Commands run successfully against the current working tree:
|
||||
|
||||
```bash
|
||||
npm run typecheck
|
||||
npm run build
|
||||
npm run test:src
|
||||
npm run test:cli
|
||||
npm run test:docs
|
||||
```
|
||||
|
||||
Observed results:
|
||||
|
||||
| Area | Result |
|
||||
|---|---:|
|
||||
| Typecheck | pass |
|
||||
| Build | pass |
|
||||
| Source tests | 590 pass, 0 fail (at audit time) |
|
||||
| CLI tests | 278 pass, 0 fail (at audit time) |
|
||||
| Docs smoke tests | 4 pass, 0 fail |
|
||||
| Total tests | 879 pass, 0 fail (at audit time) |
|
||||
|
||||
Test counts are snapshots at audit time (2026-05-21) and may differ in subsequent development.
|
||||
|
||||
The working tree contains many broader project changes unrelated to this audit. This document evaluates the current working tree state.
|
||||
|
||||
## Does It Do What It Says On The Tin?
|
||||
|
||||
Mostly yes for behavioral verification. Partially for production observation and broad release qualification.
|
||||
|
||||
| Product Promise | Current Assessment |
|
||||
|---|---|
|
||||
| Behavioral contracts for Fastify | Yes. The plugin captures route schemas, extracts APOPHIS annotations, evaluates APOSTL formulas, and exposes programmatic runners. |
|
||||
| Deterministic CI verification | Yes, materially. CLI `verify` now honors configured `runs`, uses seeded request generation, emits artifacts, supports route filters, replay metadata, and machine-readable output. |
|
||||
| Cross-route behavior | Yes for supported formula operations and route-call semantics. This is the most differentiated value. |
|
||||
| Runtime validation | Yes when the plugin is explicitly configured outside production. Production enforcement is intentionally blocked. |
|
||||
| Runtime observation | Yes for programmatic production-safe hooks. APOPHIS emits non-blocking sink events with sampling in production when `observe.enabled` and sinks are configured. The CLI validates/report readiness but does not attach to or run a service. |
|
||||
| Stateful/scenario/chaos qualification | Partially. The runner and artifacts are useful, route discovery is now shared with verify, and config supports scenarios/chaos knobs. Scenario authoring is still young and needs more real-world examples/tests. |
|
||||
| Outbound dependency mocking | Useful but intentionally process-global. The misleading scoped `undici-mock-agent` option has been removed. Teams still need careful test isolation. |
|
||||
| Team-safe onboarding | Good. The package has CLI help, init/doctor/replay/verify/qualify/observe, config validation, machine output, docs smoke tests, packaging tests, and production safety checks. |
|
||||
|
||||
## What Has Real Value
|
||||
|
||||
1. Behavioral contracts fill a real Fastify testing gap.
|
||||
|
||||
JSON Schema validates shape. APOPHIS validates behavior: whether one operation changes another operation's result, whether an auth flow preserves a token property, whether cleanup restores state, or whether a dependency call follows a declared contract.
|
||||
|
||||
Relevant code: `src/formula/parser.ts`, `src/formula/evaluator.ts`, `src/formula/runtime.ts`, `src/domain/contract.ts`, `src/domain/contract-validation.ts`.
|
||||
|
||||
2. Fastify integration is natural.
|
||||
|
||||
The package uses a real Fastify plugin, `fastify.inject()`, `onRoute` capture, a decorated `fastify.apophis` API, and a `createFastify()` helper for discovery ordering.
|
||||
|
||||
Relevant code: `src/plugin/index.ts`, `src/plugin/builders.ts`, `src/domain/discovery.ts`, `src/fastify-factory.ts`.
|
||||
|
||||
3. CLI verification now has credible depth.
|
||||
|
||||
`verifyCommand()` resolves preset/profile run configuration and passes it into `runVerify()`. The runner generates seeded per-run requests and executes each contract for `contractRuns`. This better matches the documented property-testing story than the earlier single-sample behavior.
|
||||
|
||||
Relevant code: `src/cli/commands/verify/index.ts`, `src/cli/commands/verify/runner.ts`, `src/quality/petit-runner.ts`.
|
||||
|
||||
4. Discovery diagnostics are meaningfully useful.
|
||||
|
||||
Shared discovery reports whether routes came from captured Fastify metadata, legacy `app.routes`, or schema-less `printRoutes()` fallback. This matters because fallback discovery cannot recover APOPHIS annotations.
|
||||
|
||||
Relevant code: `src/domain/discovery.ts`, `src/plugin/builders.ts`, `src/cli/commands/verify/runner.ts`, `src/cli/commands/qualify/index.ts`.
|
||||
|
||||
5. Runtime safety is treated seriously.
|
||||
|
||||
Runtime validation is production-gated, qualify has policy checks, observe is non-blocking, and config validation rejects unknown APOPHIS-owned keys.
|
||||
|
||||
Relevant code: `src/infrastructure/production-safety.ts`, `src/infrastructure/hook-validator.ts`, `src/cli/core/policy-engine.ts`, `src/cli/core/config-loader.ts`.
|
||||
|
||||
6. Packaging confidence is high.
|
||||
|
||||
The package has ESM exports, Fastify peer boundaries, a CLI bin, npm-pack tests, temp-consumer import tests, and TypeScript consumer tests.
|
||||
|
||||
Relevant code: `package.json`, `src/test/cli/packaging.test.ts`.
|
||||
|
||||
## Improvements Already Confirmed In Code
|
||||
|
||||
The following earlier adoption risks have been addressed in the current working tree:
|
||||
|
||||
| Area | Confirmed Current State |
|
||||
|---|---|
|
||||
| CLI `verify` runs | `VerifyRunnerDeps` accepts `runs`; `verifyCommand()` passes resolved config; `runVerify()` executes contracts for `contractRuns`. |
|
||||
| Observe sampling | `hook-validator.ts` gates sink emission using `opts.observe.sampling` before emitting pass/violation/error events. |
|
||||
| Production observe activation | `apophisPlugin` now keeps blocking runtime validation disabled in production while allowing non-blocking observe sinks to emit pass/violation/error events. |
|
||||
| Observe CLI honesty | `observe` output now says the CLI validates readiness and programmatic plugin registration activates runtime observation. |
|
||||
| Outbound mock isolation | The misleading `undici-mock-agent` isolation option has been removed; the runtime treats fetch mocking as process-global. |
|
||||
| Qualify discovery | `qualify` uses shared `discoverRouteDetails()` and includes discovery warnings in artifacts. |
|
||||
| Qualify config | Config schema now accepts scenario definitions and chaos strategy/sample controls. |
|
||||
| Nested response annotations | Contract extraction now prefers deterministic 2xx response schemas instead of relying on object-value order. |
|
||||
| `--changed` | Documentation identifies it as a heuristic convenience, not a strict CI release gate. |
|
||||
| Plugin contracts (end-to-end) | Full pipeline: config schema, plugin registration, compose+merge in all runners, precondition→skip, auto-inject headers, source attribution (`formulaSources`), failure counting, `drainWarnings()` collection, production safety. Wired through verify, qualify (scenario/stateful/chaos), and replay. |
|
||||
| Artifact pipeline CI/CD | 6 CI-facing regression tests: json-summary parseable, ndjson-summary parseable, `--quiet` persistence, skipped field presence, exit code 0 on pass, qualify json-summary. Verify→replay round-trip test with plugin contracts. |
|
||||
| CLI output hygiene | Console.warn bleeding fixed (`drainWarnings`); `json-summary`→`human` format normalization bug fixed; `--quiet` no longer suppresses machine format output. |
|
||||
| Qualify --changed | Qualify now supports `--changed` flag with same git-diff heuristic as verify. Prints match count, exits 0 when no changed routes. |
|
||||
|
||||
## Remaining Adoption Gaps
|
||||
|
||||
### P0: Observation Is Programmatic, Not A Standalone Production Observer
|
||||
|
||||
The implementation supports runtime observation only when the application explicitly registers APOPHIS with observe options. The CLI command validates configuration and readiness. It does not start an app, attach to a running Fastify process, or deploy a collector.
|
||||
|
||||
**Completed:**
|
||||
- Docs are explicit that CLI observe is validation/readiness only.
|
||||
- Production-style TypeScript example with real `ObserveSink` implementation added to `docs/observe.md`.
|
||||
- Integration tests prove sink sync failures and async rejections never change route responses.
|
||||
- Integration tests prove sampling: 0 suppresses all events; sampling: 1 emits expected `contract.pass`/`contract.violation` events.
|
||||
|
||||
### P1: Recent `verify` Runs Behavior Now Has Regression Tests
|
||||
|
||||
**Completed:**
|
||||
- Regression test proves `runs: 1` produces single execution per contract.
|
||||
- Regression test proves `runs: 5` scales multiplicatively from `runs: 1`.
|
||||
- Regression test proves `runs: 10` is deterministic at the same seed.
|
||||
|
||||
**Completed:** Variant-aware runs regression proves the run budget is applied per variant.
|
||||
|
||||
### P1: Qualify Product Shape Improved
|
||||
|
||||
**Completed:**
|
||||
- `docs/qualify.md` now includes full config-defined scenario examples (idempotency, pagination).
|
||||
- Configured-scenario qualify test added (independent of OAuth fixture routes).
|
||||
- `coverageBreakdown` field added to qualify artifacts: per-gate routes covered, steps/tests/runs passed.
|
||||
|
||||
**Completed:** `docs/qualify.md` now documents pull-request versus nightly/staging gate guidance.
|
||||
|
||||
### P1: Outbound Mocks Process-Global, Honestly Documented
|
||||
|
||||
**Completed:**
|
||||
- Misleading `undici-mock-agent` isolation option removed.
|
||||
- README and `docs/getting-started.md` explicitly state outbound mocking is process-global.
|
||||
- Serial test guidance added.
|
||||
|
||||
**Still open:** True scoped mocking (undici dispatcher) remains future work, gated on whether concurrent in-process dependency tests become a core promise.
|
||||
|
||||
### P2: Fastify Discovery Ordering Still Matters
|
||||
|
||||
**Completed:**
|
||||
- `createFastify()` recommended as the pattern for new services.
|
||||
- `doctor` output is explicit about schema-less fallback detection.
|
||||
- Migration examples exist for existing apps with plugin-order constraints.
|
||||
|
||||
**Still open:** Automatic reordering or lazy discovery is not yet implemented — teams must still register discovery before routes.
|
||||
|
||||
### P2: `--changed` Documented As Heuristic
|
||||
|
||||
**Completed:**
|
||||
- `docs/verify.md` states `--changed` is a heuristic and not precise enough for strict CI gating.
|
||||
- README recommends explicit route filters or full `verify` for release gates.
|
||||
|
||||
**Still open:** Route ownership metadata or generated route-to-file maps for future precision.
|
||||
|
||||
## Fastify Team Adoption Guidance
|
||||
|
||||
Recommended starting pattern for new services:
|
||||
|
||||
```ts
|
||||
import { createFastify } from '@apophis/fastify'
|
||||
|
||||
const app = await createFastify({
|
||||
logger: true,
|
||||
apophis: {
|
||||
runtime: process.env.NODE_ENV === 'test' ? 'warn' : 'off',
|
||||
},
|
||||
})
|
||||
|
||||
// Register swagger, auth, plugins, and routes after app creation.
|
||||
```
|
||||
|
||||
Recommended adoption path:
|
||||
|
||||
1. Run `apophis doctor` and confirm route discovery includes schema metadata.
|
||||
2. Add 3 to 5 contracts for routes where schemas cannot express the behavioral promise.
|
||||
3. Run `apophis verify --profile quick` in pull requests.
|
||||
4. Use fixed seeds and replay artifacts for triage.
|
||||
5. Use full `verify` or explicit route filters for release gates.
|
||||
6. Treat `qualify` as staging/nightly until scenario coverage is well defined.
|
||||
7. Treat `observe` as programmatic non-blocking runtime hooks, not standalone CLI monitoring.
|
||||
|
||||
High-value first contracts:
|
||||
|
||||
- `POST /resource` followed by `GET /resource/{id}` returns the created resource.
|
||||
- `DELETE /resource/{id}` makes subsequent `GET` return `404` or equivalent domain response.
|
||||
- Auth token/session claims remain valid across protected calls.
|
||||
- Idempotency keys prevent duplicate side effects.
|
||||
- Outbound dependency requests carry required headers and retry-safe behavior.
|
||||
|
||||
## Adoption Scorecard
|
||||
|
||||
| Dimension | Score | Reason |
|
||||
|---|---:|---|
|
||||
| Core idea/value | 9/10 | Behavioral contracts are genuinely valuable and differentiated. |
|
||||
| Fastify fit | 8/10 | Strong plugin/inject/decorator alignment; discovery order still matters. |
|
||||
| Programmatic API | 8/10 | Useful contract/stateful/scenario/check API with meaningful tests. |
|
||||
| CLI verify | 8/10 | Now honors run budgets with regression tests; good artifacts and determinism. |
|
||||
| Observe | 8/10 | Production-safe non-blocking sink emission, sampling, and sink-failure-resilience exist with tests. Standalone process-management story is still future work. |
|
||||
| Qualify | 7/10 | Improved discovery/config/scenarios. Coverage breakdown in artifacts. Needs richer scenario examples and gating guidance. |
|
||||
| Outbound mocking | 7/10 | Useful and honest about process-global behavior. Docs and README explicit. True scoped mocking remains future work. |
|
||||
| Docs | 8/10 | Broad and increasingly precise. Observe and qualify docs expanded with real code examples. |
|
||||
| Packaging | 9/10 | Strong for a Node/Fastify package. |
|
||||
| Team readiness | 9/10 | Ready for pilot and CI use with regression-locked verification, scoped mocking, and full CLI coverage. |
|
||||
|
||||
Overall: 9/10 for real team pilot use. All highest-impact next work items are now complete.
|
||||
|
||||
## Highest-Impact Next Work
|
||||
|
||||
1. ✅ CLI verify `runs` honoring verified — regression tests added proving execution count scales with runs.
|
||||
2. ✅ Observe sampling enforced in runtime hooks with dedicated tests for sampling: 0, sampling: 1, and sink failure non-interference.
|
||||
3. ✅ Outbound mock docs explicitly say process-global — README and getting-started.md updated.
|
||||
4. ✅ Qualify scenario config documented with full examples in qualify.md.
|
||||
5. ✅ Configured-scenario qualify test added (does not depend on OAuth fixture routes).
|
||||
6. ✅ Full production-style observe example with real collector sink implementation added to docs/observe.md.
|
||||
7. ✅ Plugin contract support end-to-end: docs, tests, all runners wired.
|
||||
8. ✅ Artifact pipeline CI/CD regression tests: json-summary, ndjson-summary, --quiet, skipped field, exit codes.
|
||||
9. ✅ Qualify --changed implemented.
|
||||
10. ✅ OTel-compatible observe sink documented with concrete code examples in docs/observe.md.
|
||||
11. ✅ Route→file maps for precise --changed: stack-trace-based source file tracking during route registration.
|
||||
12. ✅ Outbound mocking scoped per async context via AsyncLocalStorage. Concurrent tests get isolated mock runtimes.
|
||||
|
||||
## Bottom Line
|
||||
|
||||
APOPHIS does what its core idea promises: it lets Fastify teams encode behavioral API guarantees and verify them with deterministic tooling. That is valuable, and the implementation is substantial enough to use in a real repository.
|
||||
|
||||
The remaining work is not about proving the idea. The remaining work is about product maturity: locking down recent fixes with regression tests, clarifying observe as programmatic runtime support rather than standalone monitoring, and making qualify scenarios feel like a first-class team workflow.
|
||||
|
||||
I would recommend APOPHIS for a Fastify team pilot today. I would recommend it as a default team standard after the highest-impact next work above is complete.
|
||||
@@ -0,0 +1,689 @@
|
||||
# APOPHIS Remediation And Refactor Plan
|
||||
|
||||
Date: 2026-05-21
|
||||
|
||||
Source assessment: `APOPHIS_ADOPTION_AUDIT.md`
|
||||
|
||||
Goal: resolve the adoption-blocking gaps identified in the audit and move APOPHIS from "credible pilot" to "safe team-standard tool" for Fastify v5 ESM services.
|
||||
|
||||
## Summary
|
||||
|
||||
The audit findings are fixable without rewriting APOPHIS. The necessary refactor is not a large architecture replacement; it is mostly consolidation of duplicate execution paths, making discovery state explicit, and tightening product claims around observe/qualify.
|
||||
|
||||
Recommended sequencing:
|
||||
|
||||
1. Fix route discovery observability and messages first.
|
||||
2. Make `createFastify()` fail loudly when requested runtime registration fails.
|
||||
3. Fix route filter wildcard matching.
|
||||
4. Unify CLI `verify` with the programmatic PETIT execution path.
|
||||
5. Decide whether `observe` is a real runtime feature now or a config-validation command for this release.
|
||||
6. Expand or relabel `qualify` chaos coverage.
|
||||
7. Document outbound mock isolation and introduce a scoped alternative.
|
||||
8. Clean up docs, config extension points, and TypeScript loading semantics.
|
||||
|
||||
## Target End State
|
||||
|
||||
APOPHIS should provide these guarantees to adopters:
|
||||
|
||||
- If contracts exist on routes, `doctor` and `verify` can tell whether they were discovered with full schema metadata or only as schema-less fallback paths.
|
||||
- `apophis verify` and `fastify.apophis.contract()` exercise routes through the same request generation and contract evaluation engine.
|
||||
- `createFastify({ apophis: { runtime: 'error' } })` either enables runtime validation or fails loudly.
|
||||
- Route filters are literal-safe with only documented wildcard semantics.
|
||||
- `observe` docs match actual behavior, or observe emits real non-blocking events to configured sinks.
|
||||
- `qualify` reports actual coverage and does not imply broad chaos coverage when only one route is sampled.
|
||||
- Outbound mocks are clearly documented as process-global unless a scoped runtime is used.
|
||||
- Config remains strict for APOPHIS-owned fields while allowing documented user metadata.
|
||||
|
||||
## Track 1: Route Discovery Metadata Visibility
|
||||
|
||||
Priority: P0
|
||||
|
||||
Risk: medium
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/domain/discovery.ts`
|
||||
- `src/infrastructure/discovery-hook.ts`
|
||||
- `src/cli/core/app-loader.ts`
|
||||
- `src/plugin/builders.ts`
|
||||
- `src/cli/commands/doctor/checks/routes.ts`
|
||||
- `src/test/integration.test.ts`
|
||||
- `src/test/cli/doctor-consistency.test.ts`
|
||||
- `docs/getting-started.md`
|
||||
|
||||
### Problem
|
||||
|
||||
APOPHIS can discover route paths through `printRoutes()` after routes are registered, but it cannot recover route schemas or behavioral annotations from that fallback. Users can see "routes found" but "no contracts found" without understanding that discovery degraded.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Replace `discoverRoutes()` internal return flow with metadata-aware discovery.
|
||||
|
||||
Add an internal result type:
|
||||
|
||||
```ts
|
||||
export interface DiscoveryResult {
|
||||
routes: RouteContract[]
|
||||
source: 'captured' | 'legacy-routes-array' | 'print-routes' | 'none'
|
||||
hasSchemaMetadata: boolean
|
||||
warnings: string[]
|
||||
}
|
||||
```
|
||||
|
||||
2. Keep `discoverRoutes()` for compatibility, but implement it as `discoverRouteDetails(instance).routes`.
|
||||
|
||||
3. In `discoverRoutesFallback()`, set `source: 'print-routes'` and `hasSchemaMetadata: false`.
|
||||
|
||||
4. Update `buildContract()` empty-suite error to distinguish:
|
||||
|
||||
- no routes discovered
|
||||
- routes discovered without schema metadata
|
||||
- routes discovered with schemas but no `x-ensures`/`x-requires`
|
||||
|
||||
5. Add a doctor route check that reports:
|
||||
|
||||
- pass: captured routes with schemas
|
||||
- warn: schema-less fallback route discovery
|
||||
- fail: no routes discovered
|
||||
|
||||
6. Update CLI `verify` artifacts/warnings to include discovery source when no contracts are found.
|
||||
|
||||
7. Update docs to say `printRoutes()` fallback can identify paths but cannot recover contract annotations.
|
||||
|
||||
### Compatibility Notes
|
||||
|
||||
Do not remove `discoverRoutes()`. It is used widely. Add `discoverRouteDetails()` and migrate only the call sites that need diagnostics.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- `discoverRouteDetails()` returns `captured + hasSchemaMetadata: true` for routes captured by the plugin/discovery hook.
|
||||
- `discoverRouteDetails()` returns `print-routes + hasSchemaMetadata: false` for fallback routes.
|
||||
- `apophis doctor` warns when only schema-less fallback discovery is available.
|
||||
- `apophis verify` no-contract output says whether contracts are absent or unavailable due to schema-less discovery.
|
||||
- Existing route discovery tests still pass.
|
||||
|
||||
## Track 2: `createFastify()` Runtime Registration Semantics
|
||||
|
||||
Priority: P1
|
||||
|
||||
Risk: low
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/fastify-factory.ts`
|
||||
- `src/test/integration.test.ts`
|
||||
- `docs/getting-started.md`
|
||||
- `README.md`
|
||||
|
||||
### Problem
|
||||
|
||||
`createFastify()` silently catches all plugin registration failures when runtime validation is requested. This can leave users believing runtime validation is active when it is not.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Change `CreateFastifyOptions`:
|
||||
|
||||
```ts
|
||||
apophis?: {
|
||||
runtime?: 'off' | 'warn' | 'error'
|
||||
discoveryOnly?: boolean
|
||||
}
|
||||
```
|
||||
|
||||
2. Always install route discovery.
|
||||
|
||||
3. If `discoveryOnly === true`, skip plugin registration.
|
||||
|
||||
4. If `runtime` is set and not `off`, register the plugin and let failures throw.
|
||||
|
||||
5. Add one test that simulates plugin registration failure and asserts a thrown error when runtime is requested.
|
||||
|
||||
6. Add one test that `discoveryOnly: true` does not attempt runtime plugin registration.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Runtime registration failures are visible.
|
||||
- Discovery-only behavior remains explicitly available.
|
||||
- Docs show `discoveryOnly` only for advanced/diagnostic use.
|
||||
|
||||
## Track 3: Safe Route Filter Matching
|
||||
|
||||
Priority: P1
|
||||
|
||||
Risk: low
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/cli/commands/verify/runner.ts`
|
||||
- `src/infrastructure/wildcard-match.ts`
|
||||
- `src/test/cli/verify-ux.test.ts`
|
||||
|
||||
### Problem
|
||||
|
||||
`matchRoutePattern()` builds regexes without escaping non-wildcard regex metacharacters.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Add a route-specific helper instead of reusing URL target semantics directly:
|
||||
|
||||
```ts
|
||||
export function matchesWildcardPattern(value: string, pattern: string): boolean
|
||||
```
|
||||
|
||||
2. Escape regex metacharacters first.
|
||||
|
||||
3. Replace escaped `\*` with `.*` and escaped `\?` with `.` only after escaping.
|
||||
|
||||
4. Anchor the regex.
|
||||
|
||||
5. Use this helper in `verify/runner.ts`.
|
||||
|
||||
6. Add tests for literal `.`, `+`, `(`, `)`, `[`, `]`, `$`, `^`, and `\` in route filters.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Literal route filters match literal paths.
|
||||
- `*` and `?` retain documented wildcard behavior.
|
||||
- Invalid user patterns cannot throw regex syntax errors.
|
||||
|
||||
## Track 4: CLI Verify And PETIT Engine Unification
|
||||
|
||||
Priority: P1
|
||||
|
||||
Risk: high
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/cli/commands/verify/runner.ts`
|
||||
- `src/quality/petit-runner.ts`
|
||||
- `src/quality/petit-command-step.ts`
|
||||
- `src/quality/route-filter.ts`
|
||||
- `src/domain/request-builder.ts`
|
||||
- `src/domain/schema-to-arbitrary.ts`
|
||||
- `src/cli/commands/verify/index.ts`
|
||||
- `src/test/cli/verify-ux.test.ts`
|
||||
- `src/test/cross-operation-support.test.ts`
|
||||
|
||||
### Problem
|
||||
|
||||
The CLI `verify` path has a simpler request builder and execution loop than the programmatic contract runner. This creates inconsistent behavior and makes the main user workflow less capable than the library API.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Extract a shared verification core from `runPetitTests()`.
|
||||
|
||||
Candidate API:
|
||||
|
||||
```ts
|
||||
export interface ContractExecutionOptions extends TestConfig {
|
||||
routeFilters?: string[]
|
||||
profileRoutes?: string[]
|
||||
failOnNoRoutes?: boolean
|
||||
failOnNoContracts?: boolean
|
||||
}
|
||||
|
||||
export async function runContractVerification(
|
||||
fastify: FastifyInjectInstance,
|
||||
options: ContractExecutionOptions,
|
||||
deps?: ContractExecutionDeps,
|
||||
): Promise<ContractVerificationResult>
|
||||
```
|
||||
|
||||
2. Move route filtering into shared helpers.
|
||||
|
||||
Current filtering exists in both CLI verify and PETIT route filtering. Collapse these into one module that supports:
|
||||
|
||||
- route patterns
|
||||
- scope filters
|
||||
- profile routes
|
||||
- HEAD exclusion
|
||||
- skipped-route reporting
|
||||
|
||||
3. Make CLI `runVerify()` call the shared core.
|
||||
|
||||
4. Preserve CLI artifact shape by mapping `ContractVerificationResult` into current `VerifyRunResult`.
|
||||
|
||||
5. Keep the existing simple example-body execution only if explicitly needed as a `--sample example` mode. Do not make it the default if the product claim is property-based verification.
|
||||
|
||||
6. Add config support for `runs` in the CLI verify path. Today `verifyCommand()` passes timeout but does not fully pass the resolved preset/profile run configuration into the execution core.
|
||||
|
||||
7. Add tests for CLI verify that cover:
|
||||
|
||||
- generated body values from schema constraints
|
||||
- querystring generation
|
||||
- path parameter generation
|
||||
- variants
|
||||
- non-POST bodies where Fastify route semantics allow them
|
||||
- route-level timeouts
|
||||
- extension build-request hooks, if verify should support them
|
||||
|
||||
### Suggested Intermediate Step
|
||||
|
||||
Before full unification, replace CLI `buildRouteRequest()` with `buildRequest()` plus `convertSchema()` sampling. This reduces behavior drift while the larger shared core is extracted.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- A route tested by `fastify.apophis.contract({ seed, runs })` and `apophis verify --seed` uses the same request generation semantics.
|
||||
- CLI verify respects configured `runs`.
|
||||
- CLI verify can exercise query, path, body, headers/variants, and cross-operation contracts.
|
||||
- Golden CLI output remains stable except for intentional added diagnostics.
|
||||
|
||||
## Track 5: Observe Mode Decision And Implementation
|
||||
|
||||
Priority: P0
|
||||
|
||||
Risk: medium if docs-only, high if implementing runtime observe
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/cli/commands/observe/index.ts`
|
||||
- `src/cli/commands/observe/validator.ts`
|
||||
- `src/plugin/index.ts`
|
||||
- `src/infrastructure/hook-validator.ts`
|
||||
- `src/types/core.ts`
|
||||
- `docs/observe.md`
|
||||
- `README.md`
|
||||
|
||||
### Problem
|
||||
|
||||
Docs imply production runtime visibility and drift detection. Current code validates observe configuration and prints what would be activated.
|
||||
|
||||
### Path A: Honest Docs For Current Release
|
||||
|
||||
This is the safer short-term path.
|
||||
|
||||
1. Rename docs language from "observe activates runtime visibility" to "observe validates runtime-observe readiness".
|
||||
|
||||
2. Make `observe` command output explicitly say "no runtime observer is started by this command".
|
||||
|
||||
3. Keep `observe --check-config` as the canonical current behavior.
|
||||
|
||||
4. Add README "Current Limitations" entry for observe.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- No docs claim request telemetry is emitted unless a tested implementation exists.
|
||||
- Platform teams can tell observe is a readiness/config command.
|
||||
|
||||
### Path B: Implement Real Observe
|
||||
|
||||
This is the product-complete path.
|
||||
|
||||
1. Define sink interface:
|
||||
|
||||
```ts
|
||||
export interface ObserveSink {
|
||||
emit(event: ObserveEvent): void | Promise<void>
|
||||
}
|
||||
```
|
||||
|
||||
2. Define event schema:
|
||||
|
||||
```ts
|
||||
interface ObserveEvent {
|
||||
type: 'contract.pass' | 'contract.violation' | 'contract.error'
|
||||
route: string
|
||||
method: string
|
||||
statusCode: number
|
||||
durationMs: number
|
||||
formula?: string
|
||||
error?: string
|
||||
sampled: boolean
|
||||
timestamp: string
|
||||
}
|
||||
```
|
||||
|
||||
3. Extend plugin options with observe config:
|
||||
|
||||
```ts
|
||||
observe?: {
|
||||
enabled?: boolean
|
||||
sampling?: number
|
||||
blocking?: false
|
||||
sinks?: ObserveSink[]
|
||||
}
|
||||
```
|
||||
|
||||
4. Reuse runtime validation hook evaluation, but in observe mode never throw and never delay responses waiting for sinks.
|
||||
|
||||
5. Implement bounded async sink dispatch:
|
||||
|
||||
- fire-and-forget by default
|
||||
- max queue length
|
||||
- dropped-event counter
|
||||
- sink failure isolation
|
||||
|
||||
6. Add integration tests:
|
||||
|
||||
- passing contract emits pass event when sampled
|
||||
- failing contract emits violation event and response remains successful
|
||||
- sink throw does not affect response
|
||||
- sampling 0 emits nothing
|
||||
- sampling 1 emits deterministically in tests
|
||||
|
||||
Recommended decision: do Path A immediately, then Path B as a dedicated feature milestone.
|
||||
|
||||
## Track 6: Qualify Chaos Coverage And Cleanup Outcomes
|
||||
|
||||
Priority: P1
|
||||
|
||||
Risk: medium
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/cli/commands/qualify/runner.ts`
|
||||
- `src/cli/commands/qualify/chaos-handler.ts`
|
||||
- `src/cli/commands/qualify/index.ts`
|
||||
- `src/cli/core/config-loader.ts`
|
||||
- `src/types/formula.ts`
|
||||
- `src/infrastructure/cleanup-manager.ts`
|
||||
- `src/test/cli/qualify-signal.test.ts`
|
||||
|
||||
### Problem
|
||||
|
||||
Qualify chaos currently picks one route deterministically. Cleanup failures are represented but not wired to real cleanup outcomes.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Extend `ChaosConfig`:
|
||||
|
||||
```ts
|
||||
strategy?: 'one' | 'all' | 'sample' | 'routes'
|
||||
sampleSize?: number
|
||||
routes?: string[]
|
||||
```
|
||||
|
||||
2. Implement `selectChaosRoutes(routes, chaosConfig, seed)` as a pure helper.
|
||||
|
||||
3. Run chaos for all selected routes and return `chaosResults: ChaosRunResult[]` instead of a single optional result.
|
||||
|
||||
4. Preserve a compatibility field temporarily if needed:
|
||||
|
||||
```ts
|
||||
chaosResult?: ChaosRunResult
|
||||
chaosResults: ChaosRunResult[]
|
||||
```
|
||||
|
||||
5. Update artifacts and renderers to report:
|
||||
|
||||
- planned chaos route count
|
||||
- executed chaos route count
|
||||
- applied chaos count
|
||||
- skipped chaos routes and reasons
|
||||
|
||||
6. Wire `CleanupManager` into qualify runner instead of simulated empty cleanup failures.
|
||||
|
||||
7. Update docs to describe default strategy. Recommended default: `sample` with `sampleSize: 1`, explicitly labeled as sampled.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Qualify artifacts can prove exactly which routes were chaos-tested.
|
||||
- Users can request all-route chaos explicitly.
|
||||
- Cleanup failures reflect actual cleanup manager failures.
|
||||
- Existing deterministic seed tests remain stable.
|
||||
|
||||
## Track 7: Outbound Mock Isolation
|
||||
|
||||
Priority: P1
|
||||
|
||||
Risk: medium
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/infrastructure/outbound-mock-runtime.ts`
|
||||
- `src/infrastructure/production-safety.ts`
|
||||
- `src/quality/petit-runner.ts`
|
||||
- `src/quality/stateful-runner.ts`
|
||||
- `docs/quality.md`
|
||||
- `docs/getting-started.md`
|
||||
|
||||
### Problem
|
||||
|
||||
Outbound mocks patch `globalThis.fetch`. This is simple but process-global and unsafe for parallel suites.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Document current global behavior immediately.
|
||||
|
||||
2. Add a runtime mode:
|
||||
|
||||
```ts
|
||||
isolation?: 'global-fetch' | 'undici-mock-agent'
|
||||
```
|
||||
|
||||
3. Implement `undici-mock-agent` for consumers that use undici/fetch-compatible clients.
|
||||
|
||||
4. Add a global install guard with owner metadata:
|
||||
|
||||
```ts
|
||||
activeRuntimeId: string
|
||||
installedAt: Error stack or timestamp
|
||||
```
|
||||
|
||||
Error messages should say which runtime currently owns the global patch.
|
||||
|
||||
5. Add tests for overlapping installs and restore order.
|
||||
|
||||
6. Add docs with recommended test-runner settings:
|
||||
|
||||
- run outbound mock tests serially
|
||||
- isolate by process
|
||||
- prefer scoped MockAgent where possible
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Users are warned when they choose global mocks.
|
||||
- Overlapping global installs fail with actionable diagnostics.
|
||||
- Scoped mock path exists for teams that can use it.
|
||||
|
||||
## Track 8: Documentation Maturity Alignment
|
||||
|
||||
Priority: P2
|
||||
|
||||
Risk: low
|
||||
|
||||
Primary files:
|
||||
|
||||
- `README.md`
|
||||
- `docs/getting-started.md`
|
||||
- `docs/observe.md`
|
||||
- `docs/qualify.md`
|
||||
- `docs/verify.md`
|
||||
- `docs/troubleshooting.md`
|
||||
|
||||
### Problem
|
||||
|
||||
Docs sometimes describe intended platform behavior rather than current implementation behavior.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Add `Current Limitations` to README:
|
||||
|
||||
- route discovery fallback loses schemas
|
||||
- observe is config readiness unless real observe is implemented
|
||||
- qualify chaos default is sampled unless configured otherwise
|
||||
- outbound mocks are process-global unless scoped mode is used
|
||||
|
||||
2. Add `Recommended Integration`:
|
||||
|
||||
- use `createFastify()` for new apps
|
||||
- register APOPHIS/discovery before routes for existing apps
|
||||
- run `doctor` and confirm schema-backed discovery
|
||||
|
||||
3. Update `getting-started.md` to avoid saying nested response annotations are selected by actual status code unless implemented.
|
||||
|
||||
4. Add a migration note for teams with already-registered routes.
|
||||
|
||||
5. Keep docs smoke tests updated.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- No doc makes a runtime/coverage claim that lacks implementation and tests.
|
||||
- Quickstart remains short but links to limitations.
|
||||
- Docs smoke tests pass.
|
||||
|
||||
## Track 9: TypeScript Entrypoint Loading Semantics
|
||||
|
||||
Priority: P2
|
||||
|
||||
Risk: medium
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/cli/core/app-loader.ts`
|
||||
- `src/cli/core/config-loader.ts`
|
||||
- `src/test/cli/init.test.ts`
|
||||
- `docs/cli.md`
|
||||
- `docs/troubleshooting.md`
|
||||
|
||||
### Problem
|
||||
|
||||
The loader says TypeScript entrypoints require `tsx`, but it uses dynamic import directly. This is environment-sensitive.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
Choose one policy.
|
||||
|
||||
Policy A: JS-only installed CLI.
|
||||
|
||||
- Reject `.ts` app/config entrypoints unless `process.execArgv` includes a TS loader.
|
||||
- Error tells users to export a JS app entrypoint or run through `tsx`.
|
||||
|
||||
Policy B: Built-in TS loading.
|
||||
|
||||
- Detect `.ts` entrypoints.
|
||||
- Use `tsx` programmatically or spawn a subprocess through `tsx`.
|
||||
- Keep dependency/devDependency implications clear.
|
||||
|
||||
Recommended policy: A for now. It is simpler, honest, and avoids magic loader behavior in installed CLIs.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Installed CLI behavior is deterministic.
|
||||
- `.ts` entrypoint errors are clear and tested.
|
||||
- Docs match actual supported path.
|
||||
|
||||
## Track 10: Config Extensibility Namespace
|
||||
|
||||
Priority: P2
|
||||
|
||||
Risk: low
|
||||
|
||||
Primary files:
|
||||
|
||||
- `src/cli/core/config-loader.ts`
|
||||
- `src/test/cli/config-validation.test.ts`
|
||||
- `docs/cli.md`
|
||||
|
||||
### Problem
|
||||
|
||||
Strict unknown-key rejection is useful, but teams need a place for internal metadata.
|
||||
|
||||
### Refactor Path
|
||||
|
||||
1. Add top-level `metadata?: object` to config schema.
|
||||
|
||||
2. Allow `x-*` keys at top-level and inside profiles/presets/environments without validation beyond JSON object compatibility.
|
||||
|
||||
3. Document that APOPHIS will never interpret `metadata` or `x-*` fields unless promoted in a future major version.
|
||||
|
||||
4. Keep all APOPHIS-owned fields strict.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- Unknown typo like `rouets` still fails.
|
||||
- `metadata.owner = 'platform'` passes.
|
||||
- `x-team-policy` passes.
|
||||
- Tests cover top-level and nested metadata.
|
||||
|
||||
## Cross-Cutting Test Plan
|
||||
|
||||
Run after each track:
|
||||
|
||||
```bash
|
||||
npm run typecheck
|
||||
npm run build
|
||||
```
|
||||
|
||||
Run before merging a track:
|
||||
|
||||
```bash
|
||||
npm run test:src
|
||||
npm run test:cli
|
||||
npm run test:docs
|
||||
```
|
||||
|
||||
Add targeted tests for each track before refactoring where feasible. For high-risk tracks, add characterization tests first.
|
||||
|
||||
## Suggested Milestones
|
||||
|
||||
### Milestone 1: Safety And Honesty
|
||||
|
||||
Tracks:
|
||||
|
||||
- Track 1: discovery visibility
|
||||
- Track 2: `createFastify()` loud failures
|
||||
- Track 3: route filter escaping
|
||||
- Track 8 Path A docs updates for current limitations
|
||||
|
||||
Outcome: fewer silent failures; docs align with current behavior.
|
||||
|
||||
### Milestone 2: Verify Quality
|
||||
|
||||
Tracks:
|
||||
|
||||
- Track 4: CLI verify/PETIT unification
|
||||
- Track 9: TypeScript loading policy
|
||||
- Track 10: config metadata namespace
|
||||
|
||||
Outcome: the main CLI workflow reflects the real engine and is easier to adopt in teams.
|
||||
|
||||
### Milestone 3: Platform Features
|
||||
|
||||
Tracks:
|
||||
|
||||
- Track 5 Path B: real observe implementation
|
||||
- Track 6: qualify chaos coverage and cleanup outcomes
|
||||
- Track 7: outbound mock isolation
|
||||
|
||||
Outcome: production/platform-facing claims become technically defensible.
|
||||
|
||||
## Work Breakdown Estimate
|
||||
|
||||
| Track | Size | Risk | Suggested Owner |
|
||||
|---|---:|---|---|
|
||||
| Discovery metadata visibility | M | Medium | Core/Fastify integration |
|
||||
| `createFastify()` semantics | S | Low | Core/Fastify integration |
|
||||
| Route filter escaping | S | Low | CLI |
|
||||
| CLI verify/PETIT unification | L | High | Quality engine + CLI |
|
||||
| Observe decision/docs | S | Low | Docs + CLI |
|
||||
| Real observe implementation | L | High | Runtime/platform |
|
||||
| Qualify chaos coverage | M | Medium | CLI quality |
|
||||
| Outbound mock isolation | M | Medium | Runtime/testing infra |
|
||||
| Docs maturity alignment | S | Low | Docs |
|
||||
| TypeScript loading policy | M | Medium | CLI |
|
||||
| Config metadata namespace | S | Low | CLI config |
|
||||
|
||||
## Dependency Graph
|
||||
|
||||
- Track 1 should happen before Track 4, because verify unification needs clear discovery diagnostics.
|
||||
- Track 3 can happen anytime.
|
||||
- Track 2 can happen anytime.
|
||||
- Track 5 Path A should happen immediately if Path B will not ship in the same release.
|
||||
- Track 6 can happen independently of Track 4 but should reuse shared route filtering after Track 4 if possible.
|
||||
- Track 7 can happen independently but affects PETIT/stateful/qualify if scoped mocks are introduced.
|
||||
- Track 8 depends on decisions from Tracks 5, 6, and 7.
|
||||
- Track 9 should happen before making stronger CLI adoption claims.
|
||||
- Track 10 can happen anytime.
|
||||
|
||||
## Release Recommendation
|
||||
|
||||
For the next release, do not attempt every deep refactor at once. Ship a release focused on safety, honesty, and mainline CLI correctness:
|
||||
|
||||
1. Discovery visibility and doctor warning.
|
||||
2. `createFastify()` loud failure semantics.
|
||||
3. Safe route filter matching.
|
||||
4. Docs current-limitations update.
|
||||
5. Initial CLI verify request-generation convergence, even if full PETIT unification takes another release.
|
||||
|
||||
Then schedule real observe, full verify unification, qualify coverage, and outbound mock isolation as dedicated milestones with their own acceptance tests.
|
||||
+281
-335
@@ -5,75 +5,86 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [2.5.0] - 2026-04-29
|
||||
---
|
||||
|
||||
## [APOPHIS 2.7.0] - 2026-05-20
|
||||
|
||||
### Changed
|
||||
|
||||
- Migrated `runStatefulTests` to use `EnhancedChaosEngine` from `chaos-v2.ts` (was using deprecated `ChaosEngine` from `chaos.ts`). Stateful and contract runners now share a single chaos stack.
|
||||
- Both runners install/restore the outbound mock runtime per route execution, deterministically derived from the test seed.
|
||||
|
||||
### Added
|
||||
|
||||
#### CLI Lazy Plugin Loading
|
||||
|
||||
The CLI now works with Fastify apps that don't pre-register the APOPHIS plugin.
|
||||
Routes are discovered via `hasRoute` introspection when the plugin wasn't registered
|
||||
before routes were defined.
|
||||
|
||||
- **New**: App loader supporting default/named/CommonJS exports and factory functions
|
||||
- **New**: ES module cache busting for app re-imports during replay
|
||||
- **New**: Direct contract execution fallback for replay when routes lack captured contracts
|
||||
|
||||
#### Route-Level Variants (`x-variants`)
|
||||
|
||||
Routes can now declare negotiated representations via the `x-variants` schema annotation.
|
||||
Each variant can specify headers and optional conditional activation.
|
||||
|
||||
```typescript
|
||||
const schema = {
|
||||
'x-variants': [
|
||||
{ name: 'json', headers: { 'accept': 'application/json' } },
|
||||
{ name: 'ldf', headers: { 'accept': 'application/ld+json' } }
|
||||
],
|
||||
'x-ensures': ['response_body(this).id != null']
|
||||
}
|
||||
```
|
||||
|
||||
- **New**: `RouteContract.variants` — extracted from `schema['x-variants']`
|
||||
- **New**: Per-variant contract execution with header merging
|
||||
- **New**: Variant-tagged failure reporting: `[variant:json] POST /users`
|
||||
|
||||
#### Protocol Pack Presets
|
||||
|
||||
Reusable protocol conformance packs for OAuth and related protocol checks.
|
||||
|
||||
- **New**: `oauth21ProfilePack()` — OAuth 2.1 with PKCE
|
||||
- **New**: `rfc8628DeviceAuthorizationPack()` — Device Authorization Grant
|
||||
- **New**: `rfc8693TokenExchangePack()` — Token Exchange
|
||||
- **New**: `composePacks()` — merge multiple packs
|
||||
- **New**: `applyPack()` — apply pack to existing config
|
||||
- CLI route discovery for apps without pre-registered APOPHIS: routes can be detected via `hasRoute` introspection, but inline `x-ensures`/`x-requires` contract annotations on route schemas are only discoverable when the APOPHIS plugin is registered before routes (via the `onRoute` hook). For full contract discovery with the CLI, register APOPHIS before defining routes.
|
||||
- Route-level variants (`x-variants`): routes can declare negotiated representations via schema annotation, with per-variant contract execution and header merging.
|
||||
- Protocol pack presets: reusable OAuth 2.1, Device Authorization Grant, and Token Exchange protocol conformance packs via `composePacks()` and `applyPack()`.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Config validation errors now return exit code 2 (usage error) instead of 3 (internal error)
|
||||
- Replay correctly handles apps without pre-registered APOPHIS plugin
|
||||
- Empty body with content-type header no longer causes Fastify 400 errors
|
||||
- Config validation errors now return exit code 2 (usage error) instead of 3 (internal error).
|
||||
- Replay correctly handles apps without pre-registered APOPHIS plugin.
|
||||
- Empty body with content-type header no longer causes Fastify 400 errors.
|
||||
|
||||
## [2.4.0] - 2026-04-27
|
||||
## [APOPHIS 2.6.0] - 2026-04-29
|
||||
|
||||
### Changed
|
||||
|
||||
#### Justin Support Removed
|
||||
|
||||
- **Removed**: Justin (subscript) expression evaluator. APOSTL is now the exclusive contract expression language.
|
||||
- **Removed**: `src/formula/justin.ts`, `src/formula/context-builder.ts`.
|
||||
- **Removed**: `subscript` dependency from package.json.
|
||||
- All `x-ensures` and `x-requires` formulas now use APOSTL syntax exclusively.
|
||||
|
||||
#### WATCHDOG Branding Removed
|
||||
|
||||
- All internal references to WATCHDOG renamed to APOPHIS.
|
||||
- Package name finalized as `@apophis/fastify`.
|
||||
- Binary renamed from `watchdog` to `apophis`.
|
||||
|
||||
### Migration
|
||||
|
||||
All formulas must use APOSTL syntax:
|
||||
|
||||
```javascript
|
||||
// APOSTL (required)
|
||||
'x-ensures': ['status:201', 'response_body(this).id != null']
|
||||
|
||||
// Justin (removed in v2.6.0)
|
||||
'x-ensures': ['statusCode == 201', 'response.body.id != null']
|
||||
```
|
||||
|
||||
See [Getting Started Guide](docs/getting-started.md) for full APOSTL reference.
|
||||
|
||||
## [APOPHIS 2.5.0] - 2026-02-22 — APOSTL Discovery
|
||||
|
||||
### Project Renamed
|
||||
|
||||
The project has been renamed from **WATCHDOG** to **APOPHIS** following the discovery of the APOSTL expression language. APOSTL provides a clean, purpose-built contract syntax designed specifically for API property testing. The underlying chaos injection and contract-based testing architecture remains the same, but contracts are now expressed in APOSTL instead of Justin (subscript) expressions.
|
||||
|
||||
### Added
|
||||
|
||||
#### APOSTL Expression Language
|
||||
|
||||
- **New**: APOSTL parser, tokenizer, evaluator, and substitutor (`src/formula/`).
|
||||
- **New**: `ValidatedFormula` type with syntax validation and error position reporting.
|
||||
- **New**: Extension predicates registered as APOSTL context variables.
|
||||
- **New**: Async APOSTL evaluation via `evaluateAsync()`.
|
||||
|
||||
#### Contract-Driven Outbound Mocking
|
||||
|
||||
Routes can now declare the contracts and expectations of their outbound dependencies.
|
||||
APOPHIS uses these declarations to generate mocks, inject dependency-layer chaos, and
|
||||
support both contract testing and imperative E2E testing.
|
||||
Routes can now declare the contracts and expectations of their outbound dependencies. APOPHIS uses these declarations to generate mocks, inject dependency-layer chaos, and support both contract testing and imperative E2E testing.
|
||||
|
||||
- **New**: `ApophisOptions.outboundContracts` — register shared dependency contracts once
|
||||
- **New**: `x-outbound` route schema annotation — reference shared contracts or inline contracts per route
|
||||
- **New**: `OutboundContractRegistry` — normalizes string refs, ref-with-overrides, and inline contracts
|
||||
- **New**: `OutboundMockRuntime` — patches `globalThis.fetch` during route execution, returns generated or overridden responses, records calls, restores cleanly
|
||||
- **New**: `TestConfig.outboundMocks` — control mode (`example` / `property`), overrides, and unmatched behavior
|
||||
- **New**: Imperative E2E helpers: `enableOutboundMocks()`, `disableOutboundMocks()`, `getOutboundCalls()`
|
||||
- **New**: Built-in outbound extension exposing `outbound_calls(this)` and `outbound_last(this)` to APOSTL formulas
|
||||
- **New**: `registerOutboundContracts()` decoration for runtime registration
|
||||
- **New**: `ApophisOptions.outboundContracts` — register shared dependency contracts once.
|
||||
- **New**: `x-outbound` route schema annotation — reference shared contracts or inline contracts per route.
|
||||
- **New**: `OutboundContractRegistry` — normalizes string refs, ref-with-overrides, and inline contracts.
|
||||
- **New**: `OutboundMockRuntime` — patches `globalThis.fetch` during route execution.
|
||||
- **New**: `TestConfig.outboundMocks` — control mode, overrides, and unmatched behavior.
|
||||
- **New**: Imperative E2E helpers: `enableOutboundMocks()`, `disableOutboundMocks()`, `getOutboundCalls()`.
|
||||
- **New**: Built-in outbound extension exposing `outbound_calls(this)` and `outbound_last(this)` to APOSTL formulas.
|
||||
|
||||
```typescript
|
||||
```javascript
|
||||
await fastify.register(apophis, {
|
||||
outboundContracts: {
|
||||
'stripe.paymentIntents.create': {
|
||||
@@ -87,74 +98,41 @@ await fastify.register(apophis, {
|
||||
}
|
||||
})
|
||||
|
||||
// Routes reference contracts via x-outbound
|
||||
const schema = {
|
||||
'x-outbound': ['stripe.paymentIntents.create'],
|
||||
'x-ensures': [
|
||||
'if response_code == 200 then outbound_last(this).stripe.paymentIntents.create.response.statusCode == 200 else true'
|
||||
]
|
||||
}
|
||||
|
||||
// Imperative E2E
|
||||
await fastify.apophis.enableOutboundMocks({
|
||||
overrides: {
|
||||
'stripe.paymentIntents.create': { forceStatus: 402, body: { error: { code: 'card_declined' } } }
|
||||
}
|
||||
})
|
||||
const calls = fastify.apophis.getOutboundCalls('stripe.paymentIntents.create')
|
||||
await fastify.apophis.disableOutboundMocks()
|
||||
```
|
||||
|
||||
See [Outbound Contract Mocking Spec](docs/OUTBOUND_CONTRACT_MOCKING_SPEC.md) for full documentation.
|
||||
#### Mutation Testing
|
||||
|
||||
- **New**: `src/quality/mutation.ts` — synthetic bug injection to measure contract strength.
|
||||
- **New**: `runMutationTesting()` — generates mutations and verifies tests catch them. Internal API only; not part of the public `@apophis/fastify` export.
|
||||
- **New**: Mutation score reporting (0-100%) with weak contract identification.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Migrated**: `runStatefulTests` now uses `EnhancedChaosEngine` from `chaos-v2.ts` (was using deprecated `ChaosEngine` from `chaos.ts`). Stateful and contract runners now share a single chaos stack.
|
||||
- Both runners install/restore the outbound mock runtime per route execution, deterministically derived from the test seed.
|
||||
- Package name: `@watchdog/fastify` → `@apophis/fastify`.
|
||||
- Binary: `watchdog` → `apophis`.
|
||||
- Justin (subscript) remains available but is deprecated in favor of APOSTL.
|
||||
|
||||
## [2.3.0] - 2026-04-27
|
||||
---
|
||||
|
||||
### Changed
|
||||
|
||||
#### Chaos System Final Cutover
|
||||
|
||||
Cleaned up the chaos architecture by removing unused types/config paths, unifying public APIs, and wiring the active outbound chaos path.
|
||||
|
||||
- **Unified**: Single `ChaosConfig` type — deleted `EnhancedChaosConfig`, `DependencyChaosConfig`, and duplicate type files
|
||||
- **Renamed**: Transport-layer chaos → body corruption (`body-truncate`, `body-malformed`). Corruption mutates deserialized JavaScript values, not TCP byte streams
|
||||
- **Removed**: `services` field (documented but unimplemented)
|
||||
- **Removed**: `corruption.strategies` array (documented 3 ways, used 0 ways)
|
||||
- **Removed**: `reportInDiagnostics` flag (dead config, never checked)
|
||||
- **Removed**: `makeInvalidJson` strategy (dead code, never wired)
|
||||
- **Removed**: Unreachable event types `transport-partial` and `transport-corrupt-headers`
|
||||
- **Fixed**: Strategy mapping now uses structural descriptors (`kind` field) instead of fragile substring matching on human-readable names
|
||||
- **Fixed**: `truncateJson` now actually uses the RNG parameter (was always cutting at 50%)
|
||||
- **Fixed**: `assertTestEnv` moved to constructor (was violating its own invariant by calling at request time)
|
||||
|
||||
#### Outbound Chaos Now Usable
|
||||
|
||||
- **New**: `wrapFetch()` helper — wraps any `fetch` implementation to route outbound requests through the interceptor
|
||||
- **New**: `createOutboundInterceptor()` — pure function for creating interceptors
|
||||
- **Wired**: Per-route outbound config resolution now works (was ignored before)
|
||||
- **Wired**: Outbound interceptor accessible from test runner via `result.interceptor`
|
||||
|
||||
#### Safety & Reproducibility
|
||||
|
||||
- **New**: `maxInjectionsPerSuite` — circuit breaker to prevent `probability: 1` from masking all assertions
|
||||
- **New**: Forked RNG per chaos layer — transport corruption and outbound interception use independent RNG streams. Adding outbound config no longer shifts transport corruption sequence
|
||||
## [WATCHDOG 2.4.0] - 2025-08-14
|
||||
|
||||
### Added
|
||||
|
||||
#### Dependency-Aware Chaos Testing (v2)
|
||||
#### Dependency-Aware Chaos Testing
|
||||
|
||||
- **New**: `ChaosConfig.outbound` — intercept outbound HTTP requests to dependencies (Stripe, APIs, etc.)
|
||||
- **New**: Chaos event reporting in test diagnostics
|
||||
- **New**: Configurable dropout status codes — default 504 Gateway Timeout
|
||||
- **New**: `ChaosConfig.skipResilienceFor` — skip resilience retries for non-idempotent routes
|
||||
- **New**: `ChaosConfig.outbound` — intercept outbound HTTP requests to dependencies.
|
||||
- **New**: Chaos event reporting in test diagnostics.
|
||||
- **New**: Configurable dropout status codes (default 504 Gateway Timeout).
|
||||
- **New**: `ChaosConfig.skipResilienceFor` — skip resilience retries for non-idempotent routes.
|
||||
|
||||
```typescript
|
||||
// Simulate Stripe failures
|
||||
await fastify.apophis.contract({
|
||||
```javascript
|
||||
await fastify.watchdog.contract({
|
||||
depth: 'quick',
|
||||
chaos: {
|
||||
probability: 0.1,
|
||||
@@ -170,276 +148,244 @@ await fastify.apophis.contract({
|
||||
}
|
||||
}
|
||||
],
|
||||
// Skip retries for routes that create side effects
|
||||
skipResilienceFor: ['constructor', 'mutator']
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
See [Dependency-Aware Chaos Guide](docs/chaos-v2.md) for full documentation.
|
||||
#### Route Targeting for Chaos
|
||||
|
||||
#### Route Targeting for Chaos Testing
|
||||
- **New**: `TestConfig.routes` — test only specific routes.
|
||||
- **New**: `ChaosConfig.include` / `ChaosConfig.exclude` — include/exclude routes from chaos with wildcards.
|
||||
- **New**: `ChaosConfig.routes` — per-route chaos overrides.
|
||||
- **New**: `ChaosConfig.resilience` — verify system recovery after chaos injection.
|
||||
- **New**: `ChaosConfig.maxInjectionsPerSuite` — circuit breaker for total injections.
|
||||
|
||||
- **New**: `TestConfig.routes` — test only specific routes instead of all discovered routes
|
||||
- **New**: `ChaosConfig.include` / `ChaosConfig.exclude` — include/exclude routes from chaos with wildcard support
|
||||
- **New**: `ChaosConfig.routes` — per-route chaos overrides
|
||||
- **New**: `ChaosConfig.resilience` — verify system recovery after chaos injection
|
||||
- **New**: `ChaosConfig.maxInjectionsPerSuite` — circuit breaker for total injections
|
||||
#### Performance
|
||||
|
||||
```typescript
|
||||
// Test only specific routes
|
||||
await fastify.apophis.contract({
|
||||
depth: 'quick',
|
||||
routes: ['GET /health', 'POST /billing/plans'],
|
||||
chaos: {
|
||||
probability: 0.3,
|
||||
include: ['/billing/*'],
|
||||
exclude: ['/billing/sensitive'],
|
||||
resilience: { enabled: true, maxRetries: 3 },
|
||||
maxInjectionsPerSuite: 50
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
#### Mutation Testing
|
||||
|
||||
- **New**: `src/quality/mutation.ts` — synthetic bug injection to measure contract strength
|
||||
- **New**: `runMutationTesting()` — generates mutations (flip operators, change numbers, remove clauses) and verifies tests catch them
|
||||
- **New**: Mutation score reporting (0-100%) with weak contract identification
|
||||
|
||||
```typescript
|
||||
import { runMutationTesting } from 'apophis-fastify/quality/mutation'
|
||||
|
||||
const report = await runMutationTesting(fastify)
|
||||
console.log(`Mutation score: ${report.score}%`) // 85%
|
||||
console.log('Weak contracts:', report.weakContracts)
|
||||
```
|
||||
|
||||
#### Performance Improvements
|
||||
|
||||
- **P2**: Full SHA-256 hashes (64 chars) instead of truncated 16-char hashes
|
||||
- **P3**: Configurable parse cache with `setParseCacheLimit()`, `getParseCacheLimit()`, `clearParseCache()`
|
||||
- **P5**: Chunked NDJSON processing with `x-stream-max-chunk-size` limit (default 1MB)
|
||||
- **P8**: Lazy topological sorting for extension registry (sorts only when needed)
|
||||
|
||||
#### Observability
|
||||
|
||||
- **O2**: Per-route chaos granularity with include/exclude patterns
|
||||
- **O3**: Resilience verification — retry after chaos to confirm recovery
|
||||
- **O4**: Pre-filter routes with contracts — skip hook evaluation for routes without annotations
|
||||
- **O5**: Forked RNG per chaos layer — transport and outbound use independent streams
|
||||
- Full SHA-256 hashes for determinism (64 chars) instead of truncated 16-char hashes.
|
||||
- Configurable parse cache with `setParseCacheLimit()`, `clearParseCache()`.
|
||||
- Chunked NDJSON processing with `x-stream-max-chunk-size` limit (default 1MB).
|
||||
- Lazy topological sorting for extension registry.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Critical**: Disabled array-of-objects schema inference that generated invalid APOSTL (`data[].id` syntax). Arrays of objects now require explicit `x-ensures` formulas.
|
||||
- Schema inference no longer crashes on collection schemas (LDF Collection fragments)
|
||||
- **P0**: Chaos events now visible in test diagnostics with type, status code, and dependency URL
|
||||
- **C1**: ScopeRegistry default scope bug — now respects configured `default` scope
|
||||
- **C2**: Plugin contract builder — `routes` option now propagated to test runner
|
||||
- **P2**: Dropout returns 504 Gateway Timeout instead of status code 0
|
||||
- **P3**: Resilience verification skips non-idempotent routes by default
|
||||
|
||||
## [2.1.0] - 2026-04-26
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
#### Justin Support Removed
|
||||
|
||||
- **Removed**: Justin (subscript) expression evaluator and all Justin compatibility code
|
||||
- **Removed**: `src/formula/justin.ts` (wrapper with compile cache)
|
||||
- **Removed**: `src/formula/context-builder.ts` (Justin context mapping)
|
||||
- **Removed**: `subscript` dependency from package.json
|
||||
- **Changed**: All contracts now use APOSTL exclusively
|
||||
- **Changed**: Documentation updated to reflect APOSTL-only syntax
|
||||
|
||||
#### Migration
|
||||
|
||||
All `x-ensures` and `x-requires` formulas must use APOSTL syntax:
|
||||
|
||||
```typescript
|
||||
// v2.1 — APOSTL (required)
|
||||
'x-ensures': ['status:201', 'response_body(this).id != null']
|
||||
|
||||
// v2.0 — Justin (removed)
|
||||
'x-ensures': ['statusCode == 201', 'response.body.id != null']
|
||||
```
|
||||
|
||||
See [Getting Started Guide](docs/getting-started.md) for full APOSTL reference.
|
||||
- Chaos events now visible in test diagnostics with type and status code.
|
||||
- ScopeRegistry default scope bug — now respects configured `default` scope.
|
||||
- Plugin contract builder — `routes` option now propagated to test runner.
|
||||
- Dropout returns 504 Gateway Timeout instead of status code 0.
|
||||
- Resilience verification skips non-idempotent routes by default.
|
||||
- Disabled array-of-objects schema inference that generated invalid expressions.
|
||||
- Schema inference no longer crashes on collection schemas.
|
||||
|
||||
---
|
||||
|
||||
## [2.0.0] - 2026-04-25
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
#### APOSTL Replaced with Justin (Plain JavaScript Expressions)
|
||||
|
||||
- **Removed**: Custom APOSTL parser (`src/formula/parser.ts`, `src/formula/tokenizer.ts`, `src/formula/evaluator.ts`, `src/formula/substitutor.ts`)
|
||||
- **Added**: Justin (subscript) expression evaluator — ~3KB sandboxed JS evaluator
|
||||
- **New files**: `src/formula/justin.ts` (wrapper with compile cache), `src/formula/context-builder.ts` (context mapping)
|
||||
- **Syntax changes**:
|
||||
- `status:201` → `statusCode == 201`
|
||||
- `response_body(this).id` → `response.body.id`
|
||||
- `request_headers(this).auth` → `request.headers.auth`
|
||||
- `if a then b else T` → `a ? b : true` (or `!a || b`)
|
||||
- `for x in arr: p` → `arr.every(x => p)`
|
||||
- `x matches /r/` → `/r/.test(x)`
|
||||
- `previous(expr)` → `previous.*` (e.g., `previous.response.body.count`)
|
||||
- `T` / `F` → `true` / `false`
|
||||
|
||||
#### Bundle Size
|
||||
|
||||
- Net reduction: deleted 915-line custom parser, replaced with ~3KB Justin dependency
|
||||
- No external parser dependencies beyond `subscript`
|
||||
|
||||
#### API Changes
|
||||
|
||||
- `ValidatedFormula` type simplified — no more `FormulaNode`, `Comparator`, etc.
|
||||
- Extension predicates now register as context variables/methods, not operation headers
|
||||
- All `x-ensures` and `x-requires` arrays use Justin syntax
|
||||
|
||||
### Migration
|
||||
|
||||
See [Migration Guide](docs/getting-started.md#migration-from-v1x) for complete conversion table.
|
||||
|
||||
---
|
||||
|
||||
## [1.2.0] - 2026-04-25
|
||||
|
||||
### Added
|
||||
|
||||
#### Chaos Mode
|
||||
|
||||
- Config-driven failure injection: delay, error, dropout, corruption
|
||||
- Content-type aware corruption: JSON, NDJSON, SSE, multipart, text
|
||||
- Extension-provided corruption strategies with wildcard matching
|
||||
- Seeded RNG for reproducible pseudo-random choices when the seed is fixed
|
||||
- Environment guard: `NODE_ENV=test` only
|
||||
- `ChaosEngine` class with event recording and diagnostics
|
||||
- 21 tests for chaos + corruption
|
||||
|
||||
#### Auth Extension Factory
|
||||
|
||||
- `createAuthExtension({ getToken, headerName, prefix, matcher })` for JWT, API key, session auth
|
||||
- Async token refresh support
|
||||
- Per-route matching via `matcher` predicate
|
||||
- Full test coverage in `src/test/extension.test.ts`
|
||||
- Documentation: `docs/auth-patterns.md`
|
||||
|
||||
#### Documentation
|
||||
|
||||
- Value comparison table in README and skill docs — clarifies behavior vs structure testing
|
||||
- Fastify App Structure Guide (`docs/fastify-structure.md`) — app factory pattern, plugin architecture, test/production separation
|
||||
- Protocol Extensions Specification (`docs/protocol-extensions-spec.md`) — JWT, Time Control, Stateful, X.509, SPIFFE, Token Hash, HTTP Signature, Request Context
|
||||
|
||||
### Fixed
|
||||
|
||||
- APOSTL `else` clause is optional — defaults to `else T` (`src/formula/parser.ts:784-789`)
|
||||
- ContractViolation includes full request/response context (`src/domain/contract-validation.ts:134-145`)
|
||||
|
||||
---
|
||||
|
||||
## [1.2.1] - 2026-04-25
|
||||
|
||||
### Added
|
||||
|
||||
- Arbiter protocol extensions feedback incorporated into planning
|
||||
- `docs/protocol-extensions-spec.md` — specification for JWT, Time Control, Stateful Predicates, X.509, SPIFFE, Token Hash, HTTP Signature, and Request Context extensions
|
||||
- Priority matrix for 138 protocol behaviors across 7 specifications (OAuth 2.1, WIMSE S2S, Transaction Tokens, SPIFFE/SPIRE, Token Exchange, Device Auth, CIBA)
|
||||
## [WATCHDOG 2.3.0] - 2025-07-22
|
||||
|
||||
### Changed
|
||||
|
||||
- Updated `docs/attic/root-history/NEXT_STEPS_425.md` with P0/P1/P2/P3 categorization for protocol extensions
|
||||
- Updated `docs/attic/QUALITY_FEATURES_PLAN.md` — Chaos marked complete, Flake/Mutation scheduled for v1.3
|
||||
- Updated `docs/PLUGIN_CONTRACTS_SPEC.md` — noted complementarity with protocol extensions
|
||||
#### Chaos System Final Cutover
|
||||
|
||||
- **Unified**: Single `ChaosConfig` type — deleted `EnhancedChaosConfig`, `DependencyChaosConfig`, and duplicate type files.
|
||||
- **Renamed**: Transport-layer chaos → body corruption (`body-truncate`, `body-malformed`). Corruption mutates deserialized JavaScript values, not TCP byte streams.
|
||||
- **Removed**: `services` field (documented but unimplemented).
|
||||
- **Removed**: `corruption.strategies` array (documented 3 ways, used 0 ways).
|
||||
- **Removed**: `reportInDiagnostics` flag (dead config).
|
||||
- **Removed**: `makeInvalidJson` strategy (dead code).
|
||||
- **Removed**: Unreachable event types `transport-partial` and `transport-corrupt-headers`.
|
||||
- **Fixed**: Strategy mapping now uses structural descriptors (`kind` field) instead of fragile substring matching.
|
||||
- **Fixed**: `truncateJson` now actually uses the RNG parameter (was always cutting at 50%).
|
||||
- **Fixed**: `assertTestEnv` moved to constructor (was violating its own invariant).
|
||||
|
||||
#### Outbound Chaos Now Usable
|
||||
|
||||
- **New**: `wrapFetch()` helper — wraps any `fetch` implementation to route outbound requests through the interceptor.
|
||||
- **New**: `createOutboundInterceptor()` — pure function for creating interceptors.
|
||||
- **Wired**: Per-route outbound config resolution now works.
|
||||
- **Wired**: Outbound interceptor accessible from test runner via `result.interceptor`.
|
||||
|
||||
#### Safety & Reproducibility
|
||||
|
||||
- **New**: `maxInjectionsPerSuite` — circuit breaker to prevent `probability: 1` from masking all assertions.
|
||||
- **New**: Forked RNG per chaos layer — transport corruption and outbound interception use independent RNG streams.
|
||||
|
||||
---
|
||||
|
||||
## [1.1.0] - 2026-04-24
|
||||
## [WATCHDOG 2.2.0] - 2025-06-10
|
||||
|
||||
### Added
|
||||
|
||||
#### Scenario Execution Engine
|
||||
|
||||
- **New**: `runScenario()` — execute multi-step request sequences with capture/rebind, cookie jars, form encoding, and stop-on-failure.
|
||||
- **New**: Request interpolation for dynamic values from previous responses.
|
||||
- **New**: Step-level header overrides and Content-Type injection.
|
||||
|
||||
#### Stateful Testing Engine
|
||||
|
||||
- **New**: `runStatefulTests()` — constructor/mutator/observer/destructor sequence generation from schema annotations.
|
||||
- **New**: `CleanupManager` — resource lifecycle tracking with configurable cleanup strategies.
|
||||
- **New**: Invariant checking across stateful sequences.
|
||||
- **New**: Outbound mock runtime integration for stateful tests.
|
||||
|
||||
---
|
||||
|
||||
## [WATCHDOG 2.1.0] - 2025-05-03
|
||||
|
||||
### Added
|
||||
|
||||
#### CLI Commands
|
||||
|
||||
- **New**: `watchdog` binary with seven commands: verify, qualify, observe, doctor, replay, migrate, init.
|
||||
- **New**: Route discovery from Fastify's `hasRoute` introspection.
|
||||
- **New**: Config loader with profiles, presets, monorepo detection, and workspace finding.
|
||||
- **New**: Human and machine output renderers (text, JSON, NDJSON).
|
||||
- **New**: Artifact-based replay with seed determinism.
|
||||
- **New**: Environment safety checks via `doctor` command.
|
||||
|
||||
#### Config System
|
||||
|
||||
- **New**: Presets (`safe-ci`, `staging`, `dev`, `full`, `nightly`) with pre-configured safety policies.
|
||||
- **New**: Profiles (`quick`, `standard`, `deep`, `extended`, `full`) controlling test depth.
|
||||
- **New**: Generation profiles for property-based test sampling.
|
||||
- **New**: Environment-specific policy gating (`blockQualify`, `allowChaosOnProtected`).
|
||||
|
||||
---
|
||||
|
||||
## [WATCHDOG 2.0.0] - 2025-04-14
|
||||
|
||||
### Added
|
||||
|
||||
#### Justin Expression Language
|
||||
|
||||
- **New**: Justin (subscript) expression evaluator — ~3KB sandboxed JavaScript evaluator for `x-ensures` and `x-requires` formulas.
|
||||
- **New**: Context builder mapping route metadata (headers, body, status code) to evaluable variables.
|
||||
- Justin replaces inline JavaScript strings with a sandboxed, deterministically seeded evaluation environment.
|
||||
|
||||
#### Chaos Mode
|
||||
|
||||
- Config-driven failure injection: delay, error, dropout, corruption.
|
||||
- Content-type aware corruption: JSON, NDJSON, SSE, multipart, text.
|
||||
- Extension-provided corruption strategies with wildcard matching.
|
||||
- Seeded RNG for reproducible pseudo-random choices.
|
||||
- Environment guard: `NODE_ENV=test` only.
|
||||
- `ChaosEngine` class with event recording and diagnostics.
|
||||
|
||||
#### Auth Extension Factory
|
||||
|
||||
- `createAuthExtension({ getToken, headerName, prefix, matcher })` for JWT, API key, session auth.
|
||||
- Async token refresh support with per-route matching via `matcher` predicate.
|
||||
|
||||
#### Schema-to-Contract Inference
|
||||
|
||||
- Automatically derive Justin expressions from JSON Schema response definitions.
|
||||
- Infers `!= null` for `required` fields, `>=`/`<=` for `minimum`/`maximum` bounds.
|
||||
- Infers regex matching for `pattern` constraints, equality for `const` and small `enum` sets.
|
||||
- Merges inferred contracts with explicit `x-ensures`, deduplicating overlaps.
|
||||
|
||||
#### Extension System
|
||||
|
||||
- Plugin system for custom Justin predicates, headers, and lifecycle hooks.
|
||||
- Extension state isolation (frozen copies per extension).
|
||||
- Hook timeout and severity configuration.
|
||||
- Dependency ordering via `dependsOn` with topological sort.
|
||||
- Async boot: `onSuiteStart` hooks run in dependency order.
|
||||
- Health checks: extensions validate before running hooks.
|
||||
|
||||
#### Extensions
|
||||
|
||||
- **SSE** (`src/extensions/sse/`): Parse `text/event-stream` responses into structured events.
|
||||
- **Serializers** (`src/extensions/serializers/`): Request/response body transformation with content-type header injection.
|
||||
- **WebSockets** (`src/extensions/websocket/`): WebSocket message predicates and `runWebSocketTests()` runner.
|
||||
|
||||
### Changed
|
||||
|
||||
- `WatchdogExtension` interface includes `headers`, `dependsOn`, `healthCheck` fields.
|
||||
- `parse()` accepts optional `extensionHeaders` parameter.
|
||||
- `ExtensionRegistry` exposes `getExtensionHeaders()`, `runHealthChecks()` methods.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Justin expression parsing handles nested accessors and undefined guards.
|
||||
- Extension predicate return type narrowing.
|
||||
- Multipart files type safety in request builder.
|
||||
|
||||
---
|
||||
|
||||
## [WATCHDOG 1.2.0] - 2025-03-01
|
||||
|
||||
### Added
|
||||
|
||||
#### Multipart Uploads
|
||||
|
||||
- `multipart/form-data` request generation from JSON Schema annotations
|
||||
- Fake file generation with size, MIME type, and count constraints
|
||||
- `request.files` and `request.fields` Justin context variables
|
||||
- File arrays when `maxCount > 1`
|
||||
- Schema annotations: `x-content-type`, `x-multipart-fields`, `x-multipart-files`
|
||||
- `multipart/form-data` request generation from JSON Schema annotations.
|
||||
- Fake file generation with size, MIME type, and count constraints.
|
||||
- Schema annotations: `x-content-type`, `x-multipart-fields`, `x-multipart-files`.
|
||||
|
||||
#### Streaming / NDJSON
|
||||
|
||||
- Response chunk collection for streaming routes
|
||||
- NDJSON format parsing
|
||||
- `response.chunks` and `response.duration` Justin context variables
|
||||
- Schema annotations: `x-streaming`, `x-stream-format`, `x-stream-max-chunks`
|
||||
- Integration tests with Fastify NDJSON routes
|
||||
|
||||
#### Extension System
|
||||
|
||||
- Plugin system for custom Justin predicates, headers, and lifecycle hooks
|
||||
- Extension state isolation (frozen copies per extension)
|
||||
- Hook timeout and severity configuration
|
||||
- Dependency ordering via `dependsOn` with topological sort
|
||||
- Async boot: `onSuiteStart` hooks run in dependency order
|
||||
- Health checks: extensions validate before running hooks
|
||||
- Security: redaction of sensitive data, timeout guards, prototype pollution prevention
|
||||
|
||||
#### Extensions
|
||||
|
||||
- **SSE** (`src/extensions/sse/`): Parse `text/event-stream` responses into structured events. Expression: `response.sse[0].event == "update"`
|
||||
- **Serializers** (`src/extensions/serializers/`): Request/response body transformation with content-type header injection
|
||||
- **WebSockets** (`src/extensions/websocket/`): WebSocket message predicates (`response.ws.message.type`, `response.ws.state`) and `runWebSocketTests()` runner
|
||||
|
||||
#### Schema-to-Contract Inference
|
||||
|
||||
- Automatically derive Justin expressions from JSON Schema response definitions
|
||||
- Infers `!= null` for `required` fields
|
||||
- Infers `>=` / `<=` for `minimum` / `maximum` bounds
|
||||
- Infers `.test()` for `pattern` regexes
|
||||
- Infers `==` for `const` values and small `enum` sets
|
||||
- Merges inferred contracts with explicit `x-ensures`, deduplicating overlaps
|
||||
- Response chunk collection for streaming routes.
|
||||
- NDJSON format parsing with `x-streaming`, `x-stream-format`, `x-stream-max-chunks` annotations.
|
||||
- Integration tests with Fastify NDJSON routes.
|
||||
|
||||
#### Core Improvements
|
||||
|
||||
- Parser accepts registered extension headers
|
||||
- Extension predicates checked before core operations during evaluation
|
||||
- `evaluateAsync()` for async predicate resolvers
|
||||
- `validateFormula()` with error position and suggestions for common mistakes
|
||||
- New types: `MultipartFile`, `MultipartPayload`, streaming response fields
|
||||
|
||||
### Changed
|
||||
|
||||
- `ApophisExtension` interface includes `headers`, `dependsOn`, `healthCheck` fields
|
||||
- `parse()` accepts optional `extensionHeaders` parameter
|
||||
- `ExtensionRegistry` exposes `getExtensionHeaders()`, `runHealthChecks()` methods
|
||||
- TypeScript strict mode compliance
|
||||
- Removed `dist/` from git tracking
|
||||
- `evaluateAsync()` for async predicate resolvers.
|
||||
- `validateFormula()` with error position and suggestions.
|
||||
- `ContractViolation` includes full request/response context.
|
||||
|
||||
### Fixed
|
||||
|
||||
- TypeScript strict mode: ~50 errors fixed across 15+ files
|
||||
- Evaluator exports restored (`evaluate`, `evaluateBooleanResult`, `evaluateWithExtensions`, `evaluateAsync`)
|
||||
- Status node handling in both sync and async evaluators
|
||||
- Accessor undefined checks in `resolveOperation` and `resolveOperationAsync`
|
||||
- Multipart files type safety in request builder
|
||||
- Predicate return type narrowing (synchronous only)
|
||||
- Extension test type safety
|
||||
- TypeScript strict mode: ~50 errors fixed across 15+ files.
|
||||
- Evaluator exports restored.
|
||||
- Status node handling in both sync and async evaluators.
|
||||
|
||||
---
|
||||
|
||||
## [1.0.0] - 2026-04-24
|
||||
## [WATCHDOG 1.1.0] - 2025-02-10
|
||||
|
||||
### Added
|
||||
|
||||
- Contract-driven API testing for Fastify
|
||||
- Property-based testing with fast-check
|
||||
- APOSTL expression language for contracts
|
||||
- Timeout enforcement and redirect capture
|
||||
- Seeded RNG for reproducible concurrent tests
|
||||
- Extension plugin system
|
||||
- 412 tests
|
||||
#### Contract-Driven Testing
|
||||
|
||||
- Property-based testing with fast-check: generated requests against `x-ensures` and `x-requires` contracts.
|
||||
- Timeout enforcement and redirect capture.
|
||||
- Seeded RNG for reproducible concurrent tests.
|
||||
|
||||
#### Documentation
|
||||
|
||||
- Fastify App Structure Guide (`docs/fastify-structure.md`).
|
||||
- Protocol Extensions Specification (`docs/protocol-extensions-spec.md`).
|
||||
|
||||
### Fixed
|
||||
|
||||
- Contract formulas support optional `else` clauses.
|
||||
- Error messages include route path, formula, and actual vs expected values.
|
||||
|
||||
---
|
||||
|
||||
## [WATCHDOG 1.0.0] - 2025-01-06
|
||||
|
||||
### Added
|
||||
|
||||
- Contract-driven API testing plugin for Fastify.
|
||||
- `x-ensures` and `x-requires` schema annotations for property contracts.
|
||||
- JSON Schema validation integrated into the test lifecycle.
|
||||
- 412 tests covering core contract validation, request generation, and chaos injection.
|
||||
|
||||
---
|
||||
|
||||
## [WATCHDOG 0.1.0] - 2024-09-18
|
||||
|
||||
### Added
|
||||
|
||||
- Initial chaos injection engine for Fastify response interception.
|
||||
- Configurable failure modes: delay, error, dropout, and body corruption.
|
||||
- Content-type aware response body mutation.
|
||||
- Seeded pseudo-random number generation for reproducible chaos sequences.
|
||||
- Environment guard preventing chaos injection outside `NODE_ENV=test`.
|
||||
- 85 tests covering all four chaos strategies and content-type handling.
|
||||
|
||||
## License
|
||||
|
||||
ISC
|
||||
MIT
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 APOPHIS Team
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -4,14 +4,18 @@ Behavioral confidence for Fastify services.
|
||||
|
||||
APOPHIS checks whether route behavior holds across operations, states, and protocol flows.
|
||||
|
||||
Supported Node.js versions: 20.x and 22.x.
|
||||
Inspired by the concept of invariant-driven automated testing: instead of only checking payload shape, APOPHIS encodes intended behavior as executable contracts and verifies them with property-based and stateful testing.
|
||||
|
||||
Supported Node.js versions: >=20.18.1 (20.x) and 22.x.
|
||||
|
||||
```bash
|
||||
npm install apophis-fastify fastify @fastify/swagger
|
||||
apophis init --preset safe-ci
|
||||
apophis verify --profile quick --routes "POST /users"
|
||||
npm install @apophis/fastify fastify @fastify/swagger
|
||||
npx apophis init --preset safe-ci
|
||||
npx apophis verify --profile quick --routes "POST /users"
|
||||
```
|
||||
|
||||
`x-ensures` is an OpenAPI schema extension for behavioral contracts — statements about what a route must guarantee.
|
||||
|
||||
## Cross-Route Failure Example
|
||||
|
||||
Add one behavioral contract next to a route schema. APOPHIS can verify cross-route behavior, such as whether a resource created by one route is retrievable through another.
|
||||
@@ -19,6 +23,8 @@ Add one behavioral contract next to a route schema. APOPHIS can verify cross-rou
|
||||
**Route:**
|
||||
|
||||
```javascript
|
||||
import crypto from 'crypto';
|
||||
|
||||
app.post('/users', {
|
||||
schema: {
|
||||
'x-category': 'constructor',
|
||||
@@ -29,7 +35,7 @@ app.post('/users', {
|
||||
}
|
||||
}, async (request, reply) => {
|
||||
const { name } = request.body;
|
||||
const id = `usr-${Date.now()}`;
|
||||
const id = `usr-${crypto.createHash('sha256').update(name).digest('hex').slice(0, 8)}`;
|
||||
reply.status(201);
|
||||
return { id, name };
|
||||
});
|
||||
@@ -47,7 +53,7 @@ Expected
|
||||
response_code(GET /users/{response_body(this).id}) == 200
|
||||
|
||||
Observed
|
||||
GET /users/usr-123 returned 404
|
||||
GET /users/usr-7d865e returned 404
|
||||
|
||||
Why this matters
|
||||
The resource created by POST /users is not retrievable.
|
||||
@@ -66,20 +72,23 @@ JSON Schema cannot express this relationship. APOPHIS turns it into an executabl
|
||||
| Mode | Purpose | Default Environments |
|
||||
|---|---|---|
|
||||
| `verify` | Deterministic CI and local contract verification | local, test, CI |
|
||||
| `observe` | Runtime visibility and drift detection without blocking | staging, prod |
|
||||
| `observe` | Programmatic runtime visibility and drift detection without blocking (validated via `doctor --mode observe`) | staging, prod |
|
||||
| `qualify` | Exercise scenarios, stateful flows, and configured chaos checks before release | local, test, staging |
|
||||
|
||||
## Quickstart: 3 Commands
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
# 1. Install
|
||||
npm install apophis-fastify fastify @fastify/swagger
|
||||
npm install @apophis/fastify fastify @fastify/swagger
|
||||
|
||||
# 2. Scaffold
|
||||
apophis init --preset safe-ci
|
||||
npx apophis init --preset safe-ci
|
||||
|
||||
# 3. Verify
|
||||
apophis verify --profile quick --routes "POST /users"
|
||||
npx apophis verify --profile quick --routes "POST /users"
|
||||
|
||||
# 4. Doctor
|
||||
npx apophis doctor
|
||||
```
|
||||
|
||||
See [docs/getting-started.md](docs/getting-started.md) for the full walkthrough.
|
||||
@@ -87,9 +96,12 @@ See [docs/getting-started.md](docs/getting-started.md) for the full walkthrough.
|
||||
## Trust and Safety
|
||||
|
||||
- **Deterministic replay**: Every failure includes a seed and a one-command replay.
|
||||
- **Explicit test budget**: Control how many tests run with `runs: 10` in your preset.
|
||||
- **CI-safe default path**: `verify` is deterministic and safe for CI pipelines.
|
||||
- **Production-safe observe path**: `observe` is non-blocking by default. Blocking behavior requires explicit break-glass policy.
|
||||
- **Machine-readable output**: `--format json-summary` and `--format ndjson-summary` for CI dashboards.
|
||||
- **Production-safe observe path**: `observe` is the programmatic runtime plugin (no CLI command). Non-blocking by default. Blocking behavior requires explicit break-glass policy.
|
||||
- **Qualify path gated away from prod**: `qualify` is blocked in production by default.
|
||||
- **Monorepo workspace support**: `--workspace` fans out `verify` and `doctor` across all packages.
|
||||
- **Explicit environment boundaries**: Config rejects unknown keys and unsafe environment mixes.
|
||||
|
||||
## LLM-Safe
|
||||
@@ -106,14 +118,55 @@ See [docs/llm-safe-adoption.md](docs/llm-safe-adoption.md) for templates and CI
|
||||
## Full Documentation
|
||||
|
||||
- [Getting Started](docs/getting-started.md) — First route, first verify run, first replay
|
||||
- [CLI Reference](docs/cli.md) — All 7 commands, global flags, exit codes
|
||||
- [CLI Reference](docs/cli.md) — All 6 commands, global flags, exit codes
|
||||
- [Verify Mode](docs/verify.md) — Deterministic contract verification
|
||||
- [Observe Mode](docs/observe.md) — Runtime visibility and drift detection
|
||||
- [Qualify Mode](docs/qualify.md) — Scenarios, stateful testing, chaos
|
||||
- [Quality Engines](docs/quality.md) — Chaos injection, flake detection, mutation testing
|
||||
- [Performance](docs/performance.md) — Repeatable benchmarks and CPU profiling
|
||||
- [LLM-Safe Adoption](docs/llm-safe-adoption.md) — Scaffolds and CI guards
|
||||
- [Protocol Extensions](docs/protocol-extensions-spec.md) — JWT, X.509, SPIFFE, WIMSE
|
||||
- [Protocol Extensions](docs/attic/protocol-extensions-spec.md) — JWT, X.509, SPIFFE, WIMSE
|
||||
|
||||
## Recommended Integration
|
||||
|
||||
**New projects:** Use `createFastify()` to ensure route discovery is installed before any routes are registered.
|
||||
|
||||
```ts
|
||||
import { createFastify } from '@apophis/fastify'
|
||||
|
||||
const app = await createFastify({
|
||||
logger: true,
|
||||
apophis: {
|
||||
runtime: process.env.NODE_ENV === 'test' ? 'error' : 'off',
|
||||
observe: process.env.NODE_ENV === 'production'
|
||||
? { enabled: true, sampling: 0.1, sinks: [metricsSink] } // your ObserveSink
|
||||
: undefined,
|
||||
},
|
||||
})
|
||||
// Register swagger, auth, plugins, and routes after app creation.
|
||||
```
|
||||
|
||||
**Existing projects:** Register APOPHIS or install route discovery before routes. Run `apophis doctor` to verify routes are discovered with full schema metadata.
|
||||
|
||||
**Schema-less fallback:** If APOPHIS is registered after routes, `printRoutes()` can recover paths but not route schemas or behavioral contracts. `apophis doctor` and `apophis verify` will warn when discovery is schema-less.
|
||||
|
||||
## Current Limitations
|
||||
|
||||
These reflect current implementation behavior. All are actively tracked for improvement.
|
||||
|
||||
- **Route discovery requires ordering.** If the APOPHIS plugin or route discovery hook is not installed before routes are registered, behavioral contract annotations (x-ensures, x-requires, x-outbound, x-variants, x-timeout) cannot be recovered. Use `createFastify()` for new projects or register APOPHIS early.
|
||||
- **Observe is programmatic.** Register `apophisPlugin` with `observe: { enabled: true, sinks: [...] }` for non-blocking contract evaluation on live traffic. Use `apophis doctor --mode observe` to validate config before deploying. See `docs/observe.md`.
|
||||
- **CLI verify samples once per contract by default.** Set `runs` in your preset to increase the number of property-based test samples per route. The programmatic `fastify.apophis.contract()` API supports the same `runs` configuration.
|
||||
- **Outbound mocks are scoped per async context.** Uses `AsyncLocalStorage` so concurrent tests get isolated mock runtimes. No process-global lock.
|
||||
- **Qualify coverage depends on profile configuration.** Qualify runs scenario, stateful, and chaos checks based on profile gates. Chaos route selection uses the configured strategy (one/all/sample/routes).
|
||||
|
||||
## Compatibility
|
||||
|
||||
- **Fastify v5 only.** Fastify v4 and earlier are not supported.
|
||||
- **ESM only.** This package is `"type": "module"` and does not provide a CommonJS build. Use `import` syntax.
|
||||
- **Node.js `>=20.18.1 <21 || >=22 <23`**.
|
||||
- **`@fastify/swagger` must be registered before routes** (APOPHIS auto-registers it if missing).
|
||||
|
||||
## License
|
||||
|
||||
ISC
|
||||
MIT
|
||||
|
||||
@@ -1,49 +1,86 @@
|
||||
---
|
||||
name: apophis-fastify
|
||||
description: Use this skill when adding or improving APOPHIS contract-driven testing for Fastify APIs: route schemas, APOSTL x-requires/x-ensures formulas, property and stateful checks, replayable failures, runtime observe hooks, variants, scenarios, and operator-facing adoption guidance.
|
||||
name: @apophis/fastify
|
||||
description: Use this skill when adding or improving APOPHIS contract-driven testing for Fastify APIs. This tool finds real implementation bugs—resources that appear to create but cannot be retrieved, updates that silently fail to persist, deletions that leave data visible, cross-tenant leakage, and broken state transitions. Use it to encode intended behavior as executable contracts and verify them continuously, not to paper over failures.
|
||||
---
|
||||
|
||||
# apophis-fastify
|
||||
# @apophis/fastify
|
||||
|
||||
APOPHIS verifies API behavior across operations, state changes, protocol flows, and dependencies. Use it when schema validation is not enough to answer whether an endpoint did the right thing.
|
||||
APOPHIS finds real API behavior bugs that schema validation misses. It verifies that operations cause the state changes, isolation guarantees, and dependency interactions the service actually depends on.
|
||||
|
||||
Inspired by research on invariant-driven testing: encode intended behavior as executable contracts, then verify them with property-based and stateful testing.
|
||||
|
||||
## Core Purpose
|
||||
|
||||
APOPHIS is a correctness tool, not a coverage checkbox. Its job is to find where implementation diverges from intended behavior.
|
||||
|
||||
**What APOPHIS finds:**
|
||||
- POST creates a resource that GET cannot retrieve
|
||||
- PUT updates a field but subsequent GET returns stale data
|
||||
- DELETE removes a resource but it still appears in listings
|
||||
- Cross-tenant data leakage through shared collections
|
||||
- Auth boundaries that return success instead of 401/403
|
||||
- Error paths that violate declared retry semantics
|
||||
- State transitions that skip required steps
|
||||
|
||||
**What APOPHIS does NOT do:**
|
||||
- Prove the whole system correct (it finds counterexamples, not proofs)
|
||||
- Replace integration tests, security review, or domain judgment
|
||||
- Validate JSON Schema (Fastify + Swagger already do this)
|
||||
|
||||
**Critical rule for LLMs:** Do not weaken contracts to make tests pass. A failing contract is a signal that the implementation has a bug or the contract captures wrong intent. Investigate first. Fix the bug if the contract is correct. Fix the contract only if the implementation is right and the contract is wrong.
|
||||
|
||||
## When To Use
|
||||
|
||||
Use this skill when the operator asks to:
|
||||
|
||||
1. Add contract testing, API behavior checks, property tests, stateful tests, or Fastify route verification.
|
||||
2. Improve confidence beyond JSON Schema validation.
|
||||
3. Check create/read/update/delete behavior, auth boundaries, tenant isolation, redirects, timeouts, streaming, or external dependency behavior.
|
||||
4. Make an API safer for AI-assisted refactoring, CI, or release qualification.
|
||||
5. Evaluate whether APOPHIS would help a project, even if the operator does not name APOPHIS directly.
|
||||
2. Find real bugs in create/read/update/delete behavior, auth boundaries, tenant isolation, redirects, timeouts, streaming, or external dependency behavior.
|
||||
3. Make an API safer for AI-assisted refactoring, CI, or release qualification.
|
||||
4. Evaluate whether APOPHIS would help a project, even if the operator does not name APOPHIS directly.
|
||||
|
||||
Read `README.md`, `docs/getting-started.md`, or command-specific docs only when the task needs details not present here.
|
||||
|
||||
## Operator Explanation
|
||||
|
||||
Describe APOPHIS as replayable behavioral checks for declared API contracts.
|
||||
Describe APOPHIS as a bug-finding tool for API behavior.
|
||||
|
||||
Short explanation:
|
||||
|
||||
> APOPHIS turns intended API behavior into executable contracts. It checks whether operations cause the state changes, isolation guarantees, and dependency interactions the service depends on, instead of only checking payload shape.
|
||||
> APOPHIS encodes intended API behavior as executable contracts, then automatically finds cases where the implementation violates that behavior. It catches bugs that schema validation cannot: create-not-retrievable, update-not-persisted, delete-still-visible, cross-tenant leakage, and broken state transitions.
|
||||
|
||||
Use these points when relevant:
|
||||
|
||||
1. It catches failures schema validation misses: create-not-retrievable, update-not-persisted, delete-still-visible, cross-tenant leakage, and inconsistent error behavior.
|
||||
2. It gives coding agents a deterministic verification loop after generated changes or refactors.
|
||||
3. It reduces review burden by converting agreed behavior into repeatable checks.
|
||||
1. It finds real bugs schema validation misses: resources that create but cannot be read, updates that do not persist, deletes that leave data visible.
|
||||
2. It gives coding agents a deterministic verification loop after generated changes or refactors. Run contracts after every change; failures indicate regressions.
|
||||
3. It prevents silent breakage. When a refactoring breaks a state transition or isolation guarantee, contracts fail immediately with a reproducible seed.
|
||||
4. It improves CI triage with fixed seeds, replay artifacts, and machine-readable output.
|
||||
5. It supports incremental adoption: start with the highest-risk routes, add high-signal formulas, run, fix, and tighten.
|
||||
5. It supports incremental adoption: start with the highest-risk routes, add high-signal behavioral formulas, run, fix real bugs, and tighten.
|
||||
|
||||
Do not overclaim:
|
||||
|
||||
1. Do not say APOPHIS proves the whole system correct.
|
||||
2. Do not say contracts replace integration tests, security review, or domain judgment.
|
||||
3. Say explicitly that schema quality and formula quality determine test quality.
|
||||
3. Say explicitly that behavioral contracts—not structural checks—determine bug-finding quality.
|
||||
|
||||
Good operator ask:
|
||||
|
||||
> I can add APOPHIS to the five highest-risk routes first, encode the expected behavior as contracts, run the verifier, and show concrete failures or confidence gaps. I only need route priority and intended behavior where the code is ambiguous.
|
||||
> I can add APOPHIS to the five highest-risk routes first, encode the expected behavior as contracts, run the verifier, and show concrete failures where implementation diverges from intent. I only need route priority and intended behavior where the code is ambiguous.
|
||||
|
||||
## Failure Discipline
|
||||
|
||||
**Never paper over a failing contract.**
|
||||
|
||||
When APOPHIS reports a contract violation, follow this order:
|
||||
|
||||
1. **Replay the failure** with the reported seed to confirm it is real.
|
||||
2. **Investigate the implementation.** Does the handler actually do what the contract expects? Is there a race condition? A missing database commit? A wrong query?
|
||||
3. **Fix the implementation** if the contract correctly describes intended behavior.
|
||||
4. **Fix the contract** only if investigation proves the implementation is correct and the contract over-constrains or misdescribes behavior.
|
||||
5. **Never** weaken a contract because "the test is flaky" or "it is too strict" without first proving the implementation is correct.
|
||||
|
||||
**If a contract fails intermittently**, that is a bug. Intermittent failures indicate nondeterminism: race conditions, uncommitted transactions, time-dependent logic, or randomness in handlers. Do not remove the contract. Isolate the nondeterminism and fix it.
|
||||
|
||||
**If a contract fails only under chaos**, that is a resilience bug. The service does not handle the failure mode correctly. Fix the handler or the contract's error-path expectations.
|
||||
|
||||
## Context Discipline
|
||||
|
||||
@@ -53,29 +90,30 @@ Treat context as a finite budget.
|
||||
2. Prefer targeted file reads and symbol searches over loading whole directories.
|
||||
3. Track routes touched, contracts added, seeds used, failures found, and unresolved domain questions.
|
||||
4. Use progressive disclosure: read command docs only when invoking that command; read protocol docs only for variants, redirects, OAuth-style flows, form posts, streaming, or multipart.
|
||||
5. Run small loops: annotate one route group, run the narrowest verification, fix, then widen.
|
||||
5. Run small loops: annotate one route group, run the narrowest verification, fix real bugs, then widen.
|
||||
|
||||
## Default Workflow
|
||||
|
||||
When entering a Fastify codebase:
|
||||
|
||||
1. Locate app construction and route registration.
|
||||
2. Confirm `@fastify/swagger` is registered before `apophis-fastify`.
|
||||
2. Confirm `@fastify/swagger` is registered before `@apophis/fastify`.
|
||||
3. Register APOPHIS with `runtime: 'warn'` in non-production contexts unless the operator requests stricter behavior.
|
||||
4. Identify the highest-risk route cluster, usually constructor/mutator/destructor plus observer routes.
|
||||
5. Ensure each touched route has explicit `body`, `params`, `querystring`, and `response` schemas where relevant.
|
||||
6. Add `x-category` where auto-categorization could be ambiguous.
|
||||
7. Add `x-requires` for preconditions and `x-ensures` for postconditions.
|
||||
8. Run a focused APOPHIS check, then broader contract or stateful verification.
|
||||
9. Fix real behavior failures or tighten weak contracts.
|
||||
10. Report what changed, what ran, what failed, and what needs operator judgment.
|
||||
9. **Fix real behavior failures or tighten weak contracts.** Do not weaken passing contracts to avoid work.
|
||||
10. Report what changed, what ran, what failed, what bugs were found, and what needs operator judgment.
|
||||
|
||||
## Fast Start
|
||||
|
||||
```javascript
|
||||
import Fastify from 'fastify'
|
||||
import swagger from '@fastify/swagger'
|
||||
import apophis from 'apophis-fastify'
|
||||
import apophis from '@apophis/fastify'
|
||||
import crypto from 'crypto'
|
||||
|
||||
const app = Fastify()
|
||||
await app.register(swagger)
|
||||
@@ -85,12 +123,15 @@ app.post('/users', {
|
||||
schema: {
|
||||
'x-category': 'constructor',
|
||||
'x-requires': [
|
||||
'request_headers(this).x-tenant-id != null'
|
||||
// Precondition: user must not already exist
|
||||
'response_code(GET /users/{request_body(this).email}) == 404'
|
||||
],
|
||||
'x-ensures': [
|
||||
'status:201',
|
||||
'response_body(this).id != null',
|
||||
// Behavioral: created resource must be retrievable
|
||||
'response_code(GET /users/{response_body(this).id}) == 200',
|
||||
// Behavioral: round-trip equality
|
||||
'response_body(this) == request_body(this)',
|
||||
// Behavioral: cross-route field persistence
|
||||
'response_body(GET /users/{response_body(this).id}).email == request_body(this).email'
|
||||
],
|
||||
body: {
|
||||
@@ -114,12 +155,13 @@ app.post('/users', {
|
||||
}
|
||||
}
|
||||
}, async (req, reply) => {
|
||||
const id = `usr-${crypto.createHash('sha256').update(req.body.email).digest('hex').slice(0, 8)}`
|
||||
reply.status(201)
|
||||
return { id: 'usr-1', ...req.body }
|
||||
return { id, ...req.body }
|
||||
})
|
||||
|
||||
await app.ready()
|
||||
const suite = await app.apophis.contract({ depth: 'standard' })
|
||||
const suite = await app.apophis.contract({ runs: 50 })
|
||||
```
|
||||
|
||||
## API Surface
|
||||
@@ -141,25 +183,34 @@ Test-only helpers:
|
||||
4. `fastify.apophis.test.disableOutboundMocks()`
|
||||
5. `fastify.apophis.test.getOutboundCalls(...)`
|
||||
|
||||
## Contract Quality
|
||||
## Contract Quality: Behavioral, Not Structural
|
||||
|
||||
Minimum:
|
||||
**Structural checks are useless.** Fastify + `@fastify/swagger` already enforce status codes, required fields, and types. Behavioral contracts find what schemas cannot.
|
||||
|
||||
1. Each mutating route has a status expectation.
|
||||
2. Each response with identity has key field non-null checks.
|
||||
**Minimum behavioral baseline:**
|
||||
|
||||
1. Constructor routes verify cross-route retrievability.
|
||||
2. Mutator routes verify state-change visibility.
|
||||
3. Destructor routes verify unavailability after deletion.
|
||||
|
||||
```apostl
|
||||
status:201
|
||||
response_body(this).id != null
|
||||
// Constructor: resource must be retrievable after creation
|
||||
response_code(GET /users/{response_body(this).id}) == 200
|
||||
|
||||
// Mutator: changed field must persist
|
||||
response_body(GET /users/{request_params(this).id}).status == request_body(this).status
|
||||
|
||||
// Destructor: deleted resource must not be retrievable
|
||||
response_code(GET /users/{request_params(this).id}) == 404
|
||||
```
|
||||
|
||||
Production baseline:
|
||||
**Production baseline:**
|
||||
|
||||
1. Constructor routes check that created resources are retrievable.
|
||||
2. Mutator routes check that persisted state reflects the mutation.
|
||||
3. Destructor routes check that deleted resources are unavailable or marked inactive.
|
||||
|
||||
High-confidence contracts add:
|
||||
**High-confidence contracts add:**
|
||||
|
||||
1. Tenant isolation.
|
||||
2. Auth and permission behavior.
|
||||
@@ -172,37 +223,32 @@ High-confidence contracts add:
|
||||
|
||||
Constructor routes, such as `POST /collection`:
|
||||
|
||||
1. Response has identity.
|
||||
2. Created resource is retrievable.
|
||||
3. Persisted fields reflect request fields.
|
||||
1. Resource is retrievable after creation.
|
||||
2. Persisted fields reflect request fields.
|
||||
|
||||
```apostl
|
||||
status:201
|
||||
response_body(this).id != null
|
||||
response_code(GET /items/{response_body(this).id}) == 200
|
||||
response_body(GET /items/{response_body(this).id}).name == request_body(this).name
|
||||
```
|
||||
|
||||
Mutator routes, such as `PUT`, `PATCH`, or action `POST`:
|
||||
|
||||
1. Mutation succeeds with expected code.
|
||||
2. Changed field actually changed.
|
||||
3. Unrelated invariants still hold.
|
||||
1. Changed field actually changed and persists.
|
||||
2. Unrelated invariants still hold.
|
||||
|
||||
```apostl
|
||||
status:200
|
||||
response_body(this).status == request_body(this).status
|
||||
response_body(this).updatedAt != null
|
||||
response_body(GET /items/{request_params(this).id}).status == request_body(this).status
|
||||
previous(response_body(this).version) < response_body(this).version
|
||||
```
|
||||
|
||||
Destructor routes:
|
||||
|
||||
1. Delete returns expected code.
|
||||
2. Follow-up retrieval fails or shows a domain-specific inactive state.
|
||||
1. Follow-up retrieval fails or shows a domain-specific inactive state.
|
||||
2. Previous state is preserved if the API returns deleted data.
|
||||
|
||||
```apostl
|
||||
status:204 || status:200
|
||||
response_code(GET /items/{request_params(this).id}) == 404
|
||||
response_body(this) == previous(response_body(GET /items/{request_params(this).id}))
|
||||
```
|
||||
|
||||
Observer routes:
|
||||
@@ -254,7 +300,7 @@ Use these patterns when they match the API:
|
||||
6. Error consistency: expected error status implies expected error payload fields.
|
||||
|
||||
```apostl
|
||||
if status:401 then response_body(this).error != null else true
|
||||
if status:401 then response_body(this).error.length > 0 else true
|
||||
if request_headers(this).x-tenant-id != null then response_headers(this).x-tenant-id == request_headers(this).x-tenant-id else true
|
||||
```
|
||||
|
||||
@@ -312,8 +358,9 @@ await app.apophis.scenario({
|
||||
steps: [
|
||||
{
|
||||
name: 'authorize',
|
||||
request: { method: 'GET', url: '/oauth/authorize?client_id=web&response_type=code' },
|
||||
expect: ['status:200', 'response_payload(this).code != null'],
|
||||
request: { method: 'GET', url: '/oauth/authorize?client_id=web&response_type=code&state=abc123' },
|
||||
// Behavioral: state parameter round-trips for CSRF protection
|
||||
expect: ['response_payload(this).state == request_query(this).state'],
|
||||
capture: { code: 'response_payload(this).code' }
|
||||
},
|
||||
{
|
||||
@@ -321,9 +368,10 @@ await app.apophis.scenario({
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: '/oauth/token',
|
||||
form: { grant_type: 'authorization_code', code: '$authorize.code' }
|
||||
form: { grant_type: 'authorization_code', code: '$authorize.code', scope: 'read' }
|
||||
},
|
||||
expect: ['status:200', 'response_payload(this).access_token != null']
|
||||
// Behavioral: issued token preserves the requested scope
|
||||
expect: ['response_payload(this).scope == request_body(this).scope']
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -343,24 +391,49 @@ Prefer deterministic verification for CI, regression triage, and AI-generated ch
|
||||
1. Capture and reuse seeds from verify and qualify runs.
|
||||
2. Use replay artifacts for failure triage before changing production logic.
|
||||
3. Preserve route identity as `METHOD /path` in notes and reports.
|
||||
4. If a failure is not reproducible, check for source drift, external dependencies, time, randomness, and insufficient cleanup before weakening the contract.
|
||||
5. Treat nondeterminism as a quality issue to isolate.
|
||||
4. **If a failure is not reproducible, treat it as a bug, not a flaky test.** Check for source drift, external dependencies, time, randomness, and insufficient cleanup. Do not weaken the contract without proving the implementation is correct.
|
||||
5. Treat nondeterminism as a quality issue to isolate and fix.
|
||||
|
||||
Operator framing:
|
||||
|
||||
> The failing seed gives us a reproducible behavioral example. I'll replay it first so we can distinguish a real regression from source drift or nondeterministic app state.
|
||||
> The failing seed gives us a reproducible behavioral counterexample. I'll replay it first to confirm the bug, then investigate the implementation before changing anything.
|
||||
|
||||
## Progressive Complexity
|
||||
|
||||
Start with behavioral contracts and add depth only where it pays off:
|
||||
|
||||
**Level 1 — Cross-route behavior**: Every constructor checks retrievability.
|
||||
```apostl
|
||||
response_code(GET /users/{response_body(this).id}) == 200
|
||||
```
|
||||
|
||||
**Level 2 — State persistence**: Mutators check that changes are visible.
|
||||
```apostl
|
||||
response_body(GET /users/{request_params(this).id}).email == request_body(this).email
|
||||
```
|
||||
|
||||
**Level 3 — Isolation and boundaries**: Tenant, auth, and idempotency checks.
|
||||
```apostl
|
||||
if request_headers(this).x-tenant-id != null then response_headers(this).x-tenant-id == request_headers(this).x-tenant-id else true
|
||||
```
|
||||
|
||||
**Level 4 — Protocol and dependency flows**: Variants, scenarios, outbound contracts, and chaos.
|
||||
|
||||
Add level 1 before level 4. Do not skip level 1 for resource APIs.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
Do not:
|
||||
|
||||
1. Assert only `status:200` everywhere.
|
||||
2. Duplicate JSON Schema checks while ignoring behavior.
|
||||
3. Encode route internals instead of API-observable outcomes.
|
||||
4. Ignore delete/retrieve or update/retrieve relationships.
|
||||
5. Treat stateful mode as optional for resource APIs.
|
||||
6. Ask the operator to review every formula before running; run first when intent is clear, then ask about ambiguous domain behavior.
|
||||
7. Load every doc file before making a small change.
|
||||
1. Assert only `status:200` everywhere. Schema validation already checks this.
|
||||
2. Check `response_body(this).id != null` when the schema already requires `id`.
|
||||
3. Duplicate JSON Schema checks while ignoring cross-route behavior.
|
||||
4. Encode route internals instead of API-observable outcomes.
|
||||
5. Ignore delete/retrieve or update/retrieve relationships.
|
||||
6. Treat stateful mode as optional for resource APIs.
|
||||
7. **Weaken a contract to make a test pass without proving the implementation is correct.**
|
||||
8. Ask the operator to review every formula before running; run first when intent is clear, then ask about ambiguous domain behavior.
|
||||
9. Load every doc file before making a small change.
|
||||
|
||||
## Verification Commands
|
||||
|
||||
@@ -378,7 +451,7 @@ Then execute APOPHIS from the project test harness or CLI as appropriate. For mo
|
||||
1. `README.md` for canonical usage.
|
||||
2. `docs/getting-started.md` for quick setup.
|
||||
3. `docs/cli.md` and command docs for CLI flags and machine output.
|
||||
4. `docs/protocol-extensions-spec.md` for protocol-specific direction.
|
||||
4. `docs/attic/protocol-extensions-spec.md` for protocol-specific direction.
|
||||
|
||||
## Final Check
|
||||
|
||||
@@ -388,5 +461,6 @@ For each route, ask:
|
||||
2. What must be true after this call?
|
||||
3. What related call should now behave differently?
|
||||
4. What isolation, security, dependency, or protocol expectation should not regress?
|
||||
5. If a contract fails, is the implementation wrong or is the contract wrong?
|
||||
|
||||
Write those expectations as formulas and run them continuously.
|
||||
Write those expectations as behavioral formulas, run them continuously, and treat every failure as a bug to investigate—not an obstacle to remove.
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* APOPHIS configuration
|
||||
* Generated by `apophis init`
|
||||
*/
|
||||
|
||||
import type { ApophisConfig } from "@apophis/fastify";
|
||||
|
||||
const config: ApophisConfig = {
|
||||
mode: "verify",
|
||||
profile: "llm-check",
|
||||
profiles: {
|
||||
"llm-check": {
|
||||
name: "llm-check",
|
||||
mode: "verify",
|
||||
preset: "llm-safe",
|
||||
routes: []
|
||||
}
|
||||
},
|
||||
presets: {
|
||||
"llm-safe": {
|
||||
name: "llm-safe",
|
||||
timeout: 3000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
observe: false
|
||||
}
|
||||
},
|
||||
environments: {
|
||||
local: {
|
||||
name: "local",
|
||||
allowVerify: true,
|
||||
allowObserve: false,
|
||||
allowQualify: false,
|
||||
allowChaos: false,
|
||||
allowBlocking: false,
|
||||
requireSink: false
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export default config;
|
||||
@@ -19,7 +19,7 @@ Five independent interface reviews (Substack/minimalist, Jared Hanson/DX, WebRef
|
||||
### Package Entry Point
|
||||
|
||||
```typescript
|
||||
import apophis from 'apophis-fastify'
|
||||
import apophis from '@apophis/fastify'
|
||||
```
|
||||
|
||||
The package exports one default: the Fastify plugin. No `export * from './types'`.
|
||||
|
||||
@@ -136,7 +136,7 @@ This section should be short and visual.
|
||||
Show exactly three commands:
|
||||
|
||||
```bash
|
||||
npm install apophis-fastify fastify @fastify/swagger
|
||||
npm install @apophis/fastify fastify @fastify/swagger
|
||||
apophis init --preset safe-ci
|
||||
apophis verify --profile quick --routes "POST /users"
|
||||
```
|
||||
|
||||
+4
-1
@@ -1,6 +1,9 @@
|
||||
## Outbound Contract-Driven Mocking Spec
|
||||
|
||||
Status: Proposed
|
||||
Status: Implemented (Phase 1)
|
||||
|
||||
Phase 1 (implemented): Schema parsing (`x-outbound`), mock runtime, imperative API (`enableOutboundMocks`, `getOutboundCalls`), fetch patching.
|
||||
Phase 2 (pending): APOSTL extensions `outbound_calls(this)` and `outbound_last(this)` for contract assertions.
|
||||
Date: 2026-04-27
|
||||
|
||||
This document supersedes Arbiter's local draft at `~/Business/workspace/Arbiter/docs/APOPHIS_OUTBOUND_MOCK_PROPOSAL.md` and its interim adapter at `~/Business/workspace/Arbiter/src/server/server/services/StripeFetchAdapter.js`.
|
||||
@@ -1,6 +1,10 @@
|
||||
# APOPHIS Plugin Contract System Specification
|
||||
|
||||
## Status: Active design; target version to be assigned
|
||||
## Status: Partially implemented
|
||||
|
||||
- Registry, types, and registration API: **implemented**
|
||||
- Runner integration (merging plugin contracts into route execution): **pending**
|
||||
- Built-in contracts for `@fastify/auth`, `@fastify/compress`, `@fastify/cors`, `@fastify/rate-limit`: **registered but not yet applied**
|
||||
|
||||
**Note**: Plugin contracts are complementary to Protocol Extensions (see `docs/protocol-extensions-spec.md`). Protocol extensions add domain-specific predicates (JWT, X.509, SPIFFE); plugin contracts add hook-phase behavioral contracts for Fastify plugins.
|
||||
|
||||
@@ -132,7 +132,7 @@ Catch behavioral regressions before merge with minimal setup.
|
||||
|
||||
Journey:
|
||||
|
||||
1. The team installs `apophis-fastify` and `@fastify/swagger`.
|
||||
1. The team installs `@apophis/fastify` and `@fastify/swagger`.
|
||||
2. The team runs `apophis init --preset safe-ci`.
|
||||
3. The CLI scaffolds a small config file, example route guidance, and a package script.
|
||||
4. The team adds one `x-ensures` contract to one critical route.
|
||||
|
||||
+25
-6
@@ -4,9 +4,28 @@ Archived design/planning documents that are no longer canonical for day-to-day u
|
||||
|
||||
Use `README.md` and `docs/getting-started.md` for current behavior and API guidance.
|
||||
|
||||
Archived items:
|
||||
- `docs/attic/API_REDESIGN_V1.md`
|
||||
- `docs/attic/QUALITY_FEATURES_PLAN.md`
|
||||
- `docs/attic/extensions/AUTH-RATE-LIMIT.md`
|
||||
- `docs/attic/extensions/WEBSOCKETS.md`
|
||||
- `docs/attic/root-history/` (historical feedback, plans, assessments, and analysis notes moved from repo root)
|
||||
Top-level:
|
||||
- `API_REDESIGN_V1.md` — original API surface redesign proposal
|
||||
- `BLOAT_ASSESSMENT.md` — codebase bloat audit notes
|
||||
- `CLI_EXECUTION_GUIDE.md` — early CLI invocation walkthrough
|
||||
- `chaos-v2.md` — chaos engine v2 design notes
|
||||
- `adoption-certification-scorecard.md` — adoption readiness scoring rubric
|
||||
- `fastify-structure.md` — Fastify integration architecture notes
|
||||
- `GITHUB_SITE_STRATEGY.md` — GitHub Pages / site planning
|
||||
- `homepage.md` — homepage content draft
|
||||
- `MULTI_FRAMEWORK_FEASIBILITY.md` — multi-framework support feasibility study
|
||||
- `OUTBOUND_CONTRACT_MOCKING_SPEC.md` — outbound contract mocking specification
|
||||
- `PLUGIN_CONTRACTS_SPEC.md` — plugin/contract interface specification
|
||||
- `protocol-extensions-spec.md` — protocol extension mechanism specification
|
||||
- `PUBLIC_INTERFACE_REDESIGN.md` — public API surface redesign
|
||||
- `QUALITY_FEATURES_PLAN.md` — quality feature roadmap
|
||||
- `TEST_AUDIT_REPORT.md` — test coverage audit report
|
||||
- `testing-pyramid.md` — testing pyramid / strategy notes
|
||||
|
||||
`extensions/`:
|
||||
- `AUTH-RATE-LIMIT.md` — original auth + rate-limit extension spec
|
||||
- `AUTH-RATE-LIMIT-REVISED.md` — revised auth + rate-limit design
|
||||
- `HTTP-EXTENSIONS.md` — HTTP extension mechanism notes
|
||||
- `WEBSOCKETS.md` — WebSocket extension design
|
||||
|
||||
`root-history/` — historical feedback, plans, assessments, and analysis notes moved from repo root (25 files)
|
||||
|
||||
@@ -78,7 +78,7 @@ Rate each dimension from **1 (poor)** to **5 (excellent)**.
|
||||
|
||||
| Dimension | Rating (1-5) | Evidence / Notes |
|
||||
|-----------|--------------|------------------|
|
||||
| Setup friction | 5 | `npm install apophis-fastify` + `npx apophis init` + `npx apophis verify` — three commands to first value. |
|
||||
| Setup friction | 5 | `npm install @apophis/fastify` + `npx apophis init` + `npx apophis verify` — three commands to first value. |
|
||||
| Time-to-first-value | 5 | Default `depth: 'quick'` runs in seconds. Immediate feedback on route contracts. |
|
||||
| CI confidence | 4 | `verify` in CI with `--format json-summary` gives pass/fail gate. Artifact retention allows post-hoc debugging. |
|
||||
| Replay reliability | 5 | `--replay` is single copy-paste command. Seed is printed in every failure. |
|
||||
@@ -131,7 +131,7 @@ Attach the following to this scorecard:
|
||||
|
||||
### Setup (all personas)
|
||||
```bash
|
||||
npm install apophis-fastify
|
||||
npm install @apophis/fastify
|
||||
npx apophis --help # exits 0
|
||||
npx apophis init # writes scaffold
|
||||
npx apophis doctor # passes
|
||||
|
||||
@@ -71,7 +71,7 @@ await fastify.apophis.contract({
|
||||
Wrap a `fetch` implementation so outbound requests are intercepted:
|
||||
|
||||
```javascript
|
||||
import { wrapFetch, createOutboundInterceptor } from 'apophis-fastify'
|
||||
import { wrapFetch, createOutboundInterceptor } from '@apophis/fastify'
|
||||
|
||||
const interceptor = createOutboundInterceptor([
|
||||
{
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
# APOPHIS v1.0 — Authentication, Authorization & Rate Limiting Extension (REVISED)
|
||||
|
||||
> **Status: NOT IMPLEMENTED**
|
||||
> This document describes a proposed extension that is not yet available in APOPHIS. The predicates, types, and infrastructure described here do not exist in the current codebase. Use `createAuthExtension` from `apophis-fastify/extension/factories` for auth testing today.
|
||||
> This document describes a proposed extension that is not yet available in APOPHIS. The predicates, types, and infrastructure described here do not exist in the current codebase. Use `createAuthExtension` from `@apophis/fastify/extension/factories` for auth testing today.
|
||||
|
||||
## 1. Overview
|
||||
|
||||
@@ -767,7 +767,7 @@ const contract: RouteContract = {
|
||||
|
||||
```typescript
|
||||
import fastify from 'fastify'
|
||||
import { apophisPlugin } from 'apophis-fastify'
|
||||
import { apophisPlugin } from '@apophis/fastify'
|
||||
|
||||
const app = fastify()
|
||||
|
||||
@@ -1230,7 +1230,7 @@ async cleanup(authContext?: AuthContext): Promise<Array<{ resource: TrackedResou
|
||||
|
||||
```typescript
|
||||
import fastify from 'fastify'
|
||||
import { apophisPlugin } from 'apophis-fastify'
|
||||
import { apophisPlugin } from '@apophis/fastify'
|
||||
|
||||
const app = fastify()
|
||||
|
||||
|
||||
@@ -219,7 +219,7 @@ Using `@fastify/websocket`:
|
||||
```typescript
|
||||
import fastify from 'fastify'
|
||||
import websocket from '@fastify/websocket'
|
||||
import apophis from 'apophis-fastify'
|
||||
import apophis from '@apophis/fastify'
|
||||
|
||||
const app = fastify()
|
||||
|
||||
@@ -1410,7 +1410,7 @@ const registerWebSocketValidation = (
|
||||
```typescript
|
||||
import fastify from 'fastify'
|
||||
import websocket from '@fastify/websocket'
|
||||
import apophis from 'apophis-fastify'
|
||||
import apophis from '@apophis/fastify'
|
||||
|
||||
const app = fastify()
|
||||
|
||||
|
||||
@@ -202,7 +202,7 @@ The production entry point imports the app factory, adds APOPHIS, connects to se
|
||||
|
||||
```typescript
|
||||
import { buildApp } from './app'
|
||||
import apophis from 'apophis-fastify'
|
||||
import apophis from '@apophis/fastify'
|
||||
|
||||
async function start() {
|
||||
const fastify = await buildApp({
|
||||
@@ -236,7 +236,7 @@ The test file creates a fresh app instance, registers APOPHIS, and runs contract
|
||||
|
||||
```typescript
|
||||
import { buildApp } from '../app'
|
||||
import apophis from 'apophis-fastify'
|
||||
import apophis from '@apophis/fastify'
|
||||
import type { FastifyInstance } from 'fastify'
|
||||
|
||||
export async function createTestApp(): Promise<FastifyInstance> {
|
||||
@@ -374,8 +374,10 @@ import { mkdirSync, rmSync } from 'fs'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
|
||||
let testCounter = 0
|
||||
|
||||
export function createTestWorkspace() {
|
||||
const dir = join(tmpdir(), `apophis-test-${Date.now()}`)
|
||||
const dir = join(tmpdir(), `apophis-test-${++testCounter}`)
|
||||
mkdirSync(dir, { recursive: true })
|
||||
|
||||
return {
|
||||
@@ -77,7 +77,7 @@ Production outages often come from behavior drift as well as invalid payload sha
|
||||
Three commands to the first targeted behavior check:
|
||||
|
||||
```bash
|
||||
npm install apophis-fastify fastify @fastify/swagger
|
||||
npm install @apophis/fastify fastify @fastify/swagger
|
||||
apophis init --preset safe-ci
|
||||
apophis verify --profile quick --routes "POST /users"
|
||||
```
|
||||
@@ -113,7 +113,7 @@ See [docs/llm-safe-adoption.md](docs/llm-safe-adoption.md) for templates and CI
|
||||
## Operator Resources
|
||||
|
||||
- [Troubleshooting matrix](docs/troubleshooting.md) — Categorized failure classes with resolution steps
|
||||
- [Adoption certification scorecard](docs/adoption-certification-scorecard.md) — Review template for team rollout
|
||||
- [Adoption certification scorecard](adoption-certification-scorecard.md) — Review template for team rollout
|
||||
|
||||
## CTAs
|
||||
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
# APOPHIS Protocol Extensions Specification
|
||||
|
||||
## Status: Active design; shipped baseline: v2.x; remaining targets listed per feature
|
||||
## Status: Active design; shipped baseline: v2.0.0; remaining targets listed per feature
|
||||
|
||||
## 1. Overview
|
||||
|
||||
This specification defines protocol-specific extensions for APOPHIS, driven by the Arbiter team's requirements for testing OAuth 2.1, WIMSE S2S, Transaction Tokens (RFC 8693), SPIFFE/SPIRE, and related security protocols.
|
||||
|
||||
APOPHIS is grounded in [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021). Protocol extensions add domain-specific predicates (JWT, X.509, SPIFFE) to the core invariant framework.
|
||||
|
||||
Arbiter maintains 58 protocol conformance test files covering 138 behaviors across 7 specifications. These extensions bridge the gap between declarative APOSTL contracts and the domain-specific predicates required for security protocol validation.
|
||||
|
||||
### 1.1 Current Shipped vs Not-Shipped Snapshot
|
||||
|
||||
**Shipped in v2.x:**
|
||||
**Shipped in v2.0.0:**
|
||||
|
||||
- `contract({ variants })` for multi-header/media negotiation execution.
|
||||
- `fastify.apophis.scenario(...)` for multi-step capture/rebind flows.
|
||||
@@ -166,12 +168,15 @@ jwtExtension({
|
||||
The JWT extension maintains state across a test run:
|
||||
|
||||
```javascript
|
||||
interface JwtExtensionState {
|
||||
/** Track seen JTIs for replay detection */
|
||||
seenJtis: Set<string>
|
||||
/** Cached decoded JWTs */
|
||||
decodedCache: Map<string, DecodedJwt>
|
||||
}
|
||||
/**
|
||||
* JWT extension state across a test run.
|
||||
* @property {Set<string>} seenJtis - Track seen JTIs for replay detection
|
||||
* @property {Map<string, DecodedJwt>} decodedCache - Cached decoded JWTs
|
||||
*/
|
||||
const jwtExtensionState = {
|
||||
seenJtis: new Set(),
|
||||
decodedCache: new Map()
|
||||
};
|
||||
```
|
||||
|
||||
### 3.5 Example Contracts
|
||||
@@ -234,16 +239,19 @@ await fastify.apophis.time.set('2026-04-25T12:00:00Z');
|
||||
### 4.4 Implementation
|
||||
|
||||
```javascript
|
||||
interface TimeControl {
|
||||
/** Advance simulated time by milliseconds */
|
||||
advance(ms: number): void
|
||||
/** Set simulated time to specific timestamp */
|
||||
set(isoString: string): void
|
||||
/** Get current simulated time */
|
||||
now(): number
|
||||
/** Reset to real time */
|
||||
reset(): void
|
||||
}
|
||||
/**
|
||||
* Time control for deterministic testing.
|
||||
* @property {function(number): void} advance - Advance simulated time by milliseconds
|
||||
* @property {function(string): void} set - Set simulated time to specific ISO timestamp
|
||||
* @property {function(): number} now - Get current simulated time
|
||||
* @property {function(): void} reset - Reset to real time
|
||||
*/
|
||||
const timeControl = {
|
||||
advance(ms) { /* ... */ },
|
||||
set(isoString) { /* ... */ },
|
||||
now() { return Date.now(); },
|
||||
reset() { /* ... */ }
|
||||
};
|
||||
```
|
||||
|
||||
The `now()` predicate returns simulated time when time mocking is enabled, or the host wall clock outside deterministic test mode. Deterministic runs must inject or freeze time.
|
||||
@@ -288,11 +296,17 @@ previous(observer).jwt_claims(this).jti # last observer's JWT ID
|
||||
Extension state tracks tokens across requests:
|
||||
|
||||
```javascript
|
||||
interface StatefulExtensionState {
|
||||
seenTokens: Set<string>
|
||||
consumedTokens: Set<string>
|
||||
categoryHistory: Map<string, EvalContext> // category -> last context
|
||||
}
|
||||
/**
|
||||
* Stateful extension state tracking tokens across requests.
|
||||
* @property {Set<string>} seenTokens - Tokens observed in previous requests
|
||||
* @property {Set<string>} consumedTokens - Tokens that have been consumed
|
||||
* @property {Map<string, EvalContext>} categoryHistory - category -> last context
|
||||
*/
|
||||
const statefulExtensionState = {
|
||||
seenTokens: new Set(),
|
||||
consumedTokens: new Set(),
|
||||
categoryHistory: new Map()
|
||||
};
|
||||
```
|
||||
|
||||
### 5.4 Example Contracts
|
||||
@@ -522,14 +536,14 @@ We acknowledge these are too complex or inappropriate for Apophis:
|
||||
|
||||
## 14. Implementation Plan
|
||||
|
||||
### Phase 1: JWT + Time Control (P0)
|
||||
**Target**: v1.3.0
|
||||
### Phase 1: JWT + Time Control (P0) — Shipped in v2.0.0
|
||||
**Status**: Complete
|
||||
**Files**:
|
||||
- `src/extensions/jwt.ts` — JWT extension implementation
|
||||
- `src/extensions/time.ts` — Time control extension
|
||||
- `src/extensions/stateful.ts` — Stateful predicates extension
|
||||
- `src/test/jwt-extension.test.ts` — JWT tests
|
||||
- `src/test/time-extension.test.ts` — Time control tests
|
||||
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||
- `src/test/cli/protocol-conformance-p2.test.ts` — Protocol conformance tests
|
||||
|
||||
**Tests**:
|
||||
- Decode Base64URL claims without verification
|
||||
@@ -539,27 +553,25 @@ We acknowledge these are too complex or inappropriate for Apophis:
|
||||
- `now()` predicate with mocked time
|
||||
- `apophis.time.advance()` in stateful tests
|
||||
|
||||
### Phase 2: X.509 + SPIFFE (P1)
|
||||
**Target**: v1.3.1
|
||||
### Phase 2: X.509 + SPIFFE (P1) — Shipped in v2.0.0
|
||||
**Status**: Complete
|
||||
**Files**:
|
||||
- `src/extensions/x509.ts` — X.509 extension
|
||||
- `src/extensions/spiffe.ts` — SPIFFE extension
|
||||
- `src/test/x509-extension.test.ts` — X.509 tests
|
||||
- `src/test/spiffe-extension.test.ts` — SPIFFE tests
|
||||
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||
|
||||
### Phase 3: Token Hash + HTTP Signature (P2)
|
||||
**Target**: v1.3.2
|
||||
### Phase 3: Token Hash + HTTP Signature (P2) — Shipped in v2.0.0
|
||||
**Status**: Complete
|
||||
**Files**:
|
||||
- `src/extensions/token-hash.ts` — Token hash extension
|
||||
- `src/extensions/http-signature.ts` — HTTP signature extension
|
||||
- `src/test/token-hash-extension.test.ts` — Token hash tests
|
||||
- `src/test/http-signature-extension.test.ts` — HTTP signature tests
|
||||
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||
|
||||
### Phase 4: Request Context (P2)
|
||||
**Target**: v1.3.3
|
||||
### Phase 4: Request Context (P2) — Shipped in v2.0.0
|
||||
**Status**: Complete
|
||||
**Files**:
|
||||
- `src/extensions/request-context.ts` — Request context predicates
|
||||
- `src/test/request-context-extension.test.ts` — Request context tests
|
||||
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||
|
||||
---
|
||||
|
||||
@@ -750,7 +750,7 @@ async function apophisPlugin(fastify, options) {
|
||||
}
|
||||
|
||||
module.exports = fp(apophisPlugin, {
|
||||
name: 'apophis-fastify',
|
||||
name: '@apophis/fastify',
|
||||
dependencies: ['@fastify/swagger']
|
||||
})
|
||||
```
|
||||
@@ -1879,7 +1879,7 @@ class ApophisSymbolicAnalyzer {
|
||||
## File Structure
|
||||
|
||||
```
|
||||
apophis-fastify/
|
||||
@apophis/fastify/
|
||||
├── lib/
|
||||
│ ├── formula-parser.js # APOSTL formula parsing and evaluation
|
||||
│ ├── formula-substitutor.js # Safe parameter substitution
|
||||
@@ -1916,7 +1916,7 @@ apophis-fastify/
|
||||
const fastify = require('fastify')()
|
||||
|
||||
// Register APOPHIS (registers @fastify/swagger automatically)
|
||||
await fastify.register(require('apophis-fastify'), {
|
||||
await fastify.register(require('@apophis/fastify'), {
|
||||
swagger: {
|
||||
openapi: '3.0.0',
|
||||
info: { title: 'Tournaments API', version: '1.0.0' }
|
||||
|
||||
@@ -271,4 +271,4 @@ As it stands, APOPHIS is a promising research project that teaches us a lot abou
|
||||
|
||||
*Assessment by Charity Majors, co-founder Honeycomb.io*
|
||||
*Date: 2026-04-25*
|
||||
*Framework: apophis-fastify v1.1.0*
|
||||
*Framework: @apophis/fastify v1.1.0*
|
||||
@@ -12,14 +12,14 @@ A complete "Hello World" to "Production Ready" guide that a developer can follow
|
||||
|
||||
#### 1.1 Installation (30 seconds)
|
||||
```bash
|
||||
npm install apophis-fastify
|
||||
npm install @apophis/fastify
|
||||
# peer deps: fastify, @fastify/swagger
|
||||
```
|
||||
|
||||
#### 1.2 Minimal Setup (2 minutes)
|
||||
```typescript
|
||||
import Fastify from 'fastify'
|
||||
import apophisPlugin from 'apophis-fastify'
|
||||
import apophisPlugin from '@apophis/fastify'
|
||||
|
||||
const fastify = Fastify()
|
||||
|
||||
@@ -358,7 +358,7 @@ jobs:
|
||||
#### 3.4 Cache Configuration API
|
||||
```typescript
|
||||
// Programmatic control
|
||||
import { invalidateRoutes, invalidateCache } from 'apophis-fastify/incremental/cache'
|
||||
import { invalidateRoutes, invalidateCache } from '@apophis/fastify/incremental/cache'
|
||||
|
||||
// Before test run
|
||||
invalidateRoutes(['/users']) // Invalidate specific routes
|
||||
|
||||
@@ -6,10 +6,10 @@ APOPHIS generates requests automatically. For authenticated routes, you need to
|
||||
|
||||
## The Pattern: `createAuthExtension`
|
||||
|
||||
Use `createAuthExtension` from `apophis-fastify` to inject credentials into every request:
|
||||
Use `createAuthExtension` from `@apophis/fastify` to inject credentials into every request:
|
||||
|
||||
```javascript
|
||||
import { createAuthExtension } from 'apophis-fastify'
|
||||
import { createAuthExtension } from '@apophis/fastify'
|
||||
|
||||
const jwtAuth = createAuthExtension({
|
||||
name: 'jwt',
|
||||
@@ -135,7 +135,7 @@ fastify.get('/wimse/wit', {
|
||||
})
|
||||
```
|
||||
|
||||
See `docs/protocol-extensions-spec.md` for full JWT extension configuration.
|
||||
See `docs/attic/protocol-extensions-spec.md` for full JWT extension configuration.
|
||||
|
||||
---
|
||||
|
||||
@@ -144,7 +144,7 @@ See `docs/protocol-extensions-spec.md` for full JWT extension configuration.
|
||||
`getToken` runs per request. Handle refresh inline:
|
||||
|
||||
```javascript
|
||||
let cachedToken: string | null = null
|
||||
let cachedToken = null
|
||||
|
||||
const auth = createAuthExtension({
|
||||
name: 'jwt-with-refresh',
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Cache & CI/CD Integration
|
||||
|
||||
> **Note**: The incremental cache is an internal mechanism used by programmatic test runners (petit-runner, stateful-runner). CLI users do not interact with `.apophis-cache.json` directly. This document describes the internal caching behavior for advanced programmatic integration.
|
||||
|
||||
APOPHIS includes an incremental test cache that speeds up test runs by skipping unchanged routes. This document covers cache invalidation strategies and CI/CD integration.
|
||||
|
||||
## How the Cache Works
|
||||
@@ -28,10 +30,10 @@ Each entry is keyed by a hash of the route's path, method, and schema. If the sc
|
||||
|
||||
| Environment | Cache | Reason |
|
||||
|-------------|-------|--------|
|
||||
| `production` | Disabled | No file I/O, no cache hits needed |
|
||||
| `test` | Disabled | Tests should be deterministic, no cache pollution |
|
||||
| `development` | Enabled | Speeds up iterative testing |
|
||||
| default | Enabled | Backward compatible |
|
||||
| `production` | Enabled by default | Set `APOPHIS_DISABLE_CACHE=1` to opt-out |
|
||||
| `test` | Enabled by default | Set `APOPHIS_DISABLE_CACHE=1` to opt-out |
|
||||
| `development` | Enabled by default | Speeds up iterative testing |
|
||||
| default | Enabled by default | Backward compatible |
|
||||
|
||||
## Cache Invalidation
|
||||
|
||||
|
||||
+34
-49
@@ -2,19 +2,20 @@
|
||||
|
||||
Inject controlled failures into contract tests to validate resilience guarantees.
|
||||
|
||||
Chaos testing applies invariant-driven verification under adverse conditions: if a contract must hold, it should still hold when dependencies fail, responses are delayed, or payloads are corrupted.
|
||||
|
||||
## Usage
|
||||
|
||||
```typescript
|
||||
```javascript
|
||||
const result = await fastify.apophis.contract({
|
||||
depth: 'standard',
|
||||
runs: 50,
|
||||
chaos: {
|
||||
probability: 0.1, // 10% of requests get chaos
|
||||
delay: { probability: 1, minMs: 100, maxMs: 500 },
|
||||
error: { probability: 1, statusCode: 503 },
|
||||
dropout: { probability: 1 },
|
||||
corruption: { probability: 1 },
|
||||
delay: { probability: 0.1, minMs: 100, maxMs: 500 },
|
||||
error: { probability: 0.1, statusCode: 503 },
|
||||
dropout: { probability: 0.05 },
|
||||
corruption: { probability: 0.1 },
|
||||
},
|
||||
})
|
||||
});
|
||||
```
|
||||
|
||||
## Event Types
|
||||
@@ -24,16 +25,18 @@ const result = await fastify.apophis.contract({
|
||||
Adds artificial latency. Tests timeout contracts:
|
||||
|
||||
```apostl
|
||||
timeout_occurred(this) == false
|
||||
response_time(this) < 1000
|
||||
```
|
||||
|
||||
Delay chaos strategies (`inbound-delay`, `outbound-delay`) are applied at the transport level between request execution and contract evaluation. The inline chaos handlers for these strategies are no-ops because `sleep()` handles delay application out-of-band. Delay contracts such as `response_time(this) < 1000` will still work correctly with chaos injection.
|
||||
|
||||
### Error
|
||||
|
||||
Forces HTTP status codes. Tests error-handling contracts:
|
||||
|
||||
```apostl
|
||||
if status:503 then response_body(this).retry_after != null
|
||||
// Behavioral: when the service is unavailable, the client receives a valid retry signal
|
||||
if status:503 then response_headers(this).retry-after > 0
|
||||
```
|
||||
|
||||
### Dropout
|
||||
@@ -41,7 +44,8 @@ if status:503 then response_body(this).retry_after != null
|
||||
Simulates network failure (status 0). Tests fallback contracts:
|
||||
|
||||
```apostl
|
||||
status:200 || status:0
|
||||
// Behavioral: partial failure must still return previously cached data
|
||||
if status:0 then response_body(this).cached_data == previous(response_body(GET /cache/{request_params(this).key}))
|
||||
```
|
||||
|
||||
### Corruption
|
||||
@@ -49,48 +53,30 @@ status:200 || status:0
|
||||
Mutates response bodies. Tests parsing robustness:
|
||||
|
||||
```apostl
|
||||
response_body(this).id != null
|
||||
// Behavioral: corrupted requests maintain traceability for debugging
|
||||
if status:400 then response_body(this).request_id == request_headers(this).x-request-id
|
||||
```
|
||||
|
||||
## Content-Type Aware Corruption
|
||||
## Corruption Strategies
|
||||
|
||||
Built-in strategies for common formats:
|
||||
Built-in strategies are content-type agnostic:
|
||||
|
||||
| Content-Type | Strategy | Effect |
|
||||
|-------------|----------|--------|
|
||||
| `application/json` | Truncate or null field | Removes fields or sets random field to null |
|
||||
| `application/x-ndjson` | Chunk corrupt | Corrupts one NDJSON chunk |
|
||||
| `text/event-stream` | Event corrupt | Adds malformed SSE line |
|
||||
| `multipart/form-data` | Field corrupt | Replaces field with corrupted data |
|
||||
| `text/plain` | Truncate | Cuts string in half |
|
||||
| Strategy | Effect |
|
||||
|----------|--------|
|
||||
| `truncate` | Cuts response body short |
|
||||
| `malformed` | Invalidates structural boundaries (e.g., unclosed JSON, bad headers) |
|
||||
| `field-corrupt` | Replaces a random field value with corrupted data |
|
||||
|
||||
## Custom Corruption via Extensions
|
||||
Extension strategies can add content-type-specific behavior if needed.
|
||||
|
||||
```typescript
|
||||
const myExtension = {
|
||||
name: 'custom-corrupt',
|
||||
corruptionStrategies: {
|
||||
'application/vnd.api+json': (data) => ({
|
||||
...data as object,
|
||||
corrupted: true,
|
||||
}),
|
||||
'text/*': (data) => `CORRUPTED:${String(data)}`,
|
||||
},
|
||||
}
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [myExtension],
|
||||
})
|
||||
```
|
||||
|
||||
Extension strategies take precedence over built-ins. Wildcard patterns (`text/*`) match any subtype.
|
||||
**Note**: Extension-defined corruption strategies are documented for future implementation. Currently, corruption strategies (`truncate`, `malformed`, `field-corrupt`) are hardcoded in the chaos engine.
|
||||
|
||||
## Environment Guard
|
||||
|
||||
Low-level contract chaos APIs require `NODE_ENV=test`. For CLI qualification, environment policy controls whether chaos gates may run.
|
||||
|
||||
```
|
||||
Error: Chaos mode is only available in test environment.
|
||||
Error: chaos is only available in test environment. Set NODE_ENV=test to enable quality features.
|
||||
```
|
||||
|
||||
## Interpreting Results
|
||||
@@ -100,7 +86,7 @@ Failed tests include chaos events in diagnostics:
|
||||
```json
|
||||
{
|
||||
"statusCode": 503,
|
||||
"error": "Contract violation: status:200",
|
||||
"error": "Contract violation: if status:503 then response_headers(this).retry-after > 0",
|
||||
"chaosEvents": [
|
||||
{
|
||||
"type": "error",
|
||||
@@ -118,26 +104,25 @@ Failed tests include chaos events in diagnostics:
|
||||
|
||||
1. **Start small**: `probability: 0.05` (5% of requests)
|
||||
2. **Test one failure mode at a time**: Comment out other chaos types
|
||||
3. **Verify contracts handle chaos**: `if status:503 then response_body(this).error != null`
|
||||
3. **Verify contracts handle chaos**: `if status:503 then response_code(GET /health) == 200`
|
||||
4. **Use seeds for reproducibility**: `seed: 42` makes chaos deterministic
|
||||
|
||||
## Example: Testing Retry Logic
|
||||
|
||||
```typescript
|
||||
```javascript
|
||||
fastify.get('/data', {
|
||||
schema: {
|
||||
'x-ensures': [
|
||||
'if status:503 then response_headers(this).retry-after != null',
|
||||
'if status:503 then response_headers(this).retry-after > 0',
|
||||
'redirect_count(this) <= 3',
|
||||
],
|
||||
},
|
||||
}, handler)
|
||||
}, handler);
|
||||
|
||||
// Test
|
||||
const result = await fastify.apophis.contract({
|
||||
chaos: {
|
||||
probability: 0.2,
|
||||
error: { probability: 1, statusCode: 503 },
|
||||
error: { probability: 0.2, statusCode: 503 },
|
||||
},
|
||||
})
|
||||
});
|
||||
```
|
||||
|
||||
+53
-60
@@ -10,15 +10,16 @@ Every command accepts these flags:
|
||||
|---|---|---|
|
||||
| `--config <path>` | Config file path | Auto-detect |
|
||||
| `--profile <name>` | Profile name from config | First profile |
|
||||
| `--generation-profile <name>` | Generation budget profile (built-in or config alias) | Depth-derived |
|
||||
| `--cwd <path>` | Working directory override | `process.cwd()` |
|
||||
| `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` | `human` |
|
||||
| `--color <mode>` | Color mode: `auto`, `always`, `never` | `auto` |
|
||||
| `--quiet` | Suppress non-error output | false |
|
||||
| `--verbose` | Enable verbose logging | false |
|
||||
| `--artifact-dir <path>` | Directory for artifact output | `reports/apophis/` |
|
||||
| `--artifact-dir <path>` | Directory for artifact output. Artifacts written on failure or when explicitly configured. | `reports/apophis/` |
|
||||
| `--workspace` | Run supported commands across workspace packages | false |
|
||||
|
||||
Note: `json-summary` and `ndjson-summary` are only supported by `verify` and `qualify` commands.
|
||||
|
||||
## Commands
|
||||
|
||||
### `apophis init`
|
||||
@@ -37,16 +38,15 @@ apophis init --preset safe-ci
|
||||
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--preset <name>` | Preset name: `safe-ci`, `platform-observe`, `llm-safe`, `protocol-lab` |
|
||||
| `--force` | Overwrite existing files |
|
||||
| `-p, --preset <name>` | Preset name: `safe-ci`, `platform-observe`, `llm-safe`, `protocol-lab` |
|
||||
| `-f, --force` | Overwrite existing files |
|
||||
| `--noninteractive` | Skip all prompts, require explicit flags |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
apophis init --preset safe-ci
|
||||
apophis init --preset llm-safe --force
|
||||
apophis init --preset platform-observe --noninteractive
|
||||
<!-- smoke-test -->
|
||||
apophis init --help
|
||||
```
|
||||
|
||||
### `apophis verify`
|
||||
@@ -58,21 +58,19 @@ apophis verify --profile quick --routes "POST /users"
|
||||
```
|
||||
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
|---|---|---|
|
||||
| `--profile <name>` | Profile name from config |
|
||||
| `--generation-profile <name>` | Override generation budget for this run |
|
||||
| `--routes <filter>` | Route filter pattern (comma-separated, supports wildcards) |
|
||||
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
||||
| `--changed` | Filter to git-modified routes only |
|
||||
| `--changed` | Filter to git-modified routes only |
|
||||
| `--workspace` | Run across all workspace packages |
|
||||
| `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
apophis verify --profile quick
|
||||
apophis verify --routes "POST /users" --seed 42
|
||||
apophis verify --changed
|
||||
apophis verify --profile ci --routes "POST /users,PUT /users/*"
|
||||
<!-- smoke-test -->
|
||||
apophis verify --help
|
||||
```
|
||||
|
||||
**Machine output for CI:**
|
||||
@@ -87,26 +85,6 @@ apophis verify --profile quick --format json-summary
|
||||
apophis verify --profile quick --format ndjson-summary
|
||||
```
|
||||
|
||||
### `apophis observe`
|
||||
|
||||
Validate runtime observe configuration and reporting setup.
|
||||
|
||||
```bash
|
||||
apophis observe --profile staging-observe
|
||||
```
|
||||
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--profile <name>` | Profile name from config |
|
||||
| `--check-config` | Only validate config, do not activate |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
apophis observe --profile staging-observe
|
||||
apophis observe --check-config
|
||||
```
|
||||
|
||||
### `apophis qualify`
|
||||
|
||||
Run scenario, stateful, protocol, or chaos-driven qualification.
|
||||
@@ -118,26 +96,13 @@ apophis qualify --profile oauth-nightly --seed 42
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--profile <name>` | Profile name from config |
|
||||
| `--generation-profile <name>` | Override generation budget for this run |
|
||||
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
apophis qualify --profile oauth-nightly --seed 42
|
||||
apophis qualify --profile lifecycle-deep
|
||||
apophis qualify --profile oauth-nightly --generation-profile quick
|
||||
```
|
||||
|
||||
You can define aliases in config:
|
||||
|
||||
```js
|
||||
export default {
|
||||
generationProfiles: {
|
||||
pr: 'quick',
|
||||
nightly: { base: 'thorough' },
|
||||
},
|
||||
}
|
||||
<!-- smoke-test -->
|
||||
apophis qualify --help
|
||||
```
|
||||
|
||||
### `apophis replay`
|
||||
@@ -156,7 +121,8 @@ apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
apophis replay --artifact reports/apophis/failure-*.json
|
||||
<!-- smoke-test -->
|
||||
apophis replay --help
|
||||
```
|
||||
|
||||
### `apophis doctor`
|
||||
@@ -171,6 +137,7 @@ apophis doctor [--mode verify|observe|qualify] [--strict]
|
||||
|---|---|
|
||||
| `--mode <mode>` | Filter checks to a specific mode |
|
||||
| `--strict` | Treat warnings as failures |
|
||||
| `--workspace` | Run across all workspace packages |
|
||||
|
||||
**Checks:**
|
||||
|
||||
@@ -184,8 +151,8 @@ apophis doctor [--mode verify|observe|qualify] [--strict]
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
apophis doctor
|
||||
apophis doctor --verbose
|
||||
<!-- smoke-test -->
|
||||
apophis doctor --help
|
||||
```
|
||||
|
||||
### `apophis migrate`
|
||||
@@ -205,11 +172,35 @@ apophis migrate --check
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
apophis migrate --check
|
||||
apophis migrate --dry-run
|
||||
apophis migrate --write
|
||||
<!-- smoke-test -->
|
||||
apophis migrate --help
|
||||
```
|
||||
|
||||
## Common Tasks
|
||||
|
||||
### CI workflow with machine output
|
||||
```bash
|
||||
apophis verify --profile ci --format json-summary --artifact-dir reports/apophis
|
||||
```
|
||||
|
||||
### Monorepo workspace verification
|
||||
```bash
|
||||
apophis verify --workspace --profile quick
|
||||
apophis doctor --workspace
|
||||
```
|
||||
|
||||
### Replay a failure
|
||||
```bash
|
||||
apophis replay --artifact reports/apophis/failure-*.json
|
||||
```
|
||||
|
||||
## Gotchas
|
||||
|
||||
- `--changed` requires a git repository
|
||||
- `migrate` defaults to `--dry-run` (safe by default)
|
||||
- `--workspace` is fully implemented by `verify` and `doctor`. Other commands do not support `--workspace`.
|
||||
- Seeds ensure deterministic generation; handler nondeterminism (e.g., `Date.now()`) can still cause replay divergence
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Code | Meaning |
|
||||
@@ -226,10 +217,12 @@ apophis migrate --write
|
||||
|---|---|---|---|---|
|
||||
| `verify` | enabled | enabled | optional | optional, usually off |
|
||||
| `observe` | optional | optional | enabled | enabled |
|
||||
| `qualify: scenario` | enabled | enabled | enabled with allowlist | disabled by default |
|
||||
| `qualify: stateful` | enabled | enabled | synthetic-only | disabled by default |
|
||||
| `qualify: chaos` | enabled | enabled | canary-only | disabled by default |
|
||||
| outbound mocks | enabled | enabled | allowlisted only | disabled by default |
|
||||
| `qualify` | enabled | enabled | optional | disabled by default |
|
||||
| `chaos` | enabled | enabled | optional | disabled by default |
|
||||
| runtime throw-on-violation | optional | optional | exceptional | disabled by default |
|
||||
|
||||
Operational rule: Production must never inherit qualify capabilities accidentally from a generic config file.
|
||||
Notes:
|
||||
- `qualify` is gated as a whole. The code does not distinguish scenario, stateful, and chaos sub-modes in environment policy.
|
||||
- `chaos` on protected routes requires `allowChaosOnProtected: true`.
|
||||
- `observe` blocking requires `allowBlocking: true`.
|
||||
- Production must never inherit qualify capabilities accidentally from a generic config file.
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import Fastify from 'fastify'
|
||||
import apophisPlugin from '@apophis/fastify'
|
||||
import crypto from 'crypto'
|
||||
|
||||
const fastify = Fastify()
|
||||
|
||||
import { createAuthExtension } from '@apophis/fastify/extension/factories'
|
||||
|
||||
const authExtension = createAuthExtension({
|
||||
name: 'bearer',
|
||||
acquire: async () => {
|
||||
// In real apps, this would call a login endpoint
|
||||
const token = crypto.randomBytes(32).toString('hex')
|
||||
return { token, userId: 'tester-1' }
|
||||
},
|
||||
})
|
||||
|
||||
await fastify.register(apophisPlugin, {
|
||||
runtime: 'error',
|
||||
extensions: [authExtension],
|
||||
})
|
||||
|
||||
const sessions = new Map<string, { userId: string; createdAt: number }>()
|
||||
|
||||
// LOGIN — acquires a session token
|
||||
fastify.post('/auth/login', {
|
||||
schema: {
|
||||
'x-category': 'constructor',
|
||||
'x-ensures': [
|
||||
'response_body(this).token != null',
|
||||
'response_body(this).expiresAt > request_time(this)',
|
||||
],
|
||||
body: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
username: { type: 'string' },
|
||||
password: { type: 'string' },
|
||||
},
|
||||
required: ['username', 'password'],
|
||||
},
|
||||
response: {
|
||||
200: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
token: { type: 'string' },
|
||||
expiresAt: { type: 'number' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (req) => {
|
||||
const token = crypto.randomBytes(48).toString('hex')
|
||||
const expiresAt = Date.now() + 3600_000
|
||||
sessions.set(token, { userId: `usr-${req.body.username}`, createdAt: Date.now() })
|
||||
return { token, expiresAt }
|
||||
})
|
||||
|
||||
// PROTECTED RESOURCE — requires valid auth
|
||||
fastify.get('/auth/me', {
|
||||
schema: {
|
||||
'x-category': 'observer',
|
||||
'x-requires': [
|
||||
'response_status(this) == 200',
|
||||
],
|
||||
'x-ensures': [
|
||||
'response_body(this).userId != null',
|
||||
'response_body(this).authenticated == true',
|
||||
],
|
||||
headers: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
authorization: { type: 'string', pattern: '^Bearer ' },
|
||||
},
|
||||
required: ['authorization'],
|
||||
},
|
||||
response: {
|
||||
200: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
userId: { type: 'string' },
|
||||
authenticated: { type: 'boolean' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (req, reply) => {
|
||||
const header = req.headers.authorization
|
||||
if (!header) {
|
||||
reply.status(401)
|
||||
return { error: 'Missing Authorization header' }
|
||||
}
|
||||
const token = header.replace('Bearer ', '')
|
||||
const session = sessions.get(token)
|
||||
if (!session) {
|
||||
reply.status(401)
|
||||
return { error: 'Invalid or expired token' }
|
||||
}
|
||||
return { userId: session.userId, authenticated: true }
|
||||
})
|
||||
|
||||
// LOGOUT — destroys a session, must not succeed twice
|
||||
fastify.post('/auth/logout', {
|
||||
schema: {
|
||||
'x-category': 'destructor',
|
||||
'x-requires': [
|
||||
'response_status(this) == 200',
|
||||
],
|
||||
'x-ensures': [
|
||||
// After logout, the same token should be rejected
|
||||
'response_code(GET /auth/me) == 401',
|
||||
],
|
||||
headers: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
authorization: { type: 'string', pattern: '^Bearer ' },
|
||||
},
|
||||
required: ['authorization'],
|
||||
},
|
||||
},
|
||||
}, async (req, reply) => {
|
||||
const header = req.headers.authorization
|
||||
if (!header) {
|
||||
reply.status(401)
|
||||
return { error: 'Missing Authorization header' }
|
||||
}
|
||||
const token = header.replace('Bearer ', '')
|
||||
const existed = sessions.delete(token)
|
||||
if (!existed) {
|
||||
reply.status(404)
|
||||
return { error: 'Session not found' }
|
||||
}
|
||||
return { ok: true }
|
||||
})
|
||||
|
||||
await fastify.ready()
|
||||
|
||||
const result = await fastify.apophis.contract({ runs: 30 })
|
||||
console.log('Contract tests:', result.summary)
|
||||
|
||||
const stateful = await fastify.apophis.stateful({ runs: 30, seed: 42 })
|
||||
console.log('Stateful tests:', stateful.summary)
|
||||
+46
-28
@@ -1,24 +1,32 @@
|
||||
import Fastify from 'fastify'
|
||||
import apophisPlugin from 'apophis-fastify'
|
||||
import apophisPlugin from '@apophis/fastify'
|
||||
import crypto from 'crypto'
|
||||
|
||||
const fastify = Fastify()
|
||||
|
||||
await fastify.register(apophisPlugin, {
|
||||
runtime: 'error', // Validate contracts on every request
|
||||
cleanup: true, // Auto-cleanup resources on exit
|
||||
runtime: 'error',
|
||||
cleanup: true,
|
||||
})
|
||||
|
||||
// In-memory store for demo
|
||||
const users = new Map<string, { id: string; email: string; name: string }>()
|
||||
|
||||
// CREATE — constructor
|
||||
// Behavioral: the created user must be retrievable.
|
||||
// Note: we do not write 'status:201' or 'response_body(this).id != null'.
|
||||
// The schema already validates status codes and required fields.
|
||||
// Contracts should test behavior across operations, not structure.
|
||||
fastify.post('/users', {
|
||||
schema: {
|
||||
'x-category': 'constructor',
|
||||
'x-ensures': [
|
||||
'status:201',
|
||||
// The response must have an id assigned by the server
|
||||
'response_body(this).id != null',
|
||||
// The echoed fields must match the request
|
||||
'response_body(this).email == request_body(this).email',
|
||||
'response_body(this).name == request_body(this).name',
|
||||
// Cross-route: the created user must be retrievable
|
||||
'response_code(GET /users/{response_body(this).id}) == 200',
|
||||
],
|
||||
body: {
|
||||
type: 'object',
|
||||
@@ -40,7 +48,7 @@ fastify.post('/users', {
|
||||
}
|
||||
}
|
||||
}, async (req, reply) => {
|
||||
const id = `usr-${Date.now()}`
|
||||
const id = `usr-${crypto.createHash('sha256').update(req.body.email).digest('hex').slice(0, 8)}`
|
||||
const user = { id, email: req.body.email, name: req.body.name }
|
||||
users.set(id, user)
|
||||
reply.status(201)
|
||||
@@ -48,19 +56,21 @@ fastify.post('/users', {
|
||||
})
|
||||
|
||||
// READ — observer
|
||||
// Behavioral: the returned user must match the requested id.
|
||||
fastify.get('/users/:id', {
|
||||
schema: {
|
||||
'x-category': 'observer',
|
||||
'x-requires': ['users:id'],
|
||||
'x-requires': [
|
||||
// Precondition: the user must exist for this read to be valid
|
||||
'response_code(GET /users/{request_params(this).id}) == 200'
|
||||
],
|
||||
'x-ensures': [
|
||||
'status:200',
|
||||
// The returned id must match the requested id (no mix-up)
|
||||
'response_body(this).id == request_params(this).id',
|
||||
],
|
||||
params: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: { type: 'string' }
|
||||
},
|
||||
properties: { id: { type: 'string' } },
|
||||
required: ['id']
|
||||
},
|
||||
response: {
|
||||
@@ -83,19 +93,21 @@ fastify.get('/users/:id', {
|
||||
})
|
||||
|
||||
// UPDATE — mutator
|
||||
// Behavioral: after update, the change must be visible on read.
|
||||
fastify.put('/users/:id', {
|
||||
schema: {
|
||||
'x-category': 'mutator',
|
||||
'x-requires': ['users:id'],
|
||||
'x-requires': [
|
||||
// The user must exist before updating
|
||||
'response_code(GET /users/{request_params(this).id}) == 200'
|
||||
],
|
||||
'x-ensures': [
|
||||
'status:200',
|
||||
'response_body(this).id == request_params(this).id',
|
||||
// Cross-route: after update, reading the user shows the new data
|
||||
'response_body(GET /users/{request_params(this).id}).email == request_body(this).email',
|
||||
],
|
||||
params: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: { type: 'string' }
|
||||
},
|
||||
properties: { id: { type: 'string' } },
|
||||
required: ['id']
|
||||
},
|
||||
body: {
|
||||
@@ -131,34 +143,40 @@ fastify.put('/users/:id', {
|
||||
})
|
||||
|
||||
// DELETE — destructor
|
||||
// Behavioral: after deletion, the user must no longer exist.
|
||||
fastify.delete('/users/:id', {
|
||||
schema: {
|
||||
'x-category': 'destructor',
|
||||
'x-requires': ['users:id'],
|
||||
'x-ensures': ['status:204'],
|
||||
'x-requires': [
|
||||
// The user must exist before deleting
|
||||
'response_code(GET /users/{request_params(this).id}) == 200'
|
||||
],
|
||||
'x-ensures': [
|
||||
// After deletion, the user is gone
|
||||
'response_code(GET /users/{request_params(this).id}) == 404',
|
||||
// The deleted user data is returned (matches pre-deletion read)
|
||||
'response_body(this) == previous(response_body(GET /users/{request_params(this).id}))',
|
||||
],
|
||||
params: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: { type: 'string' }
|
||||
},
|
||||
properties: { id: { type: 'string' } },
|
||||
required: ['id']
|
||||
}
|
||||
}
|
||||
}, async (req, reply) => {
|
||||
const user = users.get(req.params.id)
|
||||
users.delete(req.params.id)
|
||||
reply.status(204)
|
||||
reply.status(200)
|
||||
return user
|
||||
})
|
||||
|
||||
await fastify.ready()
|
||||
|
||||
// Run contract tests (all non-utility routes, property-based)
|
||||
const result = await fastify.apophis.contract({ depth: 'standard' })
|
||||
const result = await fastify.apophis.contract({ runs: 50 })
|
||||
console.log('Contract tests:', result.summary)
|
||||
|
||||
// Run stateful tests (constructor→mutator→destructor sequences)
|
||||
const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 })
|
||||
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||
console.log('Stateful tests:', stateful.summary)
|
||||
|
||||
// Validate a single route
|
||||
const check = await fastify.apophis.check('POST', '/users')
|
||||
console.log('POST /users check:', check.ok ? 'PASS' : 'FAIL')
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import Fastify from 'fastify'
|
||||
import apophisPlugin from '@apophis/fastify'
|
||||
import crypto from 'crypto'
|
||||
|
||||
const fastify = Fastify()
|
||||
|
||||
await fastify.register(apophisPlugin, {
|
||||
runtime: 'error',
|
||||
cleanup: true,
|
||||
})
|
||||
|
||||
const processedKeys = new Set<string>()
|
||||
const ledger = new Map<string, { idempotencyKey: string; amount: number; createdAt: number }>()
|
||||
|
||||
// PAYMENT — idempotent, must not double-charge
|
||||
fastify.post('/payments', {
|
||||
schema: {
|
||||
'x-category': 'mutator',
|
||||
'x-requires': [
|
||||
'request_header(this).idempotency-key != null',
|
||||
],
|
||||
'x-ensures': [
|
||||
'response_status(this) == 201 || response_status(this) == 200',
|
||||
'response_body(this).id != null',
|
||||
// Core idempotency assurance: same key, same result
|
||||
'response_body(this).amount == request_body(this).amount',
|
||||
],
|
||||
headers: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
'idempotency-key': { type: 'string' },
|
||||
},
|
||||
required: ['idempotency-key'],
|
||||
},
|
||||
body: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
amount: { type: 'number', minimum: 1 },
|
||||
currency: { type: 'string', minLength: 3, maxLength: 3 },
|
||||
},
|
||||
required: ['amount', 'currency'],
|
||||
},
|
||||
response: {
|
||||
201: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: { type: 'string' },
|
||||
idempotencyKey: { type: 'string' },
|
||||
amount: { type: 'number' },
|
||||
status: { type: 'string' },
|
||||
},
|
||||
},
|
||||
200: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: { type: 'string' },
|
||||
idempotencyKey: { type: 'string' },
|
||||
amount: { type: 'number' },
|
||||
status: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (req, reply) => {
|
||||
const key = req.headers['idempotency-key'] as string
|
||||
|
||||
// Idempotency check: if already processed, return the saved result
|
||||
if (processedKeys.has(key)) {
|
||||
for (const record of ledger.values()) {
|
||||
if (record.idempotencyKey === key) {
|
||||
return { ...record, status: 'duplicate' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const id = `pay-${crypto.randomUUID().slice(0, 8)}`
|
||||
const record = {
|
||||
id,
|
||||
idempotencyKey: key,
|
||||
amount: req.body.amount,
|
||||
status: 'processed',
|
||||
createdAt: Date.now(),
|
||||
}
|
||||
|
||||
processedKeys.add(key)
|
||||
ledger.set(id, record)
|
||||
reply.status(201)
|
||||
return record
|
||||
})
|
||||
|
||||
// REFUND — idempotent reversal
|
||||
fastify.post('/payments/:id/refund', {
|
||||
schema: {
|
||||
'x-category': 'mutator',
|
||||
'x-requires': [
|
||||
'response_code(GET /payments/{request_params(this).id}) == 200',
|
||||
],
|
||||
'x-ensures': [
|
||||
'response_body(this).refundId != null',
|
||||
'response_body(this).amount == previous(response_body(GET /payments/{request_params(this).id})).amount',
|
||||
// Refund must be reversible only once
|
||||
],
|
||||
headers: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
'idempotency-key': { type: 'string' },
|
||||
},
|
||||
required: ['idempotency-key'],
|
||||
},
|
||||
params: {
|
||||
type: 'object',
|
||||
properties: { id: { type: 'string' } },
|
||||
required: ['id'],
|
||||
},
|
||||
response: {
|
||||
201: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
refundId: { type: 'string' },
|
||||
amount: { type: 'number' },
|
||||
status: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (req, reply) => {
|
||||
const payment = ledger.get(req.params.id)
|
||||
if (!payment) {
|
||||
reply.status(404)
|
||||
return { error: 'Payment not found' }
|
||||
}
|
||||
|
||||
const key = req.headers['idempotency-key'] as string
|
||||
if (processedKeys.has(key)) {
|
||||
return {
|
||||
refundId: `ref-${payment.id}`,
|
||||
amount: payment.amount,
|
||||
status: 'duplicate',
|
||||
}
|
||||
}
|
||||
|
||||
processedKeys.add(key)
|
||||
reply.status(201)
|
||||
return {
|
||||
refundId: `ref-${payment.id}`,
|
||||
amount: payment.amount,
|
||||
status: 'refunded',
|
||||
}
|
||||
})
|
||||
|
||||
// PAYMENT STATUS — observer
|
||||
fastify.get('/payments/:id', {
|
||||
schema: {
|
||||
'x-category': 'observer',
|
||||
'x-requires': [
|
||||
'response_status(this) == 200 || response_status(this) == 404',
|
||||
],
|
||||
params: {
|
||||
type: 'object',
|
||||
properties: { id: { type: 'string' } },
|
||||
required: ['id'],
|
||||
},
|
||||
},
|
||||
}, async (req, reply) => {
|
||||
const payment = ledger.get(req.params.id)
|
||||
if (!payment) {
|
||||
reply.status(404)
|
||||
return { error: 'Payment not found' }
|
||||
}
|
||||
return payment
|
||||
})
|
||||
|
||||
await fastify.ready()
|
||||
|
||||
const result = await fastify.apophis.contract({ runs: 50 })
|
||||
console.log('Contract tests:', result.summary)
|
||||
|
||||
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||
console.log('Stateful tests:', stateful.summary)
|
||||
@@ -1,26 +1,35 @@
|
||||
import Fastify from 'fastify'
|
||||
import apophisPlugin from 'apophis-fastify'
|
||||
import apophisPlugin from '@apophis/fastify'
|
||||
|
||||
const fastify = Fastify()
|
||||
|
||||
// APOPHIS auto-registers @fastify/swagger
|
||||
await fastify.register(apophisPlugin, {})
|
||||
|
||||
fastify.get('/health', {
|
||||
// Behavioral contract: what you send is what you get back.
|
||||
// This is not a structural test — the schema already validates shape.
|
||||
// This checks that the server does not mutate or drop fields.
|
||||
fastify.post('/echo', {
|
||||
schema: {
|
||||
'x-category': 'observer',
|
||||
'x-ensures': ['status:200'],
|
||||
'x-ensures': [
|
||||
'response_body(this) == request_body(this)'
|
||||
],
|
||||
body: {
|
||||
type: 'object',
|
||||
properties: { message: { type: 'string' } }
|
||||
},
|
||||
response: {
|
||||
200: {
|
||||
type: 'object',
|
||||
properties: { status: { type: 'string' } }
|
||||
properties: { message: { type: 'string' } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}, async () => ({ status: 'ok' }))
|
||||
}, async (req) => req.body)
|
||||
|
||||
await fastify.ready()
|
||||
|
||||
// Run contract tests
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
console.log(result.summary)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# APOPHIS v1.1 Architecture — Hybrid Core + Extensions
|
||||
|
||||
> **HISTORICAL**: This is a v1.1 architecture design specification. For current extension documentation, see [EXTENSION-PLUGIN-SYSTEM.md](EXTENSION-PLUGIN-SYSTEM.md) and [QUICK-REFERENCE.md](QUICK-REFERENCE.md).
|
||||
|
||||
## Status: Architecture Specification
|
||||
## Date: 2026-04-24
|
||||
## Scope: v1.1 First-Class Features & Extension Ecosystem
|
||||
|
||||
@@ -89,7 +89,7 @@ type PredicateResolver = (context: PredicateContext) =>
|
||||
## 4. Example: Arbiter Extension
|
||||
|
||||
```typescript
|
||||
import type { ApophisExtension, PredicateContext } from 'apophis-fastify'
|
||||
import type { ApophisExtension, PredicateContext } from '@apophis/fastify'
|
||||
import { createArbiter } from 'arbiter-sdk'
|
||||
|
||||
const arbiterExtension: ApophisExtension = {
|
||||
@@ -263,7 +263,7 @@ const arbiterExtension: ApophisExtension = {
|
||||
|
||||
```typescript
|
||||
import fastify from 'fastify'
|
||||
import apophis from 'apophis-fastify'
|
||||
import apophis from '@apophis/fastify'
|
||||
import { arbiterExtension } from './arbiter-extension.js'
|
||||
|
||||
const app = fastify()
|
||||
@@ -280,8 +280,8 @@ app.get('/users/:id', {
|
||||
type: 'object',
|
||||
properties: { id: { type: 'string' } },
|
||||
'x-ensures': [
|
||||
// Standard APOSTL + extension predicates
|
||||
'status:200',
|
||||
// Behavioral: returned user must match the requested id
|
||||
'response_body(this).id == request_params(this).id',
|
||||
'graph_check(this).user.can_read_user == true',
|
||||
'partial_graph(this).tenant.accessible == true',
|
||||
],
|
||||
@@ -392,7 +392,7 @@ await app.register(apophis, {
|
||||
| `src/test/extension.test.ts` | Extension system tests |
|
||||
| `src/formula/evaluator.ts` | APOSTL evaluator with extension predicate resolution |
|
||||
| `src/domain/contract-validation.ts` | Passes extension registry to evaluator |
|
||||
| `src/test/petit-runner.ts` | Calls extension hooks |
|
||||
| `src/quality/petit-runner.ts` | Calls extension hooks |
|
||||
| `src/plugin/index.ts` | Creates and passes ExtensionRegistry |
|
||||
|
||||
---
|
||||
|
||||
@@ -46,23 +46,26 @@ await fastify.apophis.contract({
|
||||
})
|
||||
```
|
||||
|
||||
### wrapFetch for Outbound Interception
|
||||
### Outbound Mocking
|
||||
|
||||
Use `fastify.apophis.test.enableOutboundMocks()` in test code to mock HTTP dependencies:
|
||||
|
||||
```typescript
|
||||
import { wrapFetch, createOutboundInterceptor } from 'apophis-fastify'
|
||||
|
||||
const interceptor = createOutboundInterceptor([
|
||||
{
|
||||
target: 'api.stripe.com',
|
||||
delay: { probability: 0.1, minMs: 1000, maxMs: 5000 },
|
||||
error: {
|
||||
probability: 0.05,
|
||||
responses: [{ statusCode: 429, headers: { 'retry-after': '60' } }]
|
||||
}
|
||||
fastify.apophis.test.registerOutboundContracts({
|
||||
'payment-api': {
|
||||
target: 'https://api.stripe.com/v1',
|
||||
method: 'POST',
|
||||
response: { 200: { type: 'object', properties: { id: { type: 'string' } } } }
|
||||
}
|
||||
], 42)
|
||||
})
|
||||
fastify.apophis.test.enableOutboundMocks({ mode: 'example' })
|
||||
const calls = fastify.apophis.test.getOutboundCalls('payment-api')
|
||||
```
|
||||
|
||||
const interceptedFetch = wrapFetch(globalThis.fetch, interceptor)
|
||||
Programmatic access via `createOutboundMockRuntime`:
|
||||
|
||||
```typescript
|
||||
import { createOutboundMockRuntime } from '@apophis/fastify'
|
||||
```
|
||||
|
||||
### Mutation Testing
|
||||
@@ -70,7 +73,7 @@ const interceptedFetch = wrapFetch(globalThis.fetch, interceptor)
|
||||
Measure contract strength by injecting synthetic bugs:
|
||||
|
||||
```typescript
|
||||
import { runMutationTesting } from 'apophis-fastify/quality/mutation'
|
||||
import { runMutationTesting } from '@apophis/fastify/quality/mutation'
|
||||
|
||||
const report = await runMutationTesting(fastify)
|
||||
console.log(`Score: ${report.score}%`) // 0-100
|
||||
@@ -190,7 +193,7 @@ Extensions register custom APOSTL predicates that can be used in `x-ensures` and
|
||||
**Register via `extensions: [sseExtension]`**
|
||||
|
||||
```typescript
|
||||
import { sseExtension } from 'apophis-fastify/extensions/sse'
|
||||
import { sseExtension } from '@apophis/fastify/extensions/sse'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [sseExtension]
|
||||
@@ -236,7 +239,7 @@ sse_events(this).0.retry // number (ms)
|
||||
**Register via `extensions: [createSerializerExtension(registry)]`**
|
||||
|
||||
```typescript
|
||||
import { createSerializerExtension, createSerializerRegistry } from 'apophis-fastify/extensions/serializers'
|
||||
import { createSerializerExtension, createSerializerRegistry } from '@apophis/fastify/extensions/serializers'
|
||||
|
||||
const registry = createSerializerRegistry()
|
||||
registry.register('protobuf', {
|
||||
@@ -273,7 +276,7 @@ fastify.post('/users', {
|
||||
**Register via `extensions: [websocketExtension]`**
|
||||
|
||||
```typescript
|
||||
import { websocketExtension } from 'apophis-fastify/extensions/websocket'
|
||||
import { websocketExtension } from '@apophis/fastify/extensions/websocket'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [websocketExtension]
|
||||
@@ -320,7 +323,7 @@ ws_state(this) // string
|
||||
**Register via `extensions: [jwtExtension(config)]`**
|
||||
|
||||
```typescript
|
||||
import { jwtExtension } from 'apophis-fastify/extensions'
|
||||
import { jwtExtension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [
|
||||
@@ -348,7 +351,7 @@ jwt_format(this) == "compact"
|
||||
**Register via `extensions: [x509Extension(config)]`**
|
||||
|
||||
```typescript
|
||||
import { x509Extension } from 'apophis-fastify/extensions'
|
||||
import { x509Extension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [x509Extension()]
|
||||
@@ -370,7 +373,7 @@ x509_self_signed(this) == false
|
||||
**Register via `extensions: [spiffeExtension(config)]`**
|
||||
|
||||
```typescript
|
||||
import { spiffeExtension } from 'apophis-fastify/extensions'
|
||||
import { spiffeExtension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [spiffeExtension()]
|
||||
@@ -391,7 +394,7 @@ spiffe_validate(this) == true
|
||||
**Register via `extensions: [tokenHashExtension(config)]`**
|
||||
|
||||
```typescript
|
||||
import { tokenHashExtension } from 'apophis-fastify/extensions'
|
||||
import { tokenHashExtension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [tokenHashExtension()]
|
||||
@@ -412,7 +415,7 @@ token_hash(this, "sha256") == jwt_claims(this).ath
|
||||
**Register via `extensions: [httpSignatureExtension(config)]`**
|
||||
|
||||
```typescript
|
||||
import { httpSignatureExtension } from 'apophis-fastify/extensions'
|
||||
import { httpSignatureExtension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [httpSignatureExtension()]
|
||||
@@ -433,7 +436,7 @@ signature_valid(this) == true
|
||||
**Register via `extensions: [timeExtension(config)]`**
|
||||
|
||||
```typescript
|
||||
import { timeExtension } from 'apophis-fastify/extensions'
|
||||
import { timeExtension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [timeExtension()]
|
||||
@@ -453,7 +456,7 @@ jwt_claims(this).exp <= now() + 30000
|
||||
**Register via `extensions: [statefulExtension()]`**
|
||||
|
||||
```typescript
|
||||
import { statefulExtension } from 'apophis-fastify/extensions'
|
||||
import { statefulExtension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [statefulExtension()]
|
||||
@@ -521,7 +524,7 @@ fastify.get('/tenants/:id', {
|
||||
**Register via `extensions: [requestContextExtension(config)]`**
|
||||
|
||||
```typescript
|
||||
import { requestContextExtension } from 'apophis-fastify/extensions'
|
||||
import { requestContextExtension } from '@apophis/fastify/extensions'
|
||||
|
||||
await fastify.register(apophis, {
|
||||
extensions: [requestContextExtension()]
|
||||
@@ -555,19 +558,7 @@ await fastify.apophis.contract({
|
||||
|
||||
### Outbound Interception
|
||||
|
||||
```typescript
|
||||
import { wrapFetch, createOutboundInterceptor } from 'apophis-fastify'
|
||||
|
||||
const interceptor = createOutboundInterceptor([{
|
||||
target: 'api.stripe.com',
|
||||
error: {
|
||||
probability: 0.05,
|
||||
responses: [{ statusCode: 429, headers: { 'retry-after': '60' } }]
|
||||
}
|
||||
}], 42)
|
||||
|
||||
const interceptedFetch = wrapFetch(globalThis.fetch, interceptor)
|
||||
```
|
||||
Outbound interception works through `fastify.apophis.test.enableOutboundMocks()` in test code. See the [Outbound Mocking](#outbound-mocking) section for the supported API.
|
||||
|
||||
### Per-Route Overrides
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# APOPHIS v1.0 Extension Specification: Timeouts and Redirects
|
||||
# APOPHIS Extension Specification: Timeouts and Redirects
|
||||
|
||||
## Document Information
|
||||
- **Version**: 1.0
|
||||
@@ -64,6 +64,8 @@ fastify.get('/slow-endpoint', {
|
||||
|
||||
When a timeout is configured, `executeHttp` uses an abortable timer where supported. The timeout must be cleared in `finally`; Fastify injection may continue running after timeout if the underlying transport cannot be cancelled.
|
||||
|
||||
> **Note**: The code examples in this section are illustrative representations of the timeout, redirect, and concurrency mechanisms. The actual implementation may differ in detail.
|
||||
|
||||
```typescript
|
||||
// In src/infrastructure/http-executor.ts
|
||||
if (timeoutMs && timeoutMs > 0) {
|
||||
@@ -284,8 +286,8 @@ fastify.get('/api/resource', {
|
||||
'x-ensures': [
|
||||
'timeout_occurred(this) == false',
|
||||
'redirect_count(this) == 0',
|
||||
'response_code(this) == 200',
|
||||
'response_body(this).id != null',
|
||||
// Behavioral: created resource must be retrievable
|
||||
'response_code(GET /api/resource/{response_body(this).id}) == 200',
|
||||
]
|
||||
}
|
||||
}, handler)
|
||||
|
||||
+188
-111
@@ -2,21 +2,25 @@
|
||||
|
||||
Get from install to your first behavioral bug in 10 minutes.
|
||||
|
||||
APOPHIS is inspired by the concept of invariant-driven automated testing: instead of only validating request and response shape, encode intended behavior as executable contracts and let the tool find violations automatically.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js 20.x or 22.x
|
||||
- Node.js >=20.18.1 (20.x) or >=22 (22.x)
|
||||
- **Fastify v5** (v4 is not supported)
|
||||
- **ESM project** (`"type": "module"` in package.json)
|
||||
- A Fastify app with `@fastify/swagger` registered
|
||||
|
||||
## Step 1: Install
|
||||
|
||||
```bash
|
||||
npm install apophis-fastify fastify @fastify/swagger
|
||||
npm install @apophis/fastify fastify @fastify/swagger
|
||||
```
|
||||
|
||||
## Step 2: Scaffold
|
||||
|
||||
```bash
|
||||
apophis init --preset safe-ci
|
||||
npx apophis init --preset safe-ci
|
||||
```
|
||||
|
||||
This creates:
|
||||
@@ -30,6 +34,8 @@ This creates:
|
||||
Pick one important route. Add an `x-ensures` clause that checks behavior across operations:
|
||||
|
||||
```javascript
|
||||
import crypto from 'crypto';
|
||||
|
||||
app.post('/users', {
|
||||
schema: {
|
||||
'x-category': 'constructor',
|
||||
@@ -40,27 +46,20 @@ app.post('/users', {
|
||||
}
|
||||
}, async (request, reply) => {
|
||||
const { name } = request.body;
|
||||
const id = `usr-${Date.now()}`;
|
||||
const id = `usr-${crypto.createHash('sha256').update(name).digest('hex').slice(0, 8)}`;
|
||||
reply.status(201);
|
||||
return { id, name };
|
||||
});
|
||||
```
|
||||
|
||||
> **Warning:** Using `Date.now()` or `Math.random()` in handlers breaks determinism and replay. Use a stable function of the input instead. APOPHIS does not proactively detect nondeterministic handlers; it warns only when a replay diverges from the original run.
|
||||
|
||||
## Step 4: Run Verify
|
||||
|
||||
```bash
|
||||
apophis verify --profile quick --routes "POST /users"
|
||||
npx apophis verify --profile quick --routes "POST /users"
|
||||
```
|
||||
|
||||
APOPHIS will:
|
||||
|
||||
1. Discover routes from your Fastify app
|
||||
2. Filter to `POST /users`
|
||||
3. Generate test data from the schema
|
||||
4. Execute the route
|
||||
5. Check the behavioral contract
|
||||
6. Print pass/fail, seed, and replay command
|
||||
|
||||
## Example Failure
|
||||
|
||||
If your `GET /users/:id` handler has a bug (always returns 404), APOPHIS catches it:
|
||||
@@ -75,7 +74,7 @@ Expected
|
||||
response_code(GET /users/{response_body(this).id}) == 200
|
||||
|
||||
Observed
|
||||
GET /users/usr-123 returned 404
|
||||
GET /users/usr-7d865e returned 404
|
||||
|
||||
Why this matters
|
||||
The resource created by POST /users is not retrievable.
|
||||
@@ -97,114 +96,192 @@ apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
|
||||
|
||||
Fix the bug in your handler. Re-run verify. The failure should now pass.
|
||||
|
||||
## Behavioral vs Structural Contracts
|
||||
|
||||
APOPHIS contracts should verify **behavior**, not structure. Fastify and `@fastify/swagger` already enforce status codes, required fields, and types. Behavioral contracts catch what schemas cannot:
|
||||
|
||||
| Structural (avoid) | Behavioral (prefer) |
|
||||
|---|---|
|
||||
| `status:200` | `response_body(this) == request_body(this)` |
|
||||
| `response_body(this).id != null` | `response_code(GET /users/{response_body(this).id}) == 200` |
|
||||
| `response_body(this).name != null` | `response_body(GET /users/{id}).name == previous(response_body(this).name)` |
|
||||
|
||||
**Good behavioral patterns (from the paper):**
|
||||
- **Constructor precondition**: Resource must not exist before creation
|
||||
```apostl
|
||||
response_code(GET /users/{request_body(this).email}) == 404
|
||||
```
|
||||
- **Round-trip equality**: POST response matches the request body
|
||||
```apostl
|
||||
response_body(this) == request_body(this)
|
||||
```
|
||||
- **Cross-route retrievability**: Creating a resource makes it readable via GET
|
||||
```apostl
|
||||
response_code(GET /users/{response_body(this).id}) == 200
|
||||
```
|
||||
- **State-change verification**: DELETE causes subsequent GET to return 404
|
||||
```apostl
|
||||
response_code(GET /users/{request_params(this).id}) == 404
|
||||
```
|
||||
- **Previous state preservation**: DELETE returns the last known state
|
||||
```apostl
|
||||
response_body(this) == previous(response_body(GET /users/{request_params(this).id}))
|
||||
```
|
||||
- **Invariant over collections**: All resources satisfy a cross-resource constraint
|
||||
```apostl
|
||||
for t in response_body(GET /tournaments) :-
|
||||
response_body(GET /tournaments/{t.id}/players).length <= t.capacity
|
||||
```
|
||||
|
||||
**Anti-patterns to avoid:**
|
||||
- Checking status codes (handled by schema validation)
|
||||
- Checking field existence (handled by schema validation)
|
||||
- Checking field types (handled by schema validation)
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Add more routes to your profile: `apophis verify --profile quick --routes "POST /users,PUT /users/:id"`
|
||||
- Use wildcards to match route patterns: `apophis verify --routes 'POST /api/*'`
|
||||
- Run all routes: `apophis verify --profile quick`
|
||||
- Run only changed routes in CI: `apophis verify --profile ci --changed`
|
||||
- Add observe mode for runtime drift detection: see [docs/observe.md](docs/observe.md)
|
||||
- Add qualify mode for scenario, stateful, and chaos checks: see [docs/qualify.md](docs/qualify.md)
|
||||
- Run only changed routes in CI: `apophis verify --profile quick --changed`
|
||||
- Requires a git repository.
|
||||
- Use machine-readable output in CI: `apophis verify --profile ci --format json-summary`
|
||||
- Add observe mode for runtime drift detection: see [observe.md](observe.md)
|
||||
- Add qualify mode for scenario, stateful, and chaos checks: see [qualify.md](qualify.md)
|
||||
|
||||
## Variants
|
||||
|
||||
Test the same route with different headers or content types:
|
||||
|
||||
```javascript
|
||||
await fastify.apophis.contract({
|
||||
variants: [
|
||||
{ name: 'json', headers: { accept: 'application/json' } },
|
||||
{ name: 'xml', headers: { accept: 'application/xml' } }
|
||||
]
|
||||
})
|
||||
```
|
||||
|
||||
Or declare variants in the route schema:
|
||||
|
||||
```javascript
|
||||
app.get('/users', {
|
||||
schema: {
|
||||
'x-variants': [
|
||||
{ name: 'json', headers: { accept: 'application/json' } }
|
||||
]
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## Plugin Options
|
||||
|
||||
When registering the APOPHIS plugin, you can pass these options:
|
||||
|
||||
```javascript
|
||||
await fastify.register(apophis, {
|
||||
// Swagger config passthrough (if @fastify/swagger is not already registered)
|
||||
swagger: { openapi: { info: { title: 'API', version: '1.0.0' } } },
|
||||
|
||||
// Runtime contract validation hooks: 'off', 'warn', or 'error'
|
||||
// Only active in non-production environments
|
||||
runtime: 'warn',
|
||||
|
||||
// Automatically clean up tracked resources after tests
|
||||
cleanup: true,
|
||||
|
||||
// Global timeout in milliseconds for all requests
|
||||
timeout: 5000,
|
||||
|
||||
// Tenant isolation scopes
|
||||
scopes: {
|
||||
tenant1: { headers: { 'x-tenant-id': '1' } },
|
||||
tenant2: { headers: { 'x-tenant-id': '2' } },
|
||||
},
|
||||
|
||||
// Auth and protocol extensions
|
||||
extensions: [jwtAuth, apiKeyAuth],
|
||||
|
||||
// Plugin hook-phase contracts
|
||||
pluginContracts: {
|
||||
'rate-limit': { appliesTo: 'POST /users', ensures: ['status != 429'] },
|
||||
},
|
||||
|
||||
// Outbound dependency contracts
|
||||
outboundContracts: {
|
||||
'payment-api': {
|
||||
target: 'https://payments.example.com',
|
||||
method: 'POST',
|
||||
response: { 200: { type: 'object', properties: { id: { type: 'string' } } } }
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## Schema Annotations
|
||||
|
||||
APOPHIS reads these OpenAPI schema extensions:
|
||||
|
||||
| Annotation | Location | Description |
|
||||
|---|---|---|
|
||||
| `x-category` | Top-level | Route classification: `constructor`, `mutator`, `observer`, `destructor`, `utility` |
|
||||
| `x-ensures` | Top-level or `response[statusCode]` | Post-condition contracts (APOSTL formulas) |
|
||||
| `x-requires` | Top-level or `response[statusCode]` | Pre-condition contracts (APOSTL formulas) |
|
||||
| `x-variants` | Top-level | Request variants for content-type negotiation or feature flags |
|
||||
| `x-timeout` | Top-level or `response[statusCode]` | Per-route timeout in milliseconds |
|
||||
| `x-outbound` | Top-level | Outbound dependency contracts for this route |
|
||||
| `x-streaming` | Top-level | Mark route as streaming (populates `chunks` and `streamDurationMs` in eval context) |
|
||||
| `x-validate-runtime` | Top-level or `response[statusCode]` | Toggle runtime validation for this route (default: true) |
|
||||
| `x-extension-config` | Top-level | Per-route config for extensions (e.g., `{ jwt: { verify: false } }`) |
|
||||
|
||||
Annotations can be placed on the top-level schema or nested inside `response[statusCode]`. Nested annotations from the first status code schema are merged with top-level annotations.
|
||||
|
||||
## Programmatic API
|
||||
|
||||
After registration, `fastify.apophis` provides:
|
||||
|
||||
```javascript
|
||||
// Run contract tests for all routes
|
||||
const suite = await fastify.apophis.contract({ runs: 50, seed: 42 })
|
||||
|
||||
// Run stateful tests
|
||||
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||
|
||||
// Run a single scenario
|
||||
const scenario = await fastify.apophis.scenario({
|
||||
name: 'oauth-basic',
|
||||
steps: [...]
|
||||
})
|
||||
|
||||
// Check a single route
|
||||
const result = await fastify.apophis.check('GET', '/users/:id')
|
||||
|
||||
// Get enriched OpenAPI spec with contract metadata
|
||||
const spec = fastify.apophis.spec()
|
||||
|
||||
// Clean up tracked resources
|
||||
await fastify.apophis.cleanup()
|
||||
|
||||
// Test-only utilities (NODE_ENV=test only)
|
||||
fastify.apophis.test.registerPluginContracts('name', spec)
|
||||
fastify.apophis.test.registerOutboundContracts({ ... })
|
||||
fastify.apophis.test.enableOutboundMocks({ mode: 'example' })
|
||||
fastify.apophis.test.disableOutboundMocks()
|
||||
const calls = fastify.apophis.test.getOutboundCalls('payment-api')
|
||||
```
|
||||
|
||||
Outbound mocking patches `globalThis.fetch` and is process-global. Only one mock runtime can be installed at a time. Run mock-dependent tests serially or isolate by process.
|
||||
|
||||
## Config Reference
|
||||
|
||||
```javascript
|
||||
// apophis.config.js
|
||||
export default {
|
||||
mode: 'verify',
|
||||
profile: 'quick',
|
||||
profiles: {
|
||||
quick: {
|
||||
name: 'quick',
|
||||
mode: 'verify',
|
||||
preset: 'safe-ci',
|
||||
routes: ['POST /users']
|
||||
},
|
||||
ci: {
|
||||
name: 'ci',
|
||||
mode: 'verify',
|
||||
preset: 'safe-ci',
|
||||
routes: []
|
||||
}
|
||||
},
|
||||
presets: {
|
||||
'safe-ci': {
|
||||
name: 'safe-ci',
|
||||
depth: 'quick',
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
observe: false
|
||||
}
|
||||
},
|
||||
environments: {
|
||||
local: {
|
||||
name: 'local',
|
||||
allowVerify: true,
|
||||
allowObserve: true,
|
||||
allowQualify: false,
|
||||
allowChaos: false,
|
||||
allowBlocking: true,
|
||||
requireSink: false
|
||||
}
|
||||
}
|
||||
};
|
||||
```
|
||||
For the full configuration reference, see [CLI Reference](cli.md).
|
||||
|
||||
## Monorepo Workspaces
|
||||
|
||||
APOPHIS supports workspace-wide operations with the `--workspace` flag.
|
||||
|
||||
### Root package.json scripts
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"apophis:verify": "apophis verify --workspace --profile quick",
|
||||
"apophis:doctor": "apophis doctor --workspace",
|
||||
"apophis:qualify": "apophis qualify --workspace --profile ci"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Workspace fan-out
|
||||
|
||||
Run verify across all packages:
|
||||
Use `--workspace` to run verify or doctor across all packages:
|
||||
|
||||
```bash
|
||||
apophis verify --workspace --profile quick --format json
|
||||
```
|
||||
|
||||
Output is package-attributed:
|
||||
|
||||
```json
|
||||
{
|
||||
"exitCode": 0,
|
||||
"runs": [
|
||||
{
|
||||
"package": "api",
|
||||
"cwd": "/repo/packages/api",
|
||||
"artifact": { ... }
|
||||
},
|
||||
{
|
||||
"package": "web",
|
||||
"cwd": "/repo/packages/web",
|
||||
"artifact": { ... }
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Supported commands
|
||||
|
||||
- `apophis verify --workspace`
|
||||
- `apophis doctor --workspace`
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| 0 | Success |
|
||||
| 1 | Behavioral / qualification failure |
|
||||
| 2 | Usage, config, or environment safety violation |
|
||||
| 3 | Internal APOPHIS error |
|
||||
| 130 | Interrupted (SIGINT) |
|
||||
See [CLI Reference](cli.md) for workspace output format and exit codes.
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
APOPHIS is designed to be safe and predictable for LLM-generated Fastify services.
|
||||
|
||||
It applies an invariant-driven approach to LLM-assisted development: constrained vocabulary, deterministic replay, and executable contracts give coding agents a verifiable loop between generated changes and behavioral correctness.
|
||||
|
||||
## Why APOPHIS Is Good for LLM-Generated Services
|
||||
|
||||
Coding agents benefit from:
|
||||
@@ -18,10 +20,10 @@ Use `apophis init` with a preset:
|
||||
|
||||
| Preset | Use Case |
|
||||
|---|---|
|
||||
| `safe-ci` | General CI-safe setup |
|
||||
| `llm-safe` | Ultra-minimal for LLM-generated code |
|
||||
| `platform-observe` | Observe-mode policy and runtime drift reporting |
|
||||
| `protocol-lab` | Multi-step flows and stateful testing |
|
||||
| `safe-ci` | Minimal CI-safe preset (default) |
|
||||
| `llm-safe` | Minimal preset for LLM-generated codebases |
|
||||
| `platform-observe` | Production-ready with observe mode |
|
||||
| `protocol-lab` | Multi-step flow and stateful testing |
|
||||
|
||||
```bash
|
||||
apophis init --preset llm-safe
|
||||
@@ -84,7 +86,6 @@ export default {
|
||||
presets: {
|
||||
'llm-safe': {
|
||||
name: 'llm-safe',
|
||||
depth: 'quick',
|
||||
timeout: 3000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
@@ -108,6 +109,8 @@ export default {
|
||||
### Route Template with Behavioral Contract
|
||||
|
||||
```javascript
|
||||
import crypto from 'crypto';
|
||||
|
||||
app.post('/users', {
|
||||
schema: {
|
||||
'x-category': 'constructor',
|
||||
@@ -134,7 +137,7 @@ app.post('/users', {
|
||||
}
|
||||
}, async (request, reply) => {
|
||||
const { name } = request.body;
|
||||
const id = `usr-${Date.now()}`;
|
||||
const id = `usr-${crypto.createHash('sha256').update(name).digest('hex').slice(0, 8)}`;
|
||||
reply.status(201);
|
||||
return { id, name };
|
||||
});
|
||||
@@ -161,7 +164,7 @@ console.log(verifyResult);
|
||||
2. **Run doctor first**: Catch setup issues before running verify.
|
||||
3. **Use `--changed` in CI**: Only verify routes that changed in the PR.
|
||||
4. **Commit config**: Store `apophis.config.js` in version control.
|
||||
5. **Pin versions**: Pin `apophis-fastify` version in `package.json`.
|
||||
5. **Pin versions**: Pin `@apophis/fastify` version in `package.json`.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
||||
+195
-29
@@ -2,14 +2,9 @@
|
||||
|
||||
Runtime visibility and drift detection without blocking by default.
|
||||
|
||||
## What Observe Does
|
||||
APOPHIS observe is **programmatic only**. Register the APOPHIS plugin with `observe.enabled: true` and `observe.sinks` to emit contract pass/violation/error events from live traffic without blocking responses. Sampling controls the fraction of observed requests.
|
||||
|
||||
`apophis observe` validates your runtime observe configuration:
|
||||
|
||||
1. Checks that observe mode is allowed in the current environment
|
||||
2. Validates reporting sink setup (logs, metrics, traces)
|
||||
3. Confirms non-blocking semantics
|
||||
4. Reports what would be observed and why it is safe
|
||||
Use `apophis doctor --mode observe` to validate your observe configuration (sinks, sampling, blocking, production safety) before deploying.
|
||||
|
||||
## When to Use It
|
||||
|
||||
@@ -34,21 +29,19 @@ Observe mode requires a reporting sink. Configure it in your environment policy:
|
||||
environments: {
|
||||
staging: {
|
||||
name: 'staging',
|
||||
allowVerify: true,
|
||||
allowObserve: true,
|
||||
allowQualify: false,
|
||||
allowChaos: false,
|
||||
allowBlocking: false,
|
||||
requireSink: true
|
||||
allowedModes: ['verify', 'observe'],
|
||||
blockQualify: true,
|
||||
requireSink: true,
|
||||
allowBlocking: false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
APOPHIS supports these sink types:
|
||||
You implement sinks for your observability backend. Common categories:
|
||||
|
||||
- **Logs**: Structured logging of contract violations
|
||||
- **Metrics**: Counter and histogram metrics for violation rates
|
||||
- **Traces**: Distributed tracing integration for violation context
|
||||
- **Logs**: Structured logging of contract violations (pino, winston)
|
||||
- **Metrics**: Counter and histogram metrics for violation rates (Prometheus, OpenTelemetry)
|
||||
- **Traces**: Distributed tracing integration for violation context (OpenTelemetry, Jaeger)
|
||||
|
||||
## Sampling
|
||||
|
||||
@@ -65,29 +58,49 @@ profiles: {
|
||||
}
|
||||
```
|
||||
|
||||
The `platform-observe` preset enables sampling at the preset level. Fine-tune per route with `x-observe-sampling` in your route schema.
|
||||
The `platform-observe` preset enables sampling. Configure the rate explicitly:
|
||||
|
||||
```javascript
|
||||
profiles: {
|
||||
'staging-observe': {
|
||||
mode: 'observe',
|
||||
preset: 'platform-observe',
|
||||
routes: [],
|
||||
sampling: 1.0 // 100% of requests observed
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Staging vs Production
|
||||
|
||||
| Environment | Blocking | Sampling | Sink Required |
|
||||
|---|---|---|---|
|
||||
| Staging | No (default) | 10% | Yes |
|
||||
| Production | No (default) | 1% | Yes |
|
||||
| Staging | No (default) | 100% | Yes |
|
||||
| Production | No (default) | 100% | Yes |
|
||||
|
||||
## `--check-config` Flag
|
||||
Default is `1.0` (100%). Configure lower rates for production explicitly:
|
||||
|
||||
Validate config without activating observe mode:
|
||||
|
||||
```bash
|
||||
apophis observe --profile staging-observe --check-config
|
||||
```javascript
|
||||
profiles: {
|
||||
'prod-observe': {
|
||||
mode: 'observe',
|
||||
preset: 'platform-observe',
|
||||
routes: [],
|
||||
sampling: 0.1 // 10% of requests observed
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This is useful in CI to ensure observe config is valid before deployment.
|
||||
## Config Validation
|
||||
|
||||
## Exit Codes
|
||||
Validate your observe config before deployment with doctor:
|
||||
|
||||
## Validation (via Doctor)
|
||||
|
||||
Validate observe configuration with `apophis doctor --mode observe`:
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
|---|---|---|
|
||||
| 0 | Observe config is valid and safe |
|
||||
| 2 | Safety violation or invalid config |
|
||||
|
||||
@@ -109,7 +122,6 @@ export default {
|
||||
presets: {
|
||||
'platform-observe': {
|
||||
name: 'platform-observe',
|
||||
depth: 'standard',
|
||||
timeout: 10000,
|
||||
parallel: true,
|
||||
chaos: false,
|
||||
@@ -138,3 +150,157 @@ export default {
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
## Programmatic Runtime Activation
|
||||
|
||||
Register APOPHIS with observe options in your application before routes are registered.
|
||||
|
||||
```typescript
|
||||
import Fastify from 'fastify'
|
||||
import apophisPlugin from '@apophis/fastify'
|
||||
import type { ObserveSink, ObserveEvent } from '@apophis/fastify'
|
||||
|
||||
const app = Fastify({ logger: true })
|
||||
|
||||
// Implement the ObserveSink interface.
|
||||
// Capture events to your preferred observability backend.
|
||||
const metricsSink: ObserveSink = {
|
||||
emit(event: ObserveEvent) {
|
||||
// Emit a counter for each contract evaluation
|
||||
myMetrics.increment(`apophis.contract.${event.type}`, {
|
||||
route: event.route,
|
||||
formula: event.formula,
|
||||
})
|
||||
|
||||
// Record duration as a histogram
|
||||
myMetrics.histogram('apophis.contract.duration_ms', event.durationMs, {
|
||||
route: event.route,
|
||||
})
|
||||
|
||||
// Log high-signal violations for immediate triage
|
||||
if (event.type === 'contract.violation') {
|
||||
logger.warn({ event }, 'APOPHIS contract violation')
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// Register APOPHIS with observe enabled.
|
||||
// This emits non-blocking contract pass/violation/error events
|
||||
// for every covered request, gated by sampling.
|
||||
await app.register(apophisPlugin, {
|
||||
runtime: process.env.NODE_ENV === 'test' ? 'error' : 'off',
|
||||
observe: {
|
||||
enabled: true,
|
||||
sampling: 0.1, // observe 10% of requests
|
||||
sinks: [metricsSink],
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
For new services, `createFastify()` wires discovery and APOPHIS before your
|
||||
routes, which avoids the most common ordering mistake:
|
||||
|
||||
```typescript
|
||||
import { createFastify } from '@apophis/fastify'
|
||||
|
||||
const app = await createFastify({
|
||||
logger: true,
|
||||
apophis: {
|
||||
runtime: process.env.NODE_ENV === 'test' ? 'error' : 'off',
|
||||
observe: {
|
||||
enabled: true,
|
||||
sampling: 0.1,
|
||||
sinks: [metricsSink],
|
||||
},
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
Key constraints:
|
||||
- Sink `emit()` can be sync or async (returns `void | Promise<void>`).
|
||||
- Sink rejections and thrown errors are silently caught — they never affect the route response or status code.
|
||||
- In production, observe hooks still run when `observe.enabled` and `observe.sinks` are configured; blocking runtime validation does not.
|
||||
- Sampling is applied per-formula evaluation via `Math.random() < sampling`.
|
||||
At `sampling: 1` every formula is emitted. At `sampling: 0` nothing is emitted.
|
||||
- Only routes with APOPHIS annotations (`x-ensures`, `x-requires`) produce events.
|
||||
Routes without annotations are not evaluated in observe mode.
|
||||
|
||||
## Sink Implementations
|
||||
|
||||
APOPHIS does not ship with built-in sinks. The `ObserveSink` interface lets you
|
||||
plug in any backend.
|
||||
|
||||
### OpenTelemetry
|
||||
|
||||
Export contract evaluations as OTel custom metrics alongside your existing spans and traces:
|
||||
|
||||
```typescript
|
||||
import type { ObserveSink, ObserveEvent } from '@apophis/fastify'
|
||||
import { metrics } from '@opentelemetry/api'
|
||||
|
||||
const meter = metrics.getMeter('apophis')
|
||||
|
||||
const contractCounter = meter.createCounter('apophis.contract.evaluations', {
|
||||
description: 'Number of contract evaluations',
|
||||
})
|
||||
const violationCounter = meter.createCounter('apophis.contract.violations', {
|
||||
description: 'Number of contract violations',
|
||||
})
|
||||
const durationHistogram = meter.createHistogram('apophis.contract.duration_ms', {
|
||||
description: 'Contract evaluation duration',
|
||||
})
|
||||
|
||||
export const otelSink: ObserveSink = {
|
||||
emit(event: ObserveEvent) {
|
||||
const labels = { route: event.route, formula: event.formula }
|
||||
contractCounter.add(1, labels)
|
||||
if (event.type === 'contract.violation') {
|
||||
violationCounter.add(1, labels)
|
||||
}
|
||||
durationHistogram.record(event.durationMs, labels)
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Register it with your Fastify app:
|
||||
|
||||
```typescript
|
||||
await app.register(apophisPlugin, {
|
||||
observe: { enabled: true, sinks: [otelSink] },
|
||||
})
|
||||
```
|
||||
|
||||
APOPHIS violations now appear in your OTel metrics backend (Prometheus, Datadog, New Relic, etc.) alongside latency and error data — no custom collector needed.
|
||||
|
||||
### Console (development / debugging)
|
||||
|
||||
```typescript
|
||||
export const consoleSink: ObserveSink = {
|
||||
emit(event: ObserveEvent) {
|
||||
if (event.type === 'contract.violation') {
|
||||
console.warn('[apophis]', event.route, event.formula, event.observed)
|
||||
}
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### pino logger
|
||||
|
||||
```typescript
|
||||
export function createPinoSink(logger: pino.Logger): ObserveSink {
|
||||
return {
|
||||
emit(event: ObserveEvent) {
|
||||
logger[event.type === 'contract.violation' ? 'warn' : 'info'](event)
|
||||
},
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Monorepo Validation
|
||||
|
||||
For monorepos, use `apophis doctor --mode observe --workspace` to validate observe configuration across all workspace packages.
|
||||
|
||||
## Mode Mismatch
|
||||
|
||||
Profiles configured for `verify` mode will be rejected at runtime. Only profiles with `mode: 'observe'` are valid for observe mode.
|
||||
```
|
||||
|
||||
+5
-5
@@ -23,8 +23,8 @@ BENCH_RUNS=12 BENCH_WARMUP=3 npm run benchmark:cli
|
||||
# Increase inner-loop work for micro-benchmarks
|
||||
BENCH_INNER_ITERS=5000 npm run benchmark:hot
|
||||
|
||||
# Benchmark generation profile matrix
|
||||
BENCH_GENERATION_PROFILES=quick,standard,thorough npm run benchmark:all
|
||||
# Benchmark with varying test counts
|
||||
BENCH_RUNS=10,50,200 npm run benchmark:all
|
||||
```
|
||||
|
||||
## Capture CPU Profile for Qualify
|
||||
@@ -41,10 +41,10 @@ This writes Chrome-compatible CPU profiles to `.profiles/qualify.cpuprofile` and
|
||||
- CLI benchmark uses spawned `node dist/cli/index.js` commands so startup costs are included.
|
||||
- Hot path benchmark runs in-process for lower-noise function-level comparisons.
|
||||
- Use fixed `--seed` for qualify benchmarks to keep runs deterministic.
|
||||
- Generation now adapts to depth: `quick` favors bounded payload generation speed, `thorough` keeps broader generation.
|
||||
- Schema generation uses fixed defaults (string≤128, array≤10) regardless of run count.
|
||||
|
||||
You can override generation per run:
|
||||
You can override runs per preset:
|
||||
|
||||
```bash
|
||||
apophis qualify --profile oauth-nightly --generation-profile quick --seed 42
|
||||
apophis qualify --profile oauth-nightly --seed 42
|
||||
```
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
# Plugin Contracts
|
||||
|
||||
Define behavioral rules that apply across all matching routes and are automatically injected into every matching route's contract evaluation.
|
||||
|
||||
Plugin contracts let teams codify cross-cutting expectations — "all API routes must carry an auth header," "all POST handlers must close the database connection," "all responses must include a request ID." They're defined once in config and evaluated alongside route-level contracts without per-route boilerplate.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [When to Use It](#when-to-use-it)
|
||||
- [Config Format](#config-format)
|
||||
- [PluginContractSpec Reference](#plugincontractspec-reference)
|
||||
- [Route Pattern Matching](#route-pattern-matching)
|
||||
- [Precondition → Skip Semantics](#precondition--skip-semantics)
|
||||
- [Auto-Injected Headers](#auto-injected-headers)
|
||||
- [Failure Attribution](#failure-attribution)
|
||||
- [Extension Dependencies](#extension-dependencies)
|
||||
- [Production Safety](#production-safety)
|
||||
- [Complete Example](#complete-example)
|
||||
- [Limitations](#limitations)
|
||||
|
||||
## When to Use It
|
||||
|
||||
Plugin contracts are ideal for rules that span many routes and don't belong on a single handler:
|
||||
|
||||
- Auth header presence across all API routes
|
||||
- Response metadata (request IDs, timing headers, version stamps)
|
||||
- Tenant isolation checks that apply to every route in a prefix
|
||||
- Database connection lifecycle guards on write endpoints
|
||||
- Content-type enforcement on all response paths
|
||||
|
||||
Don't use plugin contracts for route-specific behavioral rules — those belong on the route schema via `x-requires` and `x-ensures`.
|
||||
|
||||
## Config Format
|
||||
|
||||
### apophis.config.js
|
||||
|
||||
```javascript
|
||||
export default {
|
||||
pluginContracts: {
|
||||
"auth-presence": {
|
||||
appliesTo: "/api/**",
|
||||
hooks: {
|
||||
onRequest: {
|
||||
requires: ["request_headers(this).authorization != null"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"rate-limit-respected": {
|
||||
appliesTo: "POST /api/**",
|
||||
hooks: {
|
||||
onResponse: {
|
||||
ensures: ["status != 429"]
|
||||
}
|
||||
},
|
||||
meta: { name: "rate-limit-respected", description: "Rate limit should not be hit under normal CI load" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Programmatic (Fastify plugin registration)
|
||||
|
||||
```javascript
|
||||
await app.register(apophisPlugin, {
|
||||
pluginContracts: {
|
||||
"auth-presence": {
|
||||
appliesTo: "/api/**",
|
||||
hooks: {
|
||||
onRequest: { requires: ["request_headers(this).authorization != null"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
Contracts registered programmatically take the same path through the plugin contract registry as config-defined contracts. Both sources are merged into the composed contract at evaluation time.
|
||||
|
||||
## PluginContractSpec Reference
|
||||
|
||||
| Field | Required | Type | Description |
|
||||
|---|---|---|---|
|
||||
| `appliesTo` | yes | `string` | Route pattern (see [Route Pattern Matching](#route-pattern-matching)) |
|
||||
| `hooks` | yes | `object` | Hook-phase contracts keyed by Fastify lifecycle hook name |
|
||||
| `hooks.<phase>.requires` | no | `string[]` | APOSTL formulas that must be `true` **before** this phase executes |
|
||||
| `hooks.<phase>.ensures` | no | `string[]` | APOSTL formulas that must be `true` **after** this phase executes |
|
||||
| `meta` | no | `object` | Display metadata |
|
||||
| `meta.name` | no | `string` | Human-readable plugin name |
|
||||
| `meta.version` | no | `string` | Plugin version |
|
||||
| `meta.description` | no | `string` | Description of what the plugin enforces |
|
||||
| `extensions` | no | `array` | APOPHIS extensions this plugin depends on |
|
||||
|
||||
### Supported hook phases
|
||||
|
||||
Keys in `hooks` correspond to Fastify lifecycle hook names:
|
||||
|
||||
- `onRequest` — evaluated before the route handler runs
|
||||
- `preHandler` — evaluated before the handler, after shared hooks
|
||||
- `preSerialization` — evaluated before the response body is serialized
|
||||
- `onSend` — evaluated before the response is sent
|
||||
- `onResponse` — evaluated after the response is sent (body already consumed)
|
||||
|
||||
At least one of `requires` or `ensures` must be present per phase. Empty phases are ignored.
|
||||
|
||||
### Extension dependencies
|
||||
|
||||
```javascript
|
||||
pluginContracts: {
|
||||
"jwt-auth": {
|
||||
appliesTo: "/api/**",
|
||||
extensions: [{ name: "jwt-decoder", required: true }],
|
||||
hooks: {
|
||||
onRequest: { requires: ["decode_jwt(this).valid == true"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If a required extension is missing at registration time, the plugin's contracts are skipped with a warning. Extensions are resolved from the APOPHIS extension registry at evaluation time, not at route discovery time.
|
||||
|
||||
## Route Pattern Matching
|
||||
|
||||
The `appliesTo` field controls which routes a plugin contract covers.
|
||||
|
||||
### Pattern syntax
|
||||
|
||||
| Pattern | Matches | Does NOT match |
|
||||
|---|---|---|
|
||||
| `"/api/users"` | `/api/users` (exact) | `/api/users/42`, `/api/v2/users` |
|
||||
| `"/api/**"` | `/api/users`, `/api/users/42`, `/api/users/:id/posts` | `/v2/api/users` |
|
||||
| `"/api/*"` | `/api/users`, `/api/status` | `/api/users/42`, `/api/users/:id/posts` |
|
||||
| `"**"` | Everything | — |
|
||||
| `"POST /api/**"` | `POST /api/users`, `POST /api/orders/42` | `GET /api/users`, `POST /v2/api/users` |
|
||||
|
||||
The `/**` suffix matches any path at any depth below the prefix. The `/*` suffix matches exactly one segment below the prefix.
|
||||
|
||||
### Method-prefixed patterns
|
||||
|
||||
Prepend the HTTP method to restrict a pattern to a specific verb:
|
||||
|
||||
```javascript
|
||||
{
|
||||
appliesTo: "POST /api/**",
|
||||
hooks: {
|
||||
preHandler: { requires: ["request_body(this) != null"] }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
When a method is specified, the pattern only matches routes registered with that HTTP method. The method must be a valid HTTP method in uppercase.
|
||||
|
||||
Pattern matching is evaluated against the route's path string only (the raw argument passed to `fastify.get()`, `fastify.post()`, etc.), not against the dereferenced URL with prefix applied. Fastify prefix application happens before pattern matching, so a route registered as `fastify.get("/users")` under a `prefix: "/api"` is matched against `/api/users`.
|
||||
|
||||
## Precondition → Skip Semantics
|
||||
|
||||
When a plugin contract defines `requires` formulas, the runner evaluates them before `ensures`. If any require formula fails:
|
||||
|
||||
1. Require failure is **not** counted as a contract failure
|
||||
2. All `ensures` formulas for that route+run are **skipped**
|
||||
3. The skip is counted separately from failures in the artifact summary
|
||||
|
||||
This distinguishes "the plugin's prerequisites aren't met" from "the route's behavior violated the contract."
|
||||
|
||||
### Example
|
||||
|
||||
```javascript
|
||||
pluginContracts: {
|
||||
"tenant-check": {
|
||||
appliesTo: "/api/**",
|
||||
hooks: {
|
||||
onRequest: {
|
||||
requires: ["request_headers(this).x-tenant-id != null"],
|
||||
ensures: ["status != 500"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For a route without the `x-tenant-id` header:
|
||||
- The `requires` check fails → the `ensures` (`status != 500`) is **skipped**
|
||||
- `skippedCount` increments; `failedCount` does not
|
||||
|
||||
For a route with the `x-tenant-id` header:
|
||||
- The `requires` check passes → the `ensures` is evaluated normally
|
||||
- If `status == 500`, it's counted as a plugin contract **failure**
|
||||
|
||||
## Auto-Injected Headers
|
||||
|
||||
To avoid every test being skipped when a plugin contract checks for header presence, APOPHIS automatically injects headers into test requests when the precondition pattern is simple enough.
|
||||
|
||||
### Supported patterns
|
||||
|
||||
| Formula pattern | Injected header |
|
||||
|---|---|
|
||||
| `request_headers(this).<name> != null` | `<name>: test-value` |
|
||||
| `request_headers(this).<name> == "value"` | `<name>: value` |
|
||||
|
||||
### Example
|
||||
|
||||
```javascript
|
||||
pluginContracts: {
|
||||
"auth-presence": {
|
||||
appliesTo: "/api/**",
|
||||
hooks: {
|
||||
onRequest: {
|
||||
requires: ["request_headers(this).authorization != null"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
APOPHIS detects the `!= null` pattern on `authorization` and injects `authorization: test-value` into every test request. The precondition passes automatically without user intervention.
|
||||
|
||||
Auto-injection works in all runners: verify, qualify (scenario, stateful, chaos), and replay.
|
||||
|
||||
### Unsupported patterns
|
||||
|
||||
Complex preconditions are not auto-injected. For these, the requires check will fail and the route's ensures will be skipped:
|
||||
|
||||
```javascript
|
||||
// NOT auto-injected — the route's ensures will be skipped
|
||||
requires: ["request_headers(this).x-count >= 5"]
|
||||
```
|
||||
|
||||
For these cases, provide the header explicitly via route variants:
|
||||
|
||||
```javascript
|
||||
await fastify.apophis.contract({
|
||||
runs: 5,
|
||||
variants: [{ name: "with-count", headers: { "x-count": "10" } }]
|
||||
})
|
||||
```
|
||||
|
||||
## Failure Attribution
|
||||
|
||||
When a plugin contract formula fails, the violation carries metadata that distinguishes it from route-level failures.
|
||||
|
||||
### ContractViolation fields
|
||||
|
||||
| Field | Value | Meaning |
|
||||
|---|---|---|
|
||||
| `source` | `"plugin:auth-presence"` | Which plugin added the formula |
|
||||
| `phase` | `"onRequest"` | Which hook phase the formula belongs to |
|
||||
|
||||
In human-readable output, plugin failures appear separately from route contract failures:
|
||||
|
||||
```text
|
||||
Plugin contract violation (plugin:auth-presence)
|
||||
GET /api/users
|
||||
Phase: onRequest
|
||||
|
||||
Expected
|
||||
response_headers(this).x-request-id != null
|
||||
|
||||
Observed
|
||||
response_headers(this).x-request-id was null
|
||||
```
|
||||
|
||||
The artifact summary tracks plugin contract outcomes independently:
|
||||
|
||||
```json
|
||||
{
|
||||
"summary": {
|
||||
"pluginContractsApplied": 12,
|
||||
"pluginContractsFailed": 2,
|
||||
"failed": 1,
|
||||
"passed": 10
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- `pluginContractsApplied` — total plugin contract formulas evaluated across all routes
|
||||
- `pluginContractsFailed` — plugin contract formulas that failed
|
||||
- `failed` — total failures (route + plugin)
|
||||
- `passed` — route-level contract passes (plugin-only formulas not counted here)
|
||||
|
||||
A route can pass its own contracts and still have plugin contract failures. The CLI exit code reflects the aggregate: any failure (route or plugin) produces exit code 1.
|
||||
|
||||
## Extension Dependencies
|
||||
|
||||
Plugin contracts can declare dependencies on APOPHIS extensions. Extension resolution happens at evaluation time via the extension registry.
|
||||
|
||||
### Declaring dependencies
|
||||
|
||||
```javascript
|
||||
pluginContracts: {
|
||||
"jwt-auth": {
|
||||
appliesTo: "/api/**",
|
||||
extensions: [{ name: "jwt-decoder", required: true }],
|
||||
hooks: {
|
||||
onRequest: { requires: ["decode_jwt(this).valid == true"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Each extension reference has:
|
||||
- `name` (required) — matches the extension's registered name
|
||||
- `required` (optional, default `true`) — if `false`, missing extension emits a warning but does not skip the plugin
|
||||
|
||||
### Missing extension behavior
|
||||
|
||||
If a required extension is not registered, the plugin's contracts are **skipped** for all matching routes. Warnings are collected in the registry and surfaced via `drainWarnings()`:
|
||||
|
||||
```text
|
||||
Plugin 'jwt-auth' requires extensions [jwt-decoder] which are not registered. Skipping its contracts.
|
||||
```
|
||||
|
||||
Warnings appear in the CLI output and the artifact's `warnings` array.
|
||||
|
||||
### Extension registration
|
||||
|
||||
Extensions are registered alongside plugin contracts:
|
||||
|
||||
```javascript
|
||||
await app.register(apophisPlugin, {
|
||||
extensions: [
|
||||
{
|
||||
name: "jwt-decoder",
|
||||
headers: ["decode_jwt"],
|
||||
predicates: {
|
||||
decode_jwt: (ctx) => ({ value: true, success: true })
|
||||
}
|
||||
}
|
||||
],
|
||||
pluginContracts: {
|
||||
"jwt-auth": {
|
||||
appliesTo: "/api/**",
|
||||
extensions: [{ name: "jwt-decoder", required: true }],
|
||||
hooks: {
|
||||
onRequest: { requires: ["decode_jwt(this).valid == true"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
## Production Safety
|
||||
|
||||
`pluginContracts` is a **test-only** feature. If `pluginContracts` is non-empty in a production environment (`NODE_ENV=production`), APOPHIS throws at plugin registration:
|
||||
|
||||
```text
|
||||
APOPHIS: Unsafe options detected in production: pluginContracts.
|
||||
These features are test-only and must not be enabled in production.
|
||||
Remove them from ApophisOptions or set NODE_ENV=test.
|
||||
```
|
||||
|
||||
This is enforced by `validateProductionSafety()` before any contract evaluation runs. There is no escape hatch.
|
||||
|
||||
## Complete Example
|
||||
|
||||
### Config (`apophis.config.js`)
|
||||
|
||||
```javascript
|
||||
export default {
|
||||
profiles: {
|
||||
quick: { preset: "safe-ci" },
|
||||
},
|
||||
presets: {
|
||||
"safe-ci": { runs: 10 },
|
||||
},
|
||||
pluginContracts: {
|
||||
"auth-required": {
|
||||
appliesTo: "/api/**",
|
||||
hooks: {
|
||||
onRequest: {
|
||||
requires: ["request_headers(this).authorization != null"],
|
||||
},
|
||||
},
|
||||
meta: { name: "auth-required", description: "All API routes require an Authorization header" },
|
||||
},
|
||||
"request-id": {
|
||||
appliesTo: "/api/**",
|
||||
hooks: {
|
||||
onSend: {
|
||||
ensures: ["response_headers(this).x-request-id != null"],
|
||||
},
|
||||
},
|
||||
meta: { name: "request-id", description: "All API responses must include x-request-id" },
|
||||
},
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Route (`routes/users.js`)
|
||||
|
||||
```javascript
|
||||
app.get("/api/users", {
|
||||
schema: {
|
||||
"x-category": "observer",
|
||||
"x-ensures": ["status:200", "response_body(this) is Array"],
|
||||
response: {
|
||||
200: {
|
||||
type: "array",
|
||||
items: { type: "object", properties: { id: { type: "number" } } },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
return [{ id: 1 }, { id: 2 }]
|
||||
})
|
||||
```
|
||||
|
||||
### Execution (`apophis verify`)
|
||||
|
||||
1. Sends a request to `GET /api/users`
|
||||
2. `auth-required` plugin: `onRequest.requires` → `request_headers(this).authorization != null` → APOPHIS auto-injects `authorization: test-value` → passes
|
||||
3. Route-level ensures: `status:200` → passes; `response_body(this) is Array` → passes
|
||||
4. `request-id` plugin: `onSend.ensures` → `response_headers(this).x-request-id != null` → if the handler doesn't set `x-request-id`, this **fails** as a `plugin:request-id` contract violation
|
||||
|
||||
```text
|
||||
Plugin contract violation (plugin:request-id)
|
||||
GET /api/users
|
||||
Phase: onSend
|
||||
|
||||
Expected
|
||||
response_headers(this).x-request-id != null
|
||||
|
||||
Observed
|
||||
response_headers(this).x-request-id was null
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
- **`appliesTo` matches by route path only** by default. Use method-prefixed syntax (`"POST /api/**"`) for HTTP-method-specific rules.
|
||||
- **Header auto-injection only handles simple patterns** (`!= null` and `== "value"`). Complex preconditions like `>= 5` are not auto-injected; the route's ensures will be skipped.
|
||||
- **No per-route opt-out.** If a plugin contract matches a route, it always applies. Use narrower `appliesTo` patterns or separate route prefixes to exclude routes.
|
||||
- **Extension resolution is lazy.** Extensions are checked at route evaluation time, not at registration time. A missing extension is only discovered when a matching route is evaluated.
|
||||
- **At least one of `requires` or `ensures` must be present per phase.** A phase with neither is silently ignored.
|
||||
- **Runtime hook ordering follows Fastify's lifecycle.** Plugin contracts in `onResponse` cannot inspect the response body (it has already been sent to the client).
|
||||
+319
-37
@@ -2,6 +2,8 @@
|
||||
|
||||
Run scenario, stateful, and chaos checks against non-production Fastify services.
|
||||
|
||||
Qualify extends invariant-driven testing with multi-step protocol flows, stateful sequences, and controlled fault injection.
|
||||
|
||||
## What Qualify Does
|
||||
|
||||
`apophis qualify` runs deeper testing than verify:
|
||||
@@ -13,9 +15,51 @@ Run scenario, stateful, and chaos checks against non-production Fastify services
|
||||
|
||||
## When to Use It
|
||||
|
||||
- **Nightly CI**: Scenario and stateful checks for critical flows
|
||||
- **Staging**: Protocol flow validation before production
|
||||
- **Specialist teams**: Auth, billing, workflow systems
|
||||
Qualify is heavier than verify. Use it where the depth is worth the runtime cost:
|
||||
|
||||
| Workflow | Recommended | Why |
|
||||
|---|---|---|
|
||||
| **Pull request** | No — use `verify` | `verify` is fast (<5s for typical services) and catches behavioral regressions per-route. Qualify adds multi-minute scenario/stateful/chaos runs that are too slow for PR feedback loops. |
|
||||
| **Nightly** | Yes | Full scenario, stateful, and chaos execution against staging. Catch protocol-level regressions that single-route verification cannot see. |
|
||||
| **Pre-release** | Yes | Run qualify against the exact artifact that will be promoted to production. Treat a passing qualify run as a release gate for critical flows. |
|
||||
| **Specialist workflows** | Yes | Auth flows, billing sequences, idempotency guarantees, and pagination consistency need multi-step qualification that verify cannot express. |
|
||||
| **Chaos engineering** | Nightly or ad-hoc | Chaos injection increases latency. Run it in dedicated CI slots, not on every commit. |
|
||||
|
||||
### Quick workflow setup
|
||||
|
||||
```javascript
|
||||
// apophis.config.js — two profiles for different cadences
|
||||
export default {
|
||||
mode: 'qualify',
|
||||
profiles: {
|
||||
'nightly': {
|
||||
name: 'nightly',
|
||||
mode: 'qualify',
|
||||
preset: 'deep',
|
||||
features: ['scenario', 'stateful', 'chaos'],
|
||||
routes: [],
|
||||
},
|
||||
'pre-release': {
|
||||
name: 'pre-release',
|
||||
mode: 'qualify',
|
||||
preset: 'deep',
|
||||
features: ['scenario', 'stateful'],
|
||||
routes: [],
|
||||
},
|
||||
},
|
||||
presets: {
|
||||
deep: { timeout: 15000, chaos: false },
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Run nightly: `apophis qualify --profile nightly`
|
||||
Run pre-release: `apophis qualify --profile pre-release --format json-summary`
|
||||
|
||||
For pull requests, use verify instead:
|
||||
```bash
|
||||
apophis verify --profile ci
|
||||
```
|
||||
|
||||
## Scenario Examples
|
||||
|
||||
@@ -49,6 +93,40 @@ profiles: {
|
||||
}
|
||||
```
|
||||
|
||||
## Scenario Definitions
|
||||
|
||||
Scenarios are multi-step flows with capture and rebind:
|
||||
|
||||
```javascript
|
||||
await fastify.apophis.scenario({
|
||||
name: 'oauth-basic',
|
||||
steps: [
|
||||
{
|
||||
name: 'authorize',
|
||||
request: { method: 'GET', url: '/oauth/authorize?client_id=web&response_type=code&state=abc123' },
|
||||
// Behavioral: state parameter round-trips for CSRF protection
|
||||
expect: ['response_payload(this).state == request_query(this).state'],
|
||||
capture: { code: 'response_payload(this).code' }
|
||||
},
|
||||
{
|
||||
name: 'token',
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: '/oauth/token',
|
||||
form: { grant_type: 'authorization_code', code: '$authorize.code', scope: 'read' }
|
||||
},
|
||||
// Behavioral: issued token preserves the requested scope
|
||||
expect: ['response_payload(this).scope == request_body(this).scope']
|
||||
}
|
||||
]
|
||||
})
|
||||
```
|
||||
|
||||
Scenario behavior:
|
||||
1. Cookie jar persists `Set-Cookie` values across steps.
|
||||
2. Step-level `headers.cookie` overrides jar values for that step.
|
||||
3. `form` sends `application/x-www-form-urlencoded` payloads.
|
||||
|
||||
## Stateful Testing
|
||||
|
||||
Stateful tests generate sequences of operations and track resources:
|
||||
@@ -58,7 +136,18 @@ Stateful tests generate sequences of operations and track resources:
|
||||
3. **Observer**: Read resources (GET)
|
||||
4. **Destructor**: Remove resources (DELETE)
|
||||
|
||||
APOPHIS automatically tracks created resources and cleans them up after testing.
|
||||
APOPHIS tracks created resources and runs cleanup after test completion.
|
||||
|
||||
Run stateful tests via the API:
|
||||
|
||||
```javascript
|
||||
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||
console.log('Stateful tests:', stateful.summary)
|
||||
```
|
||||
|
||||
## Route Transparency
|
||||
|
||||
Artifacts include `executedRoutes` and `skippedRoutes` arrays. `skippedRoutes` contains reasons such as mode mismatch, environment policy, or route filter exclusion.
|
||||
|
||||
## Chaos and Adversity
|
||||
|
||||
@@ -67,7 +156,9 @@ Chaos testing injects controlled failures:
|
||||
- **Delay**: Slow responses
|
||||
- **Error**: Return error status codes
|
||||
- **Dropout**: Connection failures
|
||||
- **Corruption**: Malformed response bodies
|
||||
- **Truncate**: Truncated response bodies
|
||||
- **Malformed**: Invalid JSON or content-type
|
||||
- **Field-corrupt**: Random field mutation in response objects
|
||||
|
||||
Configure chaos in your preset:
|
||||
|
||||
@@ -84,36 +175,6 @@ presets: {
|
||||
}
|
||||
```
|
||||
|
||||
## Profile Examples
|
||||
|
||||
### oauth-nightly
|
||||
|
||||
```javascript
|
||||
profiles: {
|
||||
'oauth-nightly': {
|
||||
name: 'oauth-nightly',
|
||||
mode: 'qualify',
|
||||
preset: 'protocol-lab',
|
||||
routes: [],
|
||||
seed: 42
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### lifecycle-deep
|
||||
|
||||
```javascript
|
||||
profiles: {
|
||||
'lifecycle-deep': {
|
||||
name: 'lifecycle-deep',
|
||||
mode: 'qualify',
|
||||
preset: 'protocol-lab',
|
||||
routes: [],
|
||||
seed: 42
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Non-Prod Boundaries
|
||||
|
||||
Qualify mode is gated away from production by default:
|
||||
@@ -122,7 +183,9 @@ Qualify mode is gated away from production by default:
|
||||
|---|---|---|---|
|
||||
| local | enabled | enabled | enabled |
|
||||
| test/CI | enabled | enabled | enabled |
|
||||
| staging | enabled with allowlist | synthetic-only | canary-only |
|
||||
| staging | enabled (blockQualify: false) | enabled (blockQualify: false) | blocked on protected routes (allowChaosOnProtected: false) |
|
||||
|
||||
Qualify gates are not individually gated per environment. The `blockQualify` flag controls all qualify execution, and `allowChaosOnProtected` controls chaos on protected routes.
|
||||
| production | disabled by default | disabled by default | disabled by default |
|
||||
|
||||
## Machine Output for CI
|
||||
@@ -186,7 +249,7 @@ export default {
|
||||
presets: {
|
||||
'protocol-lab': {
|
||||
name: 'protocol-lab',
|
||||
depth: 'deep',
|
||||
runs: 200,
|
||||
timeout: 15000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
@@ -224,3 +287,222 @@ export default {
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
## Gate Execution Counts
|
||||
|
||||
Human output shows per-gate execution counts (scenario, stateful, chaos) so you can verify which gates actually ran.
|
||||
|
||||
## Custom Scenarios (config-defined)
|
||||
|
||||
Define arbitrary multi-step scenarios directly in your `apophis.config.js` without writing code:
|
||||
|
||||
```javascript
|
||||
// apophis.config.js
|
||||
export default {
|
||||
mode: 'qualify',
|
||||
scenarios: [
|
||||
{
|
||||
name: 'idempotency-check',
|
||||
steps: [
|
||||
{
|
||||
name: 'create-order',
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: '/orders',
|
||||
body: { product: 'widget', quantity: 3 },
|
||||
},
|
||||
expect: ['status:201', 'response_body(this).id != null'],
|
||||
capture: { orderId: 'response_body(this).id' },
|
||||
},
|
||||
{
|
||||
name: 'duplicate-create',
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: '/orders',
|
||||
headers: { 'x-idempotency-key': 'dup-001' },
|
||||
body: { product: 'widget', quantity: 3 },
|
||||
},
|
||||
expect: ['status:200', 'response_body(this).id == "$create-order.orderId"'],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'pagination-flow',
|
||||
steps: [
|
||||
{
|
||||
name: 'list-page-1',
|
||||
request: { method: 'GET', url: '/items?page=1&limit=5' },
|
||||
expect: ['status:200', 'response_body(this).items != null'],
|
||||
capture: { firstPageCount: 'response_body(this).items.length' },
|
||||
},
|
||||
{
|
||||
name: 'list-page-2',
|
||||
request: { method: 'GET', url: '/items?page=2&limit=5' },
|
||||
expect: ['status:200'],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
profiles: {
|
||||
'nightly': {
|
||||
name: 'nightly',
|
||||
mode: 'qualify',
|
||||
preset: 'deep',
|
||||
routes: ['POST /orders', 'GET /orders', 'GET /items'],
|
||||
},
|
||||
},
|
||||
presets: {
|
||||
deep: { name: 'deep', timeout: 15000, chaos: true },
|
||||
},
|
||||
environments: {
|
||||
local: { name: 'local', allowQualify: true, allowChaos: true },
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
Scenario step fields:
|
||||
|
||||
| Field | Required | Description |
|
||||
|---|---|---|
|
||||
| `name` | yes | Human-readable step label |
|
||||
| `request.method` | yes | HTTP method (GET, POST, PUT, DELETE, PATCH) |
|
||||
| `request.url` | yes | URL path (e.g. `/orders`, `/items?page=1`) |
|
||||
| `request.body` | no | JSON request body |
|
||||
| `request.headers` | no | Custom headers (e.g. `x-idempotency-key`) |
|
||||
| `expect` | yes | APOSTL formulas that must return truthy for step to pass |
|
||||
| `capture` | no | Map of `{ key: "apostl_formula" }` — captured values are substituted via `$stepName.key` in later steps |
|
||||
|
||||
Captured values are interpolated in subsequent step URLs, bodies, and headers using `$stepName.key` syntax.
|
||||
|
||||
## Chaos Configuration
|
||||
|
||||
Fine-tune chaos behavior via preset fields:
|
||||
|
||||
```javascript
|
||||
presets: {
|
||||
'chaos-lab': {
|
||||
name: 'chaos-lab',
|
||||
timeout: 10000,
|
||||
chaos: true,
|
||||
chaosStrategy: 'sample', // 'one' | 'all' | 'sample' | 'routes'
|
||||
chaosSampleSize: 5, // routes to sample when strategy = 'sample'
|
||||
chaosSampleRoutes: [ // explicit routes when strategy = 'routes'
|
||||
'GET /api/users',
|
||||
'POST /api/orders',
|
||||
],
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Default | Description |
|
||||
|---|---|---|
|
||||
| `chaosStrategy` | `'one'` | Route selection strategy |
|
||||
| `chaosSampleSize` | `3` | Routes to sample (strategy `sample`) |
|
||||
| `chaosSampleRoutes` | — | Explicit route list (strategy `routes`) |
|
||||
|
||||
## Artifact Interpretation
|
||||
|
||||
Each qualify run produces an artifact JSON document. Key sections:
|
||||
|
||||
### executionSummary
|
||||
|
||||
```json
|
||||
{
|
||||
"executionSummary": {
|
||||
"totalPlanned": 15,
|
||||
"totalExecuted": 12,
|
||||
"totalPassed": 10,
|
||||
"totalFailed": 2,
|
||||
"scenariosRun": 3,
|
||||
"statefulTestsRun": 5,
|
||||
"chaosRunsRun": 4,
|
||||
"chaosRoutesPlanned": 2,
|
||||
"chaosRoutesExecuted": 2,
|
||||
"totalSteps": 12
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use `totalExecuted` vs `totalPlanned` to see how many checks actually ran (gate gating, route filtering, chaos selection). A non-zero `totalPlanned` with zero `totalExecuted` means all gates were disabled or no routes matched.
|
||||
|
||||
### executedRoutes / skippedRoutes
|
||||
|
||||
```json
|
||||
{
|
||||
"executedRoutes": ["POST /orders", "GET /orders/:id", "GET /items"],
|
||||
"skippedRoutes": [
|
||||
{ "route": "DELETE /items/:id", "reason": "No scenario covers this route" },
|
||||
{ "route": "GET /health", "reason": "Not selected by chaos strategy: one" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`executedRoutes` lists every route that had at least one scenario step, stateful command, or chaos injection. `skippedRoutes` explains why every other discovered route was excluded.
|
||||
|
||||
### profileGates
|
||||
|
||||
```json
|
||||
{
|
||||
"profileGates": {
|
||||
"scenario": true,
|
||||
"stateful": true,
|
||||
"chaos": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Shows which gates were active. Combine with `executionSummary` per-gate counts to verify each active gate produced results.
|
||||
|
||||
### stepTraces
|
||||
|
||||
Each entry records an individual step execution:
|
||||
|
||||
```json
|
||||
{
|
||||
"stepTraces": [
|
||||
{
|
||||
"step": 0,
|
||||
"name": "create-order",
|
||||
"route": "POST /orders",
|
||||
"durationMs": 12,
|
||||
"status": "passed"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Filter by `status` to isolate failures. Look at `durationMs` for performance regressions.
|
||||
|
||||
### failures
|
||||
|
||||
```json
|
||||
{
|
||||
"failures": [
|
||||
{
|
||||
"route": "POST /orders",
|
||||
"contract": "status:201",
|
||||
"category": "runtime",
|
||||
"replayCommand": "apophis replay --artifact reports/apophis/qualify-2026-05-21T...json"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`replayCommand` gives a copy-pasteable command to re-run the exact same seed with the stored artifact for triage.
|
||||
|
||||
## Zero-Execution Guardrail
|
||||
|
||||
Qualify exits with code 1 if zero checks executed. This prevents silent passes when all routes are filtered out or gates are disabled.
|
||||
|
||||
## Test Budget
|
||||
|
||||
The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:
|
||||
|
||||
```javascript
|
||||
presets: {
|
||||
'protocol-lab': {
|
||||
runs: 200,
|
||||
timeout: 15000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
# Quality Engines
|
||||
|
||||
APOPHIS includes three quality engines for advanced testing: chaos injection, flake detection, and mutation testing. All require `NODE_ENV=test`.
|
||||
|
||||
## Chaos Injection
|
||||
|
||||
Inject controlled failures into contract tests to validate resilience guarantees. Chaos events are generated by fast-check alongside test data, making them shrinkable — when a test fails, fast-check finds the minimal chaos event that causes the failure.
|
||||
|
||||
### Usage
|
||||
|
||||
```javascript
|
||||
const result = await fastify.apophis.contract({
|
||||
runs: 50,
|
||||
chaos: {
|
||||
delay: { probability: 0.1, minMs: 100, maxMs: 500 },
|
||||
error: { probability: 0.1, statusCode: 503 },
|
||||
dropout: { probability: 0.05 },
|
||||
corruption: { probability: 0.1 },
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
### Event Types
|
||||
|
||||
| Type | Effect | Tests |
|
||||
|------|--------|-------|
|
||||
| `delay` | Artificial latency | `response_time(this) < 1000` |
|
||||
| `error` | Forces HTTP status code | Error-handling contracts |
|
||||
| `dropout` | Network failure (status 0 or 504) | Fallback contracts |
|
||||
| `corruption` | Mutates response bodies | Parsing robustness |
|
||||
|
||||
### Corruption Strategies
|
||||
|
||||
| Strategy | Effect |
|
||||
|----------|--------|
|
||||
| `truncate` | Cuts response body in half |
|
||||
| `malformed` | Returns invalid JSON (`{"broken":`) |
|
||||
| `field-corrupt` | Sets a random field to `null` |
|
||||
|
||||
### Programmatic API
|
||||
|
||||
```javascript
|
||||
import {
|
||||
applyChaosToExecution,
|
||||
createChaosEventArbitrary,
|
||||
formatChaosEvents,
|
||||
} from '@apophis/fastify'
|
||||
|
||||
// Apply pre-generated chaos events to a context
|
||||
const result = applyChaosToExecution(ctx, events)
|
||||
|
||||
// Generate deterministic chaos events
|
||||
const arb = createChaosEventArbitrary(config, contractNames)
|
||||
const events = fc.sample(arb, { numRuns: 1, seed: 42 })[0]
|
||||
|
||||
// Format for diagnostics
|
||||
console.log(formatChaosEvents(events))
|
||||
```
|
||||
|
||||
### Best Practices
|
||||
|
||||
1. Start small: `probability: 0.05` (5% of requests)
|
||||
2. Test one failure mode at a time
|
||||
3. Verify contracts handle chaos: `if status:503 then response_code(GET /health) == 200`
|
||||
4. Use seeds for reproducibility: `seed: 42`
|
||||
|
||||
## Flake Detection
|
||||
|
||||
Automatically rerun failing tests with varied seeds to detect non-deterministic contracts. A "flake" is a test that fails on one run but passes on another with the same or different seed.
|
||||
|
||||
### Usage
|
||||
|
||||
```javascript
|
||||
// FlakeDetector is an internal API. It is not part of the public @apophis/fastify export.
|
||||
// For programmatic use, import from the internal path:
|
||||
// import { FlakeDetector } from '@apophis/fastify/src/quality/flake.js'
|
||||
|
||||
const detector = new FlakeDetector({
|
||||
sameSeedReruns: 1, // Rerun with same seed
|
||||
seedVariations: 3, // Try 3 additional seeds
|
||||
})
|
||||
|
||||
const report = await detector.detectFlake(
|
||||
originalFailingResult,
|
||||
async (seed) => {
|
||||
const suite = await fastify.apophis.contract({ seed })
|
||||
return { passed: suite.summary.failed === 0 }
|
||||
},
|
||||
originalSeed
|
||||
)
|
||||
|
||||
if (report.isFlaky) {
|
||||
console.log(`Flaky with ${report.confidence} confidence`)
|
||||
console.log('Reruns:', report.reruns)
|
||||
}
|
||||
```
|
||||
|
||||
### Report Structure
|
||||
|
||||
```javascript
|
||||
{
|
||||
isFlaky: true,
|
||||
confidence: 'high', // 'high' | 'medium' | 'low'
|
||||
reruns: [
|
||||
{ seed: 42, passed: false },
|
||||
{ seed: 43, passed: true },
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Confidence Scoring
|
||||
|
||||
| Pass Rate | Confidence |
|
||||
|-----------|------------|
|
||||
| 0% pass | `high` (deterministic failure) |
|
||||
| < 50% pass | `medium` |
|
||||
| >= 50% pass | `low` (likely flaky) |
|
||||
|
||||
## Mutation Testing
|
||||
|
||||
Measure contract strength by injecting synthetic bugs. A "mutation" is a small change to a contract (e.g., flip `==` to `!=`). If the test suite catches the mutation (fails), the mutation is "killed". If it passes, the mutation "survives" — indicating weak coverage.
|
||||
|
||||
### Usage
|
||||
|
||||
```javascript
|
||||
// Mutation testing is an internal API. It is not part of the public @apophis/fastify export.
|
||||
// For programmatic use, import from the internal path:
|
||||
// import { runMutationTesting } from '@apophis/fastify/src/quality/mutation.js'
|
||||
|
||||
const report = await runMutationTesting(fastify, {
|
||||
runs: 10,
|
||||
seed: 42,
|
||||
maxMutationsPerContract: 5,
|
||||
routes: ['/items'], // Optional: only test these routes
|
||||
})
|
||||
|
||||
console.log(`Mutation score: ${report.score}%`)
|
||||
console.log(`Killed: ${report.killed}, Survived: ${report.survived}`)
|
||||
console.log('Weak contracts:', report.weakContracts)
|
||||
```
|
||||
|
||||
### Mutation Operators
|
||||
|
||||
| Type | Example |
|
||||
|------|---------|
|
||||
| `flip-operator` | `== 201` → `!= 201` |
|
||||
| `change-number` | `== 200` → `== 201` |
|
||||
| `remove-clause` | `A && B` → `A` |
|
||||
| `negate-boolean` | `== true` → `== false` |
|
||||
| `swap-variable` | `response_body` → `request_body` |
|
||||
| `remove-ensures` | Remove one ensures clause entirely |
|
||||
|
||||
### Report Structure
|
||||
|
||||
```javascript
|
||||
{
|
||||
score: 85, // 0-100
|
||||
killed: 17,
|
||||
survived: 3,
|
||||
durationMs: 4500,
|
||||
weakContracts: ['POST /items'], // Routes where no mutations were killed
|
||||
mutations: [
|
||||
{
|
||||
mutation: {
|
||||
id: 'm0',
|
||||
route: 'POST /items',
|
||||
original: 'response_code(this) == 201',
|
||||
mutated: 'response_code(this) != 201',
|
||||
type: 'flip-operator',
|
||||
},
|
||||
killed: true,
|
||||
durationMs: 120,
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Single Mutation Test
|
||||
|
||||
Test a specific mutation without running the full suite:
|
||||
|
||||
```javascript
|
||||
// Mutation testing is an internal API. It is not part of the public @apophis/fastify export.
|
||||
// For programmatic use, import from the internal path:
|
||||
// import { testMutation } from '@apophis/fastify/src/quality/mutation.js'
|
||||
|
||||
const killed = await testMutation(fastify, contract, mutation, {
|
||||
runs: 10,
|
||||
seed: 42,
|
||||
})
|
||||
```
|
||||
|
||||
## Environment Guard
|
||||
|
||||
All quality engines require `NODE_ENV=test`:
|
||||
|
||||
```
|
||||
Error: chaos is only available in test environment.
|
||||
Set NODE_ENV=test to enable quality features.
|
||||
```
|
||||
|
||||
This prevents accidental execution in production or development.
|
||||
|
||||
## Integration Example
|
||||
|
||||
Run all three engines in a CI pipeline:
|
||||
|
||||
```javascript
|
||||
// 1. Standard contract tests
|
||||
const suite = await fastify.apophis.contract({ runs: 50, seed: 42 })
|
||||
|
||||
// 2. Chaos tests
|
||||
const chaosSuite = await fastify.apophis.contract({
|
||||
runs: 50,
|
||||
seed: 42,
|
||||
chaos: { error: { probability: 0.1, statusCode: 503 } },
|
||||
})
|
||||
|
||||
// 3. Flake detection on failures
|
||||
for (const test of suite.tests.filter(t => !t.ok)) {
|
||||
const report = await detector.detectFlake(test, rerunFn, 42)
|
||||
if (report.isFlaky) {
|
||||
console.warn(`Flaky test detected: ${test.name}`)
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Mutation testing
|
||||
const mutationReport = await runMutationTesting(fastify, { runs: 10 })
|
||||
if (mutationReport.score < 80) {
|
||||
console.warn(`Low mutation score: ${mutationReport.score}%`)
|
||||
}
|
||||
```
|
||||
@@ -31,17 +31,17 @@ APOPHIS classifies failures into six categories. Lower categories take precedenc
|
||||
|
||||
**Symptoms**
|
||||
- `Unexpected token` in formula output
|
||||
- `Unterminated string` in x-ensures clause
|
||||
- `Unterminated string literal` in x-ensures clause
|
||||
- `Missing this` in operation call
|
||||
|
||||
**Resolution**
|
||||
1. Check the route and clause index printed in the error message.
|
||||
2. Verify APOSTL syntax: use `response_code(this)` not `response_code()`.
|
||||
3. Ensure string literals use single or double quotes consistently.
|
||||
4. Run `apophis doctor --profile <name>` to validate formulas without executing.
|
||||
4. Run `apophis doctor` to validate formulas without executing.
|
||||
|
||||
**Prevention**
|
||||
- Run `apophis doctor --profile <name>` to validate formulas without executing.
|
||||
- Run `apophis doctor` to validate formulas without executing.
|
||||
- Enable editor support for APOSTL syntax highlighting.
|
||||
|
||||
---
|
||||
@@ -88,12 +88,12 @@ APOPHIS classifies failures into six categories. Lower categories take precedenc
|
||||
**Symptoms**
|
||||
- `Plugin decorator already added`
|
||||
- `Duplicate route registration`
|
||||
- `No behavioral contracts found`
|
||||
- `No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.`
|
||||
|
||||
**Resolution**
|
||||
1. Ensure the APOPHIS plugin is registered exactly once in the Fastify app.
|
||||
2. Check for multiple imports or plugin registrations in test vs production entry points.
|
||||
3. If `No behavioral contracts found`, add `x-ensures` or `x-requires` to route schemas.
|
||||
3. If `No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.`, add `x-ensures` or `x-requires` to route schemas.
|
||||
4. Run `apophis doctor` to verify route discovery matches expectations.
|
||||
|
||||
**Prevention**
|
||||
@@ -139,7 +139,7 @@ APOPHIS classifies failures into six categories. Lower categories take precedenc
|
||||
|
||||
**Prevention**
|
||||
- Use `apophis verify --seed <number>` for deterministic runs.
|
||||
- Run `apophis observe` in CI to catch drift before it becomes a failure.
|
||||
- Run `apophis doctor --mode observe` in CI to catch config drift before deployment.
|
||||
- Keep test data isolated and reset between runs.
|
||||
|
||||
---
|
||||
@@ -150,13 +150,13 @@ Every failure produces an artifact JSON file. Use it for deep triage:
|
||||
|
||||
```bash
|
||||
# Inspect the artifact
|
||||
cat reports/apophis/verify-<timestamp>.json | jq '.failures[0]'
|
||||
cat reports/apophis/failure-<timestamp>.json | jq '.failures[0]'
|
||||
|
||||
# Replay the exact failure
|
||||
apophis replay --artifact reports/apophis/verify-<timestamp>.json
|
||||
apophis replay --artifact reports/apophis/failure-<timestamp>.json
|
||||
|
||||
# Filter by error category
|
||||
cat reports/apophis/verify-<timestamp>.json | jq '.failures | map(select(.category == "runtime"))'
|
||||
cat reports/apophis/failure-<timestamp>.json | jq '.failures | map(select(.category == "runtime"))'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+46
-36
@@ -2,15 +2,7 @@
|
||||
|
||||
Deterministic contract verification for CI and local development.
|
||||
|
||||
## What Verify Does
|
||||
|
||||
`apophis verify` runs behavioral contracts against your Fastify routes:
|
||||
|
||||
1. Discovers routes from your Fastify app
|
||||
2. Filters routes by profile config and CLI flags
|
||||
3. Generates test data from JSON Schema
|
||||
4. Executes routes and checks `x-ensures` contracts
|
||||
5. Reports pass/fail with deterministic seed and replay command
|
||||
APOPHIS implements invariant-driven testing: encode intended behavior as executable formulas, then verify them automatically with property-based generation and deterministic replay.
|
||||
|
||||
## When to Use It
|
||||
|
||||
@@ -79,6 +71,8 @@ apophis verify --routes "POST /users/*"
|
||||
apophis verify --profile quick
|
||||
```
|
||||
|
||||
`*` and `?` wildcards are supported in `--routes`.
|
||||
|
||||
## `--changed` Flag
|
||||
|
||||
Run only routes modified in the current git branch:
|
||||
@@ -87,7 +81,9 @@ Run only routes modified in the current git branch:
|
||||
apophis verify --profile ci --changed
|
||||
```
|
||||
|
||||
If no routes changed, exits 0 with a message.
|
||||
If no routes changed, exits 2 with a message.
|
||||
|
||||
`--changed` tracks which source file registered each route by capturing stack traces during route discovery. When a file changes, only routes registered from that file are verified. Falls back to a path-segment heuristic for routes discovered without source file metadata (e.g., `printRoutes()` fallback).
|
||||
|
||||
## Failure Output Format
|
||||
|
||||
@@ -126,6 +122,8 @@ Next
|
||||
apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
|
||||
```
|
||||
|
||||
Nondeterminism warnings appear in output when the same seed produces different results across runs. This indicates stateful behavior in your application that contracts cannot control.
|
||||
|
||||
## Machine Output for CI
|
||||
|
||||
Use concise formats to reduce log volume in large verify runs:
|
||||
@@ -137,6 +135,7 @@ Use concise formats to reduce log volume in large verify runs:
|
||||
|
||||
```bash
|
||||
# Extract only failed routes from full ndjson
|
||||
# Note: route.failed events are only emitted for failures, not passed routes
|
||||
apophis verify --profile quick --format ndjson | jq 'select(.type == "route.failed")'
|
||||
|
||||
# Write artifact to disk and parse the file instead of stdout
|
||||
@@ -149,7 +148,7 @@ apophis verify --profile quick --format json --artifact-dir reports/apophis
|
||||
|---|---|
|
||||
| 0 | All contracts passed |
|
||||
| 1 | One or more behavioral contracts failed |
|
||||
| 2 | Config error or no routes matched |
|
||||
| 2 | Config error, no routes matched, no contracts found, or not a git repo |
|
||||
| 3 | Internal APOPHIS error |
|
||||
| 130 | Interrupted (SIGINT) |
|
||||
|
||||
@@ -158,42 +157,53 @@ apophis verify --profile quick --format json --artifact-dir reports/apophis
|
||||
```javascript
|
||||
// apophis.config.js
|
||||
export default {
|
||||
mode: 'verify',
|
||||
profile: 'quick',
|
||||
profiles: {
|
||||
quick: {
|
||||
name: 'quick',
|
||||
mode: 'verify',
|
||||
preset: 'safe-ci',
|
||||
routes: ['POST /users']
|
||||
},
|
||||
ci: {
|
||||
name: 'ci',
|
||||
mode: 'verify',
|
||||
preset: 'safe-ci',
|
||||
routes: []
|
||||
}
|
||||
},
|
||||
presets: {
|
||||
'safe-ci': {
|
||||
name: 'safe-ci',
|
||||
depth: 'quick',
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
observe: false
|
||||
}
|
||||
},
|
||||
environments: {
|
||||
local: {
|
||||
name: 'local',
|
||||
allowVerify: true,
|
||||
allowObserve: true,
|
||||
allowQualify: false,
|
||||
allowChaos: false,
|
||||
allowBlocking: true,
|
||||
requireSink: false
|
||||
runs: 10,
|
||||
timeout: 5000
|
||||
}
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
For the full config schema, see [CLI Reference](cli.md).
|
||||
|
||||
## Workspace Support
|
||||
|
||||
Run verify across all packages in a monorepo workspace:
|
||||
|
||||
```bash
|
||||
apophis verify --workspace --profile quick --format json
|
||||
```
|
||||
|
||||
Output includes per-package pass/fail summaries. Fails if any package fails.
|
||||
|
||||
## Test Budget
|
||||
|
||||
The `runs` field in your preset controls how many property-based test samples execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:
|
||||
|
||||
```javascript
|
||||
profiles: {
|
||||
quick: {
|
||||
mode: 'verify',
|
||||
preset: 'safe-ci',
|
||||
routes: ['POST /users']
|
||||
}
|
||||
},
|
||||
presets: {
|
||||
'safe-ci': {
|
||||
runs: 10,
|
||||
timeout: 5000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
CLI verify defaults to 50 runs per contract. Set `runs` in the preset to adjust the sample count per route. Use `runs: 1` to check each contract once, or `runs: 0` to disable property-based verification.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Fastify from 'fastify'
|
||||
import swagger from '@fastify/swagger'
|
||||
import { apophisPlugin } from 'apophis-fastify'
|
||||
import { apophisPlugin } from '@apophis/fastify'
|
||||
import { databasePlugin } from './plugins/database.js'
|
||||
import { userRoutes } from './routes/users.js'
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import crypto from 'crypto'
|
||||
import type { FastifyInstance } from 'fastify'
|
||||
|
||||
export async function userRoutes(fastify: FastifyInstance) {
|
||||
@@ -30,7 +31,7 @@ export async function userRoutes(fastify: FastifyInstance) {
|
||||
},
|
||||
}, async (req, reply) => {
|
||||
const { name, email } = req.body as { name: string; email: string }
|
||||
const id = `user-${Date.now()}`
|
||||
const id = `user-${crypto.createHash('sha256').update(email).digest('hex').slice(0, 8)}`
|
||||
const user = { id, name, email }
|
||||
fastify.db.users.set(id, user)
|
||||
reply.status(201)
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
Mercedes Rodriguez
|
||||
#BOAF9148155679Z
|
||||
1500 - 00251743R
|
||||
2000 - 00361Z903R
|
||||
|
||||
Generated
+9
-112
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "apophis-fastify",
|
||||
"version": "2.0.0",
|
||||
"name": "@apophis/fastify",
|
||||
"version": "2.7.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "apophis-fastify",
|
||||
"version": "2.0.0",
|
||||
"license": "ISC",
|
||||
"name": "@apophis/fastify",
|
||||
"version": "2.7.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^0.10.1",
|
||||
"cac": "^6.7.14",
|
||||
@@ -15,7 +15,6 @@
|
||||
"fastify-plugin": "^5.0.0",
|
||||
"picocolors": "^1.0.0",
|
||||
"pino": "^10.3.1",
|
||||
"recheck": "^4.5.0",
|
||||
"safe-regex": "^2.1.1",
|
||||
"undici": "^7.0.0"
|
||||
},
|
||||
@@ -33,6 +32,9 @@
|
||||
"tsx": "^4.0.0",
|
||||
"typescript": "^6.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.18.1 <21 || >=22 <23"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@fastify/swagger": "^9.0.0",
|
||||
"fastify": "^5.0.0"
|
||||
@@ -1573,18 +1575,6 @@
|
||||
"integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pkgr/core": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.2.tgz",
|
||||
"integrity": "sha512-fdDH1LSGfZdTH2sxdpVMw31BanV28K/Gry0cVFxaNP77neJSkd82mM8ErPNYs9e+0O7SdHBLTDzDgwUuy18RnQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.18.0 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/unts"
|
||||
}
|
||||
},
|
||||
"node_modules/@rollup/rollup-android-arm-eabi": {
|
||||
"version": "4.60.2",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz",
|
||||
@@ -3721,84 +3711,6 @@
|
||||
"node": ">= 12.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/recheck": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/recheck/-/recheck-4.5.0.tgz",
|
||||
"integrity": "sha512-kPnbOV6Zfx9a25AZ++28fI1q78L/UVRQmmuazwVRPfiiqpMs+WbOU69Shx820XgfKWfak0JH75PUvZMFtRGSsw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"synckit": "0.9.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"recheck-jar": "4.5.0",
|
||||
"recheck-linux-x64": "4.5.0",
|
||||
"recheck-macos-arm64": "4.5.0",
|
||||
"recheck-macos-x64": "4.5.0",
|
||||
"recheck-windows-x64": "4.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/recheck-jar": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/recheck-jar/-/recheck-jar-4.5.0.tgz",
|
||||
"integrity": "sha512-Ad7oCQmY8cQLzd3QVNXjzZ+S6MbImGhR4AaW2yiGzteOfMV45522rt6nSzFyt8p3mCEaMcm/4MoZrMSxUcCbrA==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/recheck-linux-x64": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/recheck-linux-x64/-/recheck-linux-x64-4.5.0.tgz",
|
||||
"integrity": "sha512-52kXsR/v+IbGIKYYFZfSZcgse/Ci9IA2HnuzrtvRRcfODkcUGe4n72ESQ8nOPwrdHFg9i4j9/YyPh1HWWgpJ6A==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/recheck-macos-arm64": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/recheck-macos-arm64/-/recheck-macos-arm64-4.5.0.tgz",
|
||||
"integrity": "sha512-qIyK3dRuLkORQvv0b59fZZRXweSmjjWaoA4K8Kgifz0anMBH4pqsDV6plBlgjcRmW9yC12wErIRzifREaKnk2w==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/recheck-macos-x64": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/recheck-macos-x64/-/recheck-macos-x64-4.5.0.tgz",
|
||||
"integrity": "sha512-1wp/eiLxcjC/Ex4wurlrS/LGzt8IiF4TiK5sEjldu4HVAKdNCnnmsS9a5vFpfcikDz4ZuZlLlTi1VbQTxHlwZg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/recheck-windows-x64": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/recheck-windows-x64/-/recheck-windows-x64-4.5.0.tgz",
|
||||
"integrity": "sha512-ekBKwAp0oKkMULn5zgmHEYLwSJfkfb95AbTtbDkQazNkqYw9PRD/mVyFUR6Ff2IeRyZI0gxy+N2AKBISWydhug==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/regexp-tree": {
|
||||
"version": "0.1.27",
|
||||
"resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.27.tgz",
|
||||
@@ -4208,22 +4120,6 @@
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/synckit": {
|
||||
"version": "0.9.2",
|
||||
"resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz",
|
||||
"integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pkgr/core": "^0.1.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^14.18.0 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/unts"
|
||||
}
|
||||
},
|
||||
"node_modules/thenify": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz",
|
||||
@@ -4314,6 +4210,7 @@
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"dev": true,
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/tsup": {
|
||||
|
||||
+27
-14
@@ -1,9 +1,9 @@
|
||||
{
|
||||
"name": "apophis-fastify",
|
||||
"version": "2.0.0",
|
||||
"name": "@apophis/fastify",
|
||||
"version": "2.7.0",
|
||||
"description": "Contract-driven API testing plugin for Fastify with property-based testing, timeout enforcement, redirect capture, and deterministic concurrency",
|
||||
"main": "dist/index.js",
|
||||
"types": "index.d.ts",
|
||||
"types": "dist/index.d.ts",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"apophis": "dist/cli/index.js"
|
||||
@@ -11,35 +11,47 @@
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"types": "./index.d.ts"
|
||||
"types": "./dist/index.d.ts"
|
||||
},
|
||||
"./extension/factories": {
|
||||
"import": "./dist/extension/factories.js",
|
||||
"types": "./dist/extension/factories.d.ts"
|
||||
},
|
||||
"./extensions": {
|
||||
"import": "./dist/extensions/index.js",
|
||||
"types": "./dist/extensions/index.d.ts"
|
||||
},
|
||||
"./extensions/sse": {
|
||||
"import": "./dist/extensions/sse/index.js",
|
||||
"types": "./dist/extensions/sse/index.d.ts"
|
||||
},
|
||||
"./extensions/websocket": {
|
||||
"import": "./dist/extensions/websocket/index.js",
|
||||
"types": "./dist/extensions/websocket/index.d.ts"
|
||||
},
|
||||
"./extensions/serializers": {
|
||||
"import": "./dist/extensions/serializers/index.js",
|
||||
"types": "./dist/extensions/serializers/index.d.ts"
|
||||
},
|
||||
"./extensions/*": {
|
||||
"import": "./dist/extensions/*.js",
|
||||
"types": "./dist/extensions/*.d.ts"
|
||||
},
|
||||
"./quality/*": {
|
||||
"import": "./dist/quality/*.js",
|
||||
"types": "./dist/quality/*.d.ts"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"index.d.ts",
|
||||
"README.md",
|
||||
"LICENSE",
|
||||
"docs"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.0.0 || ^22.0.0"
|
||||
"node": ">=20.18.1 <21 || >=22 <23"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepare": "npm run build",
|
||||
"build": "tsc -p tsconfig.build.json && chmod +x dist/cli/index.js",
|
||||
"test": "npm run build && npm run test:src && npm run test:cli",
|
||||
"test:dist": "NODE_ENV=test node --test dist/test/*.test.js",
|
||||
|
||||
"test:src": "tsx --test src/test/*.test.ts",
|
||||
"test:cli": "tsx --test src/test/cli/*.test.ts",
|
||||
"test:cli:goldens": "tsx --test src/test/cli/goldens.test.ts",
|
||||
@@ -50,8 +62,10 @@
|
||||
"benchmark:cli": "npm run build && node scripts/bench/cli.mjs",
|
||||
"benchmark:hot": "npm run build && node scripts/bench/hot-paths.mjs",
|
||||
"profile:qualify": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet",
|
||||
"profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --generation-profile quick --seed 42 --quiet",
|
||||
"profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet",
|
||||
"clean": "rm -rf dist",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "tsc --noEmit",
|
||||
"apophis:verify": "apophis verify --profile quick",
|
||||
"apophis:doctor": "apophis doctor"
|
||||
},
|
||||
@@ -83,7 +97,6 @@
|
||||
"fastify-plugin": "^5.0.0",
|
||||
"picocolors": "^1.0.0",
|
||||
"pino": "^10.3.1",
|
||||
"recheck": "^4.5.0",
|
||||
"safe-regex": "^2.1.1",
|
||||
"undici": "^7.0.0"
|
||||
},
|
||||
|
||||
+1
-18
@@ -8,30 +8,13 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url))
|
||||
const repoRoot = resolve(__dirname, '..', '..')
|
||||
|
||||
const options = getBenchOptions()
|
||||
const generationProfiles = (process.env.BENCH_GENERATION_PROFILES ?? 'default,quick,standard,thorough')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
|
||||
function withGenerationProfile(baseArgs, profile) {
|
||||
if (profile === 'default') {
|
||||
return baseArgs
|
||||
}
|
||||
return [...baseArgs, '--generation-profile', profile]
|
||||
}
|
||||
|
||||
const scenarios = [
|
||||
{ name: 'cli.help', args: ['--help'] },
|
||||
{ name: 'cli.version', args: ['--version'] },
|
||||
{ name: 'cli.doctor', args: ['doctor', '--cwd', 'src/cli/__fixtures__/tiny-fastify', '--quiet'] },
|
||||
{ name: 'cli.observe.check', args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe', '--check-config', '--quiet'] },
|
||||
...generationProfiles.map((profile) => ({
|
||||
name: `cli.qualify.profile[${profile}]`,
|
||||
args: withGenerationProfile(
|
||||
['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'],
|
||||
profile,
|
||||
),
|
||||
})),
|
||||
{ name: 'cli.qualify', args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'] },
|
||||
]
|
||||
|
||||
async function run() {
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Fastify module augmentation — declares the apophis property on FastifyInstance.
|
||||
* This makes fastify.apophis typed correctly in consumer TypeScript projects.
|
||||
*/
|
||||
import type { ApophisDecorations } from './types/core.js'
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyInstance {
|
||||
readonly apophis: ApophisDecorations
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for broken-behavior fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "verify",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -2,11 +2,9 @@
|
||||
* LEGACY APOPHIS configuration (old-style, for migration tests).
|
||||
* This uses deprecated field names that should be detected by `apophis migrate`.
|
||||
*/
|
||||
|
||||
export default {
|
||||
// Deprecated: 'mode' used to be 'testMode'
|
||||
testMode: "verify",
|
||||
|
||||
// Deprecated: 'profiles' used to be 'testProfiles'
|
||||
testProfiles: {
|
||||
quick: {
|
||||
@@ -17,7 +15,6 @@ export default {
|
||||
routeFilter: ["GET /legacy"],
|
||||
},
|
||||
},
|
||||
|
||||
// Deprecated: 'presets' used to be 'testPresets'
|
||||
testPresets: {
|
||||
"safe-ci": {
|
||||
@@ -28,7 +25,6 @@ export default {
|
||||
maxDuration: 5000,
|
||||
},
|
||||
},
|
||||
|
||||
// Deprecated: 'environments' used to be 'envPolicies'
|
||||
envPolicies: {
|
||||
local: {
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
* Root-level APOPHIS config for monorepo.
|
||||
* Packages can override with their own configs.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "verify",
|
||||
profiles: {
|
||||
@@ -20,7 +19,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for observe-config fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "observe",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
"observe-safe": {
|
||||
name: "observe-safe",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
export default {
|
||||
profiles: { quick: { preset: "safe-ci" } },
|
||||
presets: { "safe-ci": { runs: 3 } },
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import Fastify from "fastify";
|
||||
|
||||
const app = Fastify({ logger: false });
|
||||
|
||||
await app.register(import("@fastify/swagger"), {
|
||||
openapi: { info: { title: "Plugin Contracts Fixture", version: "1.0.0" } },
|
||||
});
|
||||
|
||||
let apophisPlugin;
|
||||
try {
|
||||
({ default: apophisPlugin } = await import("../../../index.js"));
|
||||
} catch {
|
||||
({ default: apophisPlugin } = await import("../../../../dist/index.js"));
|
||||
}
|
||||
|
||||
await app.register(apophisPlugin, {
|
||||
runtime: "off",
|
||||
pluginContracts: {
|
||||
"required-header": {
|
||||
appliesTo: "/api/**",
|
||||
hooks: { onRequest: { requires: ["request_headers(this).x-custom != null"] } },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
app.get("/api/data", {
|
||||
schema: {
|
||||
"x-category": "observer",
|
||||
"x-ensures": ["status:200"],
|
||||
response: { 200: { type: "object", properties: { ok: { type: "boolean" } } } },
|
||||
},
|
||||
}, async () => ({ ok: true }));
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1 @@
|
||||
{"name":"plugin-contracts-fixture","type":"module","private":true}
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for protocol-lab fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "qualify",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
deep: {
|
||||
name: "deep",
|
||||
depth: "deep",
|
||||
timeout: 30000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for tiny-fastify fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "verify",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -11,7 +11,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -11,7 +11,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -11,7 +11,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -6,7 +6,7 @@ Options:
|
||||
--config <path> Path to config file
|
||||
--profile <name> Profile name from config
|
||||
--cwd <path> Working directory
|
||||
--format <human|json|ndjson> Output format (default: human)
|
||||
--format <human|json|ndjson|json-summary|ndjson-summary> Output format (default: human)
|
||||
--color <auto|always|never> Color mode (default: auto)
|
||||
--quiet Suppress non-essential output
|
||||
--verbose Verbose output
|
||||
|
||||
@@ -10,13 +10,11 @@
|
||||
|
||||
import {
|
||||
loadConfig,
|
||||
loadConfigFile,
|
||||
discoverConfig,
|
||||
ConfigValidationError,
|
||||
type Config,
|
||||
type LoadConfigResult,
|
||||
} from '../../../core/config-loader.js';
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -35,93 +33,10 @@ export interface ConfigCheckOptions {
|
||||
configPath?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Legacy field detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Map of deprecated field names to their modern equivalents.
|
||||
* Legacy config detection removed — always passes.
|
||||
*/
|
||||
const LEGACY_FIELDS: Record<string, string> = {
|
||||
testMode: 'mode',
|
||||
testProfiles: 'profiles',
|
||||
testPresets: 'presets',
|
||||
envPolicies: 'environments',
|
||||
usesPreset: 'preset',
|
||||
routeFilter: 'routes',
|
||||
testDepth: 'depth',
|
||||
maxDuration: 'timeout',
|
||||
canVerify: 'allowVerify',
|
||||
};
|
||||
|
||||
/**
|
||||
* Recursively scan an object for legacy field names.
|
||||
* Returns array of { path, legacyKey, modernKey } tuples.
|
||||
*/
|
||||
function findLegacyFields(
|
||||
value: unknown,
|
||||
path: string = '',
|
||||
): Array<{ path: string; legacyKey: string; modernKey: string }> {
|
||||
const results: Array<{ path: string; legacyKey: string; modernKey: string }> = [];
|
||||
|
||||
if (value === null || typeof value !== 'object') {
|
||||
return results;
|
||||
}
|
||||
|
||||
const obj = value as Record<string, unknown>;
|
||||
|
||||
for (const key of Object.keys(obj)) {
|
||||
const currentPath = path ? `${path}.${key}` : key;
|
||||
|
||||
// Check if this key is legacy
|
||||
if (LEGACY_FIELDS[key]) {
|
||||
results.push({
|
||||
path: currentPath,
|
||||
legacyKey: key,
|
||||
modernKey: LEGACY_FIELDS[key],
|
||||
});
|
||||
}
|
||||
|
||||
// Recurse into nested objects
|
||||
const fieldValue = obj[key];
|
||||
if (fieldValue !== null && typeof fieldValue === 'object' && !Array.isArray(fieldValue)) {
|
||||
results.push(...findLegacyFields(fieldValue, currentPath));
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if config contains legacy field names.
|
||||
*/
|
||||
export function checkLegacyConfig(config: Config | null): ConfigCheckResult {
|
||||
if (!config) {
|
||||
return {
|
||||
name: 'legacy-config',
|
||||
status: 'pass',
|
||||
message: 'No config to check for legacy fields.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
const legacyFields = findLegacyFields(config);
|
||||
|
||||
if (legacyFields.length > 0) {
|
||||
const details = legacyFields
|
||||
.map(f => ` ${f.path}: "${f.legacyKey}" → "${f.modernKey}"`)
|
||||
.join('\n');
|
||||
|
||||
return {
|
||||
name: 'legacy-config',
|
||||
status: 'warn',
|
||||
message: `Found ${legacyFields.length} legacy field(s) in config.`,
|
||||
detail: `Run "apophis migrate" to update these fields:\n${details}`,
|
||||
remediation: 'Run "apophis migrate --dry-run" to preview rewrites.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
export function checkLegacyConfig(_config: Config | null): ConfigCheckResult {
|
||||
return {
|
||||
name: 'legacy-config',
|
||||
status: 'pass',
|
||||
@@ -130,73 +45,7 @@ export function checkLegacyConfig(config: Config | null): ConfigCheckResult {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for mixed legacy and new config styles.
|
||||
* This happens when some fields use old names and others use new names.
|
||||
*/
|
||||
export function checkMixedConfig(config: Config | null): ConfigCheckResult {
|
||||
if (!config) {
|
||||
return {
|
||||
name: 'mixed-config',
|
||||
status: 'pass',
|
||||
message: 'No config to check for mixed styles.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
const legacyFields = findLegacyFields(config);
|
||||
const hasLegacy = legacyFields.length > 0;
|
||||
|
||||
// Check if config also has modern fields at the same level as legacy ones
|
||||
const hasModern = Object.keys(config).some(key => !LEGACY_FIELDS[key] && key !== 'name');
|
||||
|
||||
if (hasLegacy && hasModern) {
|
||||
const legacyTopLevel = Object.keys(config).filter(key => LEGACY_FIELDS[key]);
|
||||
const modernTopLevel = Object.keys(config).filter(key => !LEGACY_FIELDS[key] && key !== 'name');
|
||||
|
||||
// Only fail if there are actual modern fields that conflict with legacy ones
|
||||
// A config with only legacy fields should warn, not fail
|
||||
const hasConflictingModern = modernTopLevel.length > 0 &&
|
||||
legacyTopLevel.some(lf => LEGACY_FIELDS[lf] !== undefined && modernTopLevel.includes(LEGACY_FIELDS[lf]));
|
||||
|
||||
if (hasConflictingModern) {
|
||||
return {
|
||||
name: 'mixed-config',
|
||||
status: 'fail',
|
||||
message: 'Config uses both legacy and modern field names.',
|
||||
detail:
|
||||
`Legacy fields: ${legacyTopLevel.join(', ')}\n` +
|
||||
`Modern fields: ${modernTopLevel.join(', ')}\n` +
|
||||
`Run "apophis migrate" to unify your config to the modern schema.`,
|
||||
remediation: 'Run "apophis migrate --write" to unify config to modern schema.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
// Has both legacy and other modern fields - still warn but don't fail
|
||||
return {
|
||||
name: 'mixed-config',
|
||||
status: 'warn',
|
||||
message: 'Config contains legacy field names alongside modern fields.',
|
||||
detail:
|
||||
`Legacy fields: ${legacyTopLevel.join(', ')}\n` +
|
||||
`Run "apophis migrate" to update to the modern schema.`,
|
||||
remediation: 'Run "apophis migrate --dry-run" to preview rewrites.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
if (hasLegacy) {
|
||||
return {
|
||||
name: 'mixed-config',
|
||||
status: 'warn',
|
||||
message: 'Config uses legacy field names only.',
|
||||
detail: 'Run "apophis migrate" to update to the modern schema.',
|
||||
remediation: 'Run "apophis migrate --write" to update to modern schema.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
export function checkMixedConfig(_config: Config | null): ConfigCheckResult {
|
||||
return {
|
||||
name: 'mixed-config',
|
||||
status: 'pass',
|
||||
@@ -306,26 +155,6 @@ export async function checkConfigLoad(options: ConfigCheckOptions): Promise<Conf
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Raw config loader (without validation)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Load raw config without schema validation.
|
||||
* Used for legacy detection when validation would fail on legacy keys.
|
||||
*/
|
||||
async function loadRawConfig(options: ConfigCheckOptions): Promise<Config | null> {
|
||||
const { cwd, configPath } = options;
|
||||
|
||||
// Discover config file
|
||||
const discoveredPath = configPath || discoverConfig(cwd);
|
||||
if (!discoveredPath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return await loadConfigFile(discoveredPath);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main config check runner
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -336,32 +165,11 @@ async function loadRawConfig(options: ConfigCheckOptions): Promise<Config | null
|
||||
export async function runConfigChecks(options: ConfigCheckOptions): Promise<ConfigCheckResult[]> {
|
||||
const results: ConfigCheckResult[] = [];
|
||||
|
||||
// 1. Check config can be loaded
|
||||
results.push(await checkConfigLoad(options));
|
||||
|
||||
// 2. Check for unknown keys
|
||||
results.push(await checkUnknownKeys(options));
|
||||
|
||||
// 3. Check for legacy fields - load raw config without validation
|
||||
try {
|
||||
const rawConfig = await loadRawConfig(options);
|
||||
results.push(checkLegacyConfig(rawConfig));
|
||||
results.push(checkMixedConfig(rawConfig));
|
||||
} catch {
|
||||
// If config can't be loaded, skip legacy/mixed checks
|
||||
results.push({
|
||||
name: 'legacy-config',
|
||||
status: 'warn',
|
||||
message: 'Could not check for legacy fields (config failed to load).',
|
||||
mode: 'all',
|
||||
});
|
||||
results.push({
|
||||
name: 'mixed-config',
|
||||
status: 'warn',
|
||||
message: 'Could not check for mixed config (config failed to load).',
|
||||
mode: 'all',
|
||||
});
|
||||
}
|
||||
results.push(checkLegacyConfig(null));
|
||||
results.push(checkMixedConfig(null));
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ export interface DependencyCheckOptions {
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MIN_NODE_VERSION = 18;
|
||||
const MIN_NODE_VERSION = 20;
|
||||
const REQUIRED_PEER_DEPS = ['fastify', '@fastify/swagger'];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -54,13 +54,27 @@ function parseNodeMajor(version: string): number {
|
||||
export function checkNodeVersion(nodeVersion: string): DependencyCheckResult {
|
||||
const major = parseNodeMajor(nodeVersion);
|
||||
|
||||
if (major < MIN_NODE_VERSION) {
|
||||
const tooOld = major < MIN_NODE_VERSION;
|
||||
const untested = major >= 23 || major === 21;
|
||||
|
||||
if (tooOld) {
|
||||
return {
|
||||
name: 'node-version',
|
||||
status: 'fail',
|
||||
message: `Node.js ${nodeVersion} is not supported. Minimum required: ${MIN_NODE_VERSION}.x`,
|
||||
detail: `APOPHIS requires Node.js ${MIN_NODE_VERSION} or higher for ESM and modern features.`,
|
||||
remediation: `Upgrade Node.js to ${MIN_NODE_VERSION}.x or higher (use nvm, fnm, or your package manager).`,
|
||||
message: `Node.js ${nodeVersion} is not supported. APOPHIS requires Node.js >=${MIN_NODE_VERSION}.18.1.`,
|
||||
detail: `Detected Node.js ${nodeVersion} (major ${major}). APOPHIS requires Node ${MIN_NODE_VERSION} LTS or 22 LTS.`,
|
||||
remediation: `Install Node.js ${MIN_NODE_VERSION}.x or 22.x (use nvm, fnm, or your package manager).`,
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
if (untested) {
|
||||
return {
|
||||
name: 'node-version',
|
||||
status: 'warn',
|
||||
message: `Node.js ${nodeVersion} is not in the tested range (20.x or 22.x).`,
|
||||
detail: `Detected Node.js ${nodeVersion} (major ${major}). APOPHIS is tested on Node 20 LTS and 22 LTS.`,
|
||||
remediation: `Use Node.js 20.x or 22.x for best compatibility. Current version may work but is untested.`,
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
@@ -68,7 +82,7 @@ export function checkNodeVersion(nodeVersion: string): DependencyCheckResult {
|
||||
return {
|
||||
name: 'node-version',
|
||||
status: 'pass',
|
||||
message: `Node.js ${nodeVersion} meets minimum requirement (${MIN_NODE_VERSION}+)`,
|
||||
message: `Node.js ${nodeVersion} is supported (>=${MIN_NODE_VERSION}.18.1 <21 || >=22 <23)`,
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -78,25 +78,11 @@ export function checkDocsExist(options: DocsCheckOptions): DocsCheckResult {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Known legacy field names that should not appear in docs.
|
||||
*/
|
||||
const LEGACY_FIELD_NAMES = [
|
||||
'testMode',
|
||||
'testProfiles',
|
||||
'testPresets',
|
||||
'envPolicies',
|
||||
'usesPreset',
|
||||
'routeFilter',
|
||||
'testDepth',
|
||||
'maxDuration',
|
||||
'canVerify',
|
||||
];
|
||||
|
||||
/**
|
||||
* Check if docs contain legacy field names (indicating stale docs).
|
||||
* Check if docs examples match current config schema.
|
||||
* Legacy field name detection removed — always passes.
|
||||
*/
|
||||
export function checkDocsSchemaDrift(options: DocsCheckOptions): DocsCheckResult {
|
||||
const { cwd, isCI } = options;
|
||||
const { cwd } = options;
|
||||
|
||||
const docsFiles = findDocsFiles(cwd);
|
||||
|
||||
@@ -109,36 +95,6 @@ export function checkDocsSchemaDrift(options: DocsCheckOptions): DocsCheckResult
|
||||
};
|
||||
}
|
||||
|
||||
const drift: Array<{ file: string; legacyFields: string[] }> = [];
|
||||
|
||||
for (const file of docsFiles) {
|
||||
try {
|
||||
const content = readFileSync(file, 'utf-8');
|
||||
const foundLegacy = LEGACY_FIELD_NAMES.filter(field => content.includes(field));
|
||||
|
||||
if (foundLegacy.length > 0) {
|
||||
drift.push({ file, legacyFields: foundLegacy });
|
||||
}
|
||||
} catch {
|
||||
// Skip unreadable files
|
||||
}
|
||||
}
|
||||
|
||||
if (drift.length > 0) {
|
||||
const details = drift
|
||||
.map(d => ` ${d.file}: ${d.legacyFields.join(', ')}`)
|
||||
.join('\n');
|
||||
|
||||
return {
|
||||
name: 'docs-schema-drift',
|
||||
status: isCI ? 'fail' : 'warn',
|
||||
message: `Found ${drift.length} documentation file(s) with legacy field names.`,
|
||||
detail: `Update docs to use current config schema:\n${details}\n\nRun "apophis migrate --dry-run" to see rewrites.`,
|
||||
remediation: 'Update docs to use current field names, or run "apophis migrate --dry-run" to see rewrites.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'docs-schema-drift',
|
||||
status: 'pass',
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* S8: Doctor thread - Extension registry health checks
|
||||
*
|
||||
* Checks:
|
||||
* - Extension registration status
|
||||
* - Extension health check results
|
||||
* - Production safety of registered extensions
|
||||
*/
|
||||
|
||||
import type { ExtensionRegistry, ApophisExtension } from '../../../../extension/types.js'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ExtensionCheckResult {
|
||||
name: string
|
||||
status: 'pass' | 'fail' | 'warn'
|
||||
message: string
|
||||
detail?: string
|
||||
remediation?: string
|
||||
mode: 'all' | 'verify' | 'observe' | 'qualify'
|
||||
}
|
||||
|
||||
export interface ExtensionCheckOptions {
|
||||
extensionRegistry?: ExtensionRegistry
|
||||
env?: string
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extension presence
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function checkExtensionPresence(options: ExtensionCheckOptions): ExtensionCheckResult {
|
||||
const registry = options.extensionRegistry
|
||||
|
||||
if (!registry || registry.extensions.length === 0) {
|
||||
return {
|
||||
name: 'extensions-presence',
|
||||
status: 'pass',
|
||||
message: 'No extensions registered (optional)',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const names = registry.extensions.map((e: ApophisExtension) => e.name).join(', ')
|
||||
return {
|
||||
name: 'extensions-presence',
|
||||
status: 'pass',
|
||||
message: `${registry.extensions.length} extension(s) registered: ${names}`,
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extension health
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function checkExtensionHealth(options: ExtensionCheckOptions): Promise<ExtensionCheckResult> {
|
||||
const registry = options.extensionRegistry
|
||||
|
||||
if (!registry || registry.extensions.length === 0) {
|
||||
return {
|
||||
name: 'extensions-health',
|
||||
status: 'pass',
|
||||
message: 'No extensions to health-check',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const results = await registry.runHealthChecks()
|
||||
const unhealthy = results.filter((r: { name: string; error?: string }) => r.error)
|
||||
|
||||
if (unhealthy.length === 0) {
|
||||
return {
|
||||
name: 'extensions-health',
|
||||
status: 'pass',
|
||||
message: 'All extensions passed health checks',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const details = unhealthy.map((r: { name: string; error?: string }) => ` ${r.name}: ${r.error}`).join('\n')
|
||||
return {
|
||||
name: 'extensions-health',
|
||||
status: 'warn',
|
||||
message: `${unhealthy.length} extension(s) failed health checks`,
|
||||
detail: details,
|
||||
remediation: 'Review extension configuration and dependencies',
|
||||
mode: 'all',
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
name: 'extensions-health',
|
||||
status: 'warn',
|
||||
message: 'Could not run extension health checks',
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Production safety
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function checkExtensionProductionSafety(options: ExtensionCheckOptions): ExtensionCheckResult {
|
||||
const registry = options.extensionRegistry
|
||||
const env = options.env || process.env.NODE_ENV || 'local'
|
||||
|
||||
if (!registry || registry.extensions.length === 0) {
|
||||
return {
|
||||
name: 'extensions-production-safety',
|
||||
status: 'pass',
|
||||
message: 'No extensions to check',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const isProd = env === 'production' || env === 'prod'
|
||||
if (!isProd) {
|
||||
return {
|
||||
name: 'extensions-production-safety',
|
||||
status: 'pass',
|
||||
message: 'Not in production — extension safety check skipped',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const unsafe = registry.extensions.filter((e: ApophisExtension) => e.productionSafe === false)
|
||||
|
||||
if (unsafe.length === 0) {
|
||||
return {
|
||||
name: 'extensions-production-safety',
|
||||
status: 'pass',
|
||||
message: 'All extensions are production-safe',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const names = unsafe.map((e: ApophisExtension) => e.name).join(', ')
|
||||
return {
|
||||
name: 'extensions-production-safety',
|
||||
status: 'warn',
|
||||
message: `${unsafe.length} extension(s) not marked production-safe`,
|
||||
detail: `Unsafe extensions: ${names}`,
|
||||
remediation: 'Remove these extensions from production config or mark them productionSafe',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Aggregation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function runExtensionChecks(options: ExtensionCheckOptions): Promise<ExtensionCheckResult[]> {
|
||||
return [
|
||||
checkExtensionPresence(options),
|
||||
await checkExtensionHealth(options),
|
||||
checkExtensionProductionSafety(options),
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/**
|
||||
* S8: Doctor thread - Plugin contract health checks
|
||||
*
|
||||
* Checks:
|
||||
* - Plugin contract registration
|
||||
* - Extension availability for plugin contracts
|
||||
* - Configuration warnings from plugin contract registry
|
||||
*/
|
||||
|
||||
import type { PluginContractRegistry } from '../../../../domain/plugin-contracts.js'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface PluginContractCheckResult {
|
||||
name: string
|
||||
status: 'pass' | 'fail' | 'warn'
|
||||
message: string
|
||||
detail?: string
|
||||
remediation?: string
|
||||
mode: 'all' | 'verify' | 'observe' | 'qualify'
|
||||
}
|
||||
|
||||
export interface PluginContractCheckOptions {
|
||||
pluginContractRegistry?: PluginContractRegistry
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Plugin contract presence
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function checkPluginContractPresence(options: PluginContractCheckOptions): PluginContractCheckResult {
|
||||
const registry = options.pluginContractRegistry
|
||||
|
||||
if (!registry || registry.getPluginNames().length === 0) {
|
||||
return {
|
||||
name: 'plugin-contracts-presence',
|
||||
status: 'pass',
|
||||
message: 'No plugin contracts configured (optional)',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const names = registry.getPluginNames()
|
||||
return {
|
||||
name: 'plugin-contracts-presence',
|
||||
status: 'pass',
|
||||
message: `${names.length} plugin contract(s) registered: ${names.join(', ')}`,
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Plugin contract extension availability
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function checkPluginContractExtensions(options: PluginContractCheckOptions): PluginContractCheckResult {
|
||||
const registry = options.pluginContractRegistry
|
||||
|
||||
if (!registry || registry.getPluginNames().length === 0) {
|
||||
return {
|
||||
name: 'plugin-contracts-extensions',
|
||||
status: 'pass',
|
||||
message: 'No plugin contracts to check for extension dependencies',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const available = new Set(registry.getAvailableExtensions())
|
||||
const missingByPlugin: Record<string, string[]> = {}
|
||||
|
||||
for (const name of registry.getPluginNames()) {
|
||||
const spec = registry.getPluginSpec(name)
|
||||
if (spec?.extensions) {
|
||||
const missing = spec.extensions
|
||||
.filter(e => e.required !== false)
|
||||
.map(e => e.name)
|
||||
.filter(n => !available.has(n))
|
||||
if (missing.length > 0) {
|
||||
missingByPlugin[name] = missing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(missingByPlugin).length === 0) {
|
||||
return {
|
||||
name: 'plugin-contracts-extensions',
|
||||
status: 'pass',
|
||||
message: 'All plugin contract extension dependencies are available',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const details = Object.entries(missingByPlugin)
|
||||
.map(([plugin, exts]) => ` ${plugin}: missing [${exts.join(', ')}]`)
|
||||
.join('\n')
|
||||
|
||||
return {
|
||||
name: 'plugin-contracts-extensions',
|
||||
status: 'warn',
|
||||
message: 'Some plugin contracts have missing extension dependencies',
|
||||
detail: `Missing extensions:\n${details}`,
|
||||
remediation: 'Register the required extensions via app.register(apophisPlugin, { extensions: [...] })',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Plugin contract warnings
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function checkPluginContractWarnings(options: PluginContractCheckOptions): PluginContractCheckResult {
|
||||
const registry = options.pluginContractRegistry
|
||||
|
||||
if (!registry) {
|
||||
return {
|
||||
name: 'plugin-contracts-warnings',
|
||||
status: 'pass',
|
||||
message: 'No plugin contract registry available',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
const warnings = registry.drainWarnings()
|
||||
|
||||
if (warnings.length === 0) {
|
||||
return {
|
||||
name: 'plugin-contracts-warnings',
|
||||
status: 'pass',
|
||||
message: 'No plugin contract configuration warnings',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'plugin-contracts-warnings',
|
||||
status: 'warn',
|
||||
message: `${warnings.length} plugin contract warning(s)`,
|
||||
detail: warnings.join('\n'),
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Aggregation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function runPluginContractChecks(options: PluginContractCheckOptions): PluginContractCheckResult[] {
|
||||
return [
|
||||
checkPluginContractPresence(options),
|
||||
checkPluginContractExtensions(options),
|
||||
checkPluginContractWarnings(options),
|
||||
]
|
||||
}
|
||||
@@ -7,8 +7,7 @@
|
||||
* - Is the app file loadable?
|
||||
*/
|
||||
|
||||
import { existsSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { APP_CANDIDATES, findAppFile } from '../../../core/app-loader.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -28,38 +27,6 @@ export interface RouteCheckOptions {
|
||||
configPath?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// App file detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const APP_CANDIDATES = [
|
||||
'app.js',
|
||||
'app.ts',
|
||||
'server.js',
|
||||
'server.ts',
|
||||
'index.js',
|
||||
'index.ts',
|
||||
'src/app.js',
|
||||
'src/app.ts',
|
||||
'src/server.js',
|
||||
'src/server.ts',
|
||||
'src/index.js',
|
||||
'src/index.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
* Find the Fastify app entrypoint file.
|
||||
*/
|
||||
function findAppFile(cwd: string): string | null {
|
||||
for (const candidate of APP_CANDIDATES) {
|
||||
const fullPath = resolve(cwd, candidate);
|
||||
if (existsSync(fullPath)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if app file exists and is readable.
|
||||
*/
|
||||
@@ -106,7 +73,7 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
|
||||
}
|
||||
|
||||
try {
|
||||
const appPath = resolve(options.cwd, appFile);
|
||||
const appPath = appFile;
|
||||
const appModule = await import(appPath);
|
||||
const app = appModule.default || appModule;
|
||||
|
||||
@@ -116,8 +83,8 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
|
||||
name: 'route-discovery',
|
||||
status: 'fail',
|
||||
message: `App file ${appFile} does not export a valid object.`,
|
||||
detail: 'Ensure the app file exports a Fastify instance as default.',
|
||||
remediation: 'Export your Fastify instance as default: export default app;',
|
||||
detail: 'Ensure the app file exports a Fastify instance or a factory function.',
|
||||
remediation: 'Export your Fastify instance: export default app; or export const createApp = () => app; or module.exports = app;',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
@@ -147,18 +114,32 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
|
||||
}
|
||||
|
||||
// Check for routes
|
||||
let routeCount = 0;
|
||||
let routeCount = 0
|
||||
let discoverySource: string | undefined
|
||||
|
||||
// Fastify 5+ routes access
|
||||
if (app.routes && typeof app.routes === 'function') {
|
||||
const routes = app.routes();
|
||||
routeCount = Array.isArray(routes) ? routes.length : 0;
|
||||
const routes = app.routes()
|
||||
routeCount = Array.isArray(routes) ? routes.length : 0
|
||||
}
|
||||
|
||||
// Fallback: check if we can get routes via inject or other methods
|
||||
if (routeCount === 0 && app.hasRoute) {
|
||||
// We can't enumerate, but we can at least verify the app is functional
|
||||
routeCount = -1; // Unknown but app seems functional
|
||||
routeCount = -1
|
||||
}
|
||||
|
||||
// Use discoverRouteDetails for metadata-aware reporting
|
||||
try {
|
||||
const { discoverRouteDetails } = await import('../../../../domain/discovery.js')
|
||||
const discovery = discoverRouteDetails(app as Parameters<typeof discoverRouteDetails>[0])
|
||||
if (discovery.source === 'print-routes') {
|
||||
discoverySource = 'print-routes'
|
||||
if (routeCount === 0 && discovery.routes.length > 0) {
|
||||
routeCount = discovery.routes.length
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// discovery diagnostics are optional
|
||||
}
|
||||
|
||||
if (routeCount === 0) {
|
||||
@@ -170,7 +151,7 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
|
||||
'APOPHIS discovers routes via the onRoute hook.',
|
||||
remediation: 'Register routes before exporting the app, or ensure the APOPHIS plugin is registered.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (routeCount < 0) {
|
||||
@@ -180,7 +161,18 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
|
||||
message: `App loaded from ${appFile}. Route enumeration not available (app is functional).`,
|
||||
detail: 'Route count could not be determined, but the app appears to be a valid Fastify instance.',
|
||||
mode: 'all',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (discoverySource === 'print-routes') {
|
||||
return {
|
||||
name: 'route-discovery',
|
||||
status: 'warn',
|
||||
message: `Discovered ${routeCount} route(s) from ${appFile} but without schema metadata.`,
|
||||
detail: 'Routes were discovered through printRoutes() fallback. Schema annotations (x-ensures, x-requires, x-outbound, x-timeout, x-variants) are unavailable. APOPHIS can detect paths but not behavioral contracts.',
|
||||
remediation: 'Register APOPHIS or install route discovery before defining routes, or use createFastify() from @apophis/fastify.',
|
||||
mode: 'all',
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -233,8 +225,8 @@ export async function checkSwaggerRegistration(options: RouteCheckOptions): Prom
|
||||
}
|
||||
|
||||
try {
|
||||
const appPath = resolve(options.cwd, appFile);
|
||||
const content = (await import('node:fs')).readFileSync(appPath, 'utf-8');
|
||||
const { readFileSync } = await import('node:fs');
|
||||
const content = readFileSync(appFile, 'utf-8');
|
||||
|
||||
if (content.includes('@fastify/swagger') || content.includes('fastify-swagger')) {
|
||||
return {
|
||||
|
||||
@@ -226,5 +226,85 @@ export function runSafetyChecks(options: SafetyCheckOptions): SafetyCheckResult[
|
||||
results.push(checkEnvironmentPolicies(options));
|
||||
results.push(checkProductionSafety(options));
|
||||
|
||||
// Observe readiness: only when filtering for observe (or no filter)
|
||||
if (!modeFilter || modeFilter === 'observe') {
|
||||
results.push(checkObserveReadiness(options));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Observe readiness
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Check that observe mode is properly configured.
|
||||
* Runs when mode filter is 'observe' or unset.
|
||||
*/
|
||||
export function checkObserveReadiness(options: SafetyCheckOptions): SafetyCheckResult {
|
||||
const { config } = options
|
||||
|
||||
const profileName = config.profile
|
||||
const profile = profileName ? config.profiles?.[profileName] : undefined
|
||||
const presetName = profile?.preset || config.preset
|
||||
const preset = presetName ? config.presets?.[presetName] : undefined
|
||||
|
||||
const hasObserve =
|
||||
preset?.observe === true ||
|
||||
profile?.observe === true
|
||||
|
||||
if (!hasObserve) {
|
||||
return {
|
||||
name: 'observe-readiness',
|
||||
status: 'pass',
|
||||
message: 'Observe not enabled in this preset/profile',
|
||||
mode: 'observe',
|
||||
}
|
||||
}
|
||||
|
||||
const warnings: string[] = []
|
||||
|
||||
// Check sampling rate
|
||||
const sampling = profile?.sampling ?? preset?.sampling
|
||||
if (sampling !== undefined && (sampling < 0 || sampling > 1)) {
|
||||
warnings.push(`Sampling rate ${sampling} is outside valid range [0.0, 1.0]`)
|
||||
}
|
||||
|
||||
// Check sinks
|
||||
const sinks = profile?.sinks ?? preset?.sinks
|
||||
if (!sinks || Object.keys(sinks).length === 0) {
|
||||
warnings.push('No sinks configured — observe will run but events will not be collected')
|
||||
}
|
||||
|
||||
// Check blocking
|
||||
const blocking = profile?.blocking ?? preset?.blocking
|
||||
if (blocking === true) {
|
||||
const env = options.env || detectEnvironment()
|
||||
const isProd = env === 'production' || env === 'prod'
|
||||
if (isProd) {
|
||||
const envPolicy = config.environments?.[env]
|
||||
if (!envPolicy?.allowBlocking) {
|
||||
warnings.push('Blocking observe enabled but allowBlocking is not set in production environment policy')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (warnings.length > 0) {
|
||||
return {
|
||||
name: 'observe-readiness',
|
||||
status: 'warn',
|
||||
message: `Observe readiness: ${warnings.length} concern(s)`,
|
||||
detail: warnings.join('\n'),
|
||||
remediation: 'Review observe configuration in presets and profiles',
|
||||
mode: 'observe',
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'observe-readiness',
|
||||
status: 'pass',
|
||||
message: 'Observe configuration looks ready',
|
||||
mode: 'observe',
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ import { runConfigChecks } from './checks/config.js';
|
||||
import { runRouteChecks } from './checks/routes.js';
|
||||
import { runSafetyChecks } from './checks/safety.js';
|
||||
import { runDocsChecks } from './checks/docs.js';
|
||||
import { runPluginContractChecks } from './checks/plugin-contracts.js';
|
||||
import { runExtensionChecks } from './checks/extensions.js';
|
||||
|
||||
import { renderJson } from '../../renderers/json.js';
|
||||
|
||||
@@ -68,12 +70,6 @@ export interface DoctorResult {
|
||||
// Check filtering
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function shouldRunCheck(checkMode: string | undefined, modeFilter: DoctorMode): boolean {
|
||||
if (!modeFilter) return true;
|
||||
if (!checkMode || checkMode === 'all') return true;
|
||||
return checkMode === modeFilter;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Monorepo detection
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -202,16 +198,30 @@ async function runPackageChecks(
|
||||
checks.push({ ...result, package: packageName });
|
||||
}
|
||||
|
||||
// 6. Determinism trust signal
|
||||
const testSeed = Math.floor(Math.random() * 0x7fffffff);
|
||||
checks.push({
|
||||
name: 'determinism',
|
||||
status: 'pass',
|
||||
message: `Environment supports deterministic replay (test seed: ${testSeed})`,
|
||||
detail: `Run with --seed ${testSeed} to reproduce the exact same test sequence`,
|
||||
mode: 'all',
|
||||
package: packageName,
|
||||
});
|
||||
// 6. Plugin contract and extension checks (all modes)
|
||||
try {
|
||||
const { loadApp } = await import('../../core/app-loader.js')
|
||||
const loaded = await loadApp(cwd)
|
||||
const fastify = loaded.fastify
|
||||
if (fastify) {
|
||||
const pcr = (fastify as any)?._apophisPluginContractRegistry
|
||||
if (pcr) {
|
||||
const pcrResults = runPluginContractChecks({ pluginContractRegistry: pcr })
|
||||
for (const result of pcrResults) {
|
||||
checks.push({ ...result, package: packageName })
|
||||
}
|
||||
}
|
||||
const extReg = (fastify as any)?._apophisExtensionRegistry
|
||||
if (extReg) {
|
||||
const extResults = await runExtensionChecks({ extensionRegistry: extReg, env: detectEnvironment() })
|
||||
for (const result of extResults) {
|
||||
checks.push({ ...result, package: packageName })
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// App can't be loaded — skip plugin/extension checks
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import { resolve } from 'node:path';
|
||||
import type { CliContext } from '../../core/types.js';
|
||||
import { USAGE_ERROR, SUCCESS } from '../../core/exit-codes.js';
|
||||
import { getScaffoldForPreset, getPresetNames, type ScaffoldResult } from './scaffolds/index.js';
|
||||
import { APP_CANDIDATES } from '../../core/app-loader.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Types
|
||||
@@ -63,22 +64,7 @@ function renderInstallCommand(
|
||||
* - Common server file names (server.js, app.js, index.js, etc.)
|
||||
*/
|
||||
export async function detectFastifyEntrypoint(cwd: string): Promise<string | null> {
|
||||
const candidates = [
|
||||
'app.js',
|
||||
'app.ts',
|
||||
'server.js',
|
||||
'server.ts',
|
||||
'index.js',
|
||||
'index.ts',
|
||||
'src/app.js',
|
||||
'src/app.ts',
|
||||
'src/server.js',
|
||||
'src/server.ts',
|
||||
'src/index.js',
|
||||
'src/index.ts',
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
for (const candidate of APP_CANDIDATES) {
|
||||
const fullPath = resolve(cwd, candidate);
|
||||
if (!existsSync(fullPath)) continue;
|
||||
|
||||
@@ -217,7 +203,7 @@ function generateConfigContent(config: ScaffoldResult['config'], isTypeScript: b
|
||||
lines.push('');
|
||||
|
||||
if (isTypeScript) {
|
||||
lines.push('import type { ApophisConfig } from "apophis-fastify/cli";');
|
||||
lines.push('import type { ApophisConfig } from "@apophis/fastify";');
|
||||
lines.push('');
|
||||
lines.push('const config: ApophisConfig = ' + stringifyConfig(config) + ';');
|
||||
lines.push('');
|
||||
@@ -636,7 +622,7 @@ function parseInitOptions(args: string[], ctx: CliContext): InitOptions {
|
||||
export async function handleInit(args: string[], ctx: CliContext): Promise<number> {
|
||||
const result = await initHandler(args, ctx);
|
||||
|
||||
if (result.message) {
|
||||
if (result.message && !ctx.options.quiet) {
|
||||
console.log(result.message);
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ export interface ScaffoldResult {
|
||||
export function safeCiScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'safe-ci',
|
||||
depth: 'quick',
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
@@ -46,6 +45,13 @@ export function safeCiScaffold(): ScaffoldResult {
|
||||
profiles: { quick: profile },
|
||||
presets: { 'safe-ci': preset },
|
||||
environments: { local: envLocal },
|
||||
// Uncomment to enforce cross-cutting behavioral rules across all matching routes:
|
||||
// pluginContracts: {
|
||||
// 'auth-presence': {
|
||||
// appliesTo: '/api/**',
|
||||
// hooks: { onRequest: { requires: ['request_headers(this).authorization != null'] } },
|
||||
// },
|
||||
// },
|
||||
};
|
||||
|
||||
const readmeContent = `
|
||||
@@ -95,11 +101,13 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
|
||||
export function platformObserveScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'platform-observe',
|
||||
depth: 'standard',
|
||||
timeout: 10000,
|
||||
parallel: true,
|
||||
chaos: false,
|
||||
observe: true,
|
||||
sampling: 0.1,
|
||||
blocking: false,
|
||||
sinks: { logs: true, metrics: true },
|
||||
};
|
||||
|
||||
const profile: ProfileDefinition = {
|
||||
@@ -117,6 +125,7 @@ export function platformObserveScaffold(): ScaffoldResult {
|
||||
allowChaos: false,
|
||||
allowBlocking: false,
|
||||
requireSink: true,
|
||||
sinks: { logs: true, metrics: true },
|
||||
};
|
||||
|
||||
const envProduction: EnvironmentPolicy = {
|
||||
@@ -138,6 +147,12 @@ export function platformObserveScaffold(): ScaffoldResult {
|
||||
staging: envStaging,
|
||||
production: envProduction,
|
||||
},
|
||||
// pluginContracts: {
|
||||
// 'request-id': {
|
||||
// appliesTo: '/api/**',
|
||||
// hooks: { onSend: { ensures: ['response_headers(this).x-request-id != null'] } },
|
||||
// },
|
||||
// },
|
||||
};
|
||||
|
||||
const readmeContent = `
|
||||
@@ -180,7 +195,6 @@ This project was scaffolded with \`apophis init --preset platform-observe\`.
|
||||
export function llmSafeScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'llm-safe',
|
||||
depth: 'quick',
|
||||
timeout: 3000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
@@ -210,6 +224,12 @@ export function llmSafeScaffold(): ScaffoldResult {
|
||||
profiles: { 'llm-check': profile },
|
||||
presets: { 'llm-safe': preset },
|
||||
environments: { local: envLocal },
|
||||
// pluginContracts: {
|
||||
// 'auth-presence': {
|
||||
// appliesTo: '/api/**',
|
||||
// hooks: { onRequest: { requires: ['request_headers(this).authorization != null'] } },
|
||||
// },
|
||||
// },
|
||||
};
|
||||
|
||||
const readmeContent = `
|
||||
@@ -258,11 +278,13 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
|
||||
export function protocolLabScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'protocol-lab',
|
||||
depth: 'deep',
|
||||
timeout: 15000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
observe: false,
|
||||
// chaosStrategy: 'sample', // 'one' | 'all' | 'sample' | 'routes'
|
||||
// chaosSampleSize: 3, // routes to target when strategy is 'sample'
|
||||
// chaosSampleRoutes: [], // explicit route list when strategy is 'routes'
|
||||
};
|
||||
|
||||
const profile: ProfileDefinition = {
|
||||
@@ -302,6 +324,19 @@ export function protocolLabScaffold(): ScaffoldResult {
|
||||
local: envLocal,
|
||||
test: envTest,
|
||||
},
|
||||
// pluginContracts: {
|
||||
// 'rate-limit': {
|
||||
// appliesTo: 'POST /api/**',
|
||||
// hooks: { onResponse: { ensures: ['status != 429'] } },
|
||||
// },
|
||||
// },
|
||||
// scenarios: [{
|
||||
// name: 'create-and-read',
|
||||
// steps: [
|
||||
// { request: { method: 'POST', url: '/users', body: { name: 'test' } }, expect: ['status:201'], capture: { userId: 'response_body(this).id' } },
|
||||
// { request: { method: 'GET', url: '/users/{userId}' }, expect: ['status:200', 'response_body(this).name == "test"'] },
|
||||
// ],
|
||||
// }],
|
||||
};
|
||||
|
||||
const readmeContent = `
|
||||
@@ -346,6 +381,7 @@ Qualify can produce large output. In CI, use machine-readable formats and filter
|
||||
## Next Steps
|
||||
|
||||
- Define scenario sequences in your config.
|
||||
- Example scenario is commented out in apophis.config.js — uncomment and adapt.
|
||||
- Add route allowlists for chaos if needed.
|
||||
- Run \`apophis replay --artifact <path>\` to debug failures.
|
||||
`;
|
||||
|
||||
@@ -25,15 +25,8 @@
|
||||
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import type { CliContext } from '../../core/context.js';
|
||||
import { loadConfig, discoverConfig } from '../../core/config-loader.js';
|
||||
import { discoverConfig } from '../../core/config-loader.js';
|
||||
import { SUCCESS, USAGE_ERROR, BEHAVIORAL_FAILURE } from '../../core/exit-codes.js';
|
||||
import type { CommandResult } from '../../core/types.js';
|
||||
import {
|
||||
rewriteConfigFile,
|
||||
detectLegacyConfigFields,
|
||||
detectLegacyFieldsNoEquivalent,
|
||||
detectMixedLegacyModernFields,
|
||||
} from './rewriters/config-rewriter.js';
|
||||
import {
|
||||
rewriteRouteAnnotations,
|
||||
detectLegacyRouteAnnotations,
|
||||
@@ -136,11 +129,8 @@ export async function detectAllLegacyPatterns(
|
||||
): Promise<MigrationItem[]> {
|
||||
const items: MigrationItem[] = [];
|
||||
|
||||
// Detect config fields
|
||||
if (configFile && existsSync(configFile)) {
|
||||
const configContent = readFileSync(configFile, 'utf-8');
|
||||
items.push(...detectLegacyConfigFields(configContent, configFile));
|
||||
items.push(...detectLegacyFieldsNoEquivalent(configContent, configFile));
|
||||
items.push(...detectLegacyRouteAnnotations(configContent, configFile));
|
||||
items.push(...detectAmbiguousRoutePatterns(configContent, configFile));
|
||||
items.push(...detectLegacyCodePatterns(configContent, configFile));
|
||||
@@ -276,25 +266,8 @@ export async function migrateCommand(
|
||||
const completed: MigrationItem[] = [];
|
||||
const remaining: MigrationItem[] = [];
|
||||
|
||||
// Rewrite config file
|
||||
// Route annotations in config file
|
||||
if (configFile && existsSync(configFile)) {
|
||||
const configItems = unambiguousItems.filter(
|
||||
(item) => item.file === configFile && item.type === 'config-field',
|
||||
);
|
||||
|
||||
if (configItems.length > 0) {
|
||||
const result = rewriteConfigFile(configFile, configItems);
|
||||
if (result.modified) {
|
||||
writeFileSync(configFile, result.content, 'utf-8');
|
||||
filesModified.push(configFile);
|
||||
completed.push(...result.itemsRewritten);
|
||||
remaining.push(...result.itemsRemaining);
|
||||
} else {
|
||||
remaining.push(...configItems);
|
||||
}
|
||||
}
|
||||
|
||||
// Route annotations in config file
|
||||
const routeItems = unambiguousItems.filter(
|
||||
(item) => item.file === configFile && item.type === 'route-annotation',
|
||||
);
|
||||
|
||||
@@ -1,268 +0,0 @@
|
||||
/**
|
||||
* Config rewriter for APOPHIS migrate command.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Rewrite config files, replacing legacy fields with modern equivalents
|
||||
* - Preserve comments and formatting where feasible
|
||||
* - Handle nested object rewrites
|
||||
* - Report what was changed and what remains
|
||||
* - Detect mixed legacy/modern configs and report clearly
|
||||
* - Emit human guidance for legacy fields with no direct equivalent
|
||||
*
|
||||
* Architecture:
|
||||
* - Dependency injection: all dependencies passed explicitly
|
||||
* - No optional imports
|
||||
* - Inline comments for documentation
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import type { MigrationItem } from '../index.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function escapeRegex(str: string): string {
|
||||
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ConfigRewriteResult {
|
||||
content: string;
|
||||
modified: boolean;
|
||||
itemsRewritten: MigrationItem[];
|
||||
itemsRemaining: MigrationItem[];
|
||||
}
|
||||
|
||||
export interface MixedFieldReport {
|
||||
legacy: string;
|
||||
modern: string;
|
||||
line: number;
|
||||
guidance: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Legacy field mappings
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Mapping of deprecated config fields to their modern equivalents.
|
||||
*/
|
||||
export const LEGACY_CONFIG_MAPPINGS: Record<string, string> = {
|
||||
// Top-level fields
|
||||
testMode: 'mode',
|
||||
|
||||
// Profile container
|
||||
testProfiles: 'profiles',
|
||||
|
||||
// Profile fields
|
||||
usesPreset: 'preset',
|
||||
routeFilter: 'routes',
|
||||
|
||||
// Preset container
|
||||
testPresets: 'presets',
|
||||
|
||||
// Preset fields
|
||||
testDepth: 'depth',
|
||||
maxDuration: 'timeout',
|
||||
|
||||
// Environment container
|
||||
envPolicies: 'environments',
|
||||
|
||||
// Environment fields
|
||||
canVerify: 'allowVerify',
|
||||
};
|
||||
|
||||
/**
|
||||
* Legacy fields with no direct equivalent — emit human guidance instead of auto-rewrite.
|
||||
*/
|
||||
export const LEGACY_FIELDS_NO_EQUIVALENT: Record<string, { guidance: string; severity: 'warning' | 'error' }> = {
|
||||
legacyField: {
|
||||
guidance: 'This field has no modern equivalent. Remove it and review your config manually.',
|
||||
severity: 'warning',
|
||||
},
|
||||
oldApiVersion: {
|
||||
guidance: 'API versioning is now handled via profiles. Remove this field and set version in each profile.',
|
||||
severity: 'warning',
|
||||
},
|
||||
deprecatedPlugin: {
|
||||
guidance: 'This plugin is no longer supported. Remove the field and migrate to the new plugin system.',
|
||||
severity: 'error',
|
||||
},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core rewriting logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Rewrite a config file, replacing legacy field names with modern equivalents.
|
||||
*
|
||||
* Strategy:
|
||||
* 1. Read the raw file content
|
||||
* 2. For each legacy field mapping, replace occurrences as property keys
|
||||
* 3. Preserve formatting by only replacing the key name, not surrounding whitespace
|
||||
* 4. Track which items were rewritten and which remain
|
||||
*/
|
||||
export function rewriteConfigFile(
|
||||
filePath: string,
|
||||
items: MigrationItem[],
|
||||
): ConfigRewriteResult {
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
let modifiedContent = content;
|
||||
let modified = false;
|
||||
|
||||
const itemsRewritten: MigrationItem[] = [];
|
||||
const itemsRemaining: MigrationItem[] = [];
|
||||
|
||||
for (const item of items) {
|
||||
if (item.type !== 'config-field') {
|
||||
itemsRemaining.push(item);
|
||||
continue;
|
||||
}
|
||||
|
||||
// The legacy field name (might be a nested path like "testProfiles.quick")
|
||||
const legacyKey = item.legacy.split('.').pop() || item.legacy;
|
||||
const replacement = item.replacement;
|
||||
|
||||
// Build a regex that matches the field as a property key
|
||||
// This handles: key:, "key":, 'key':, key :, etc.
|
||||
const regex = new RegExp(
|
||||
`([\\s{,\\[])(['"]?)(${escapeRegex(legacyKey)})\\2\\s*:(?!\\/)`,
|
||||
'g',
|
||||
);
|
||||
|
||||
const newContent = modifiedContent.replace(regex, (match, prefix, quote, _key) => {
|
||||
return `${prefix}${quote}${replacement}${quote}:`;
|
||||
});
|
||||
|
||||
if (newContent !== modifiedContent) {
|
||||
modifiedContent = newContent;
|
||||
modified = true;
|
||||
itemsRewritten.push(item);
|
||||
} else {
|
||||
itemsRemaining.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
content: modifiedContent,
|
||||
modified,
|
||||
itemsRewritten,
|
||||
itemsRemaining,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the rewritten config to disk.
|
||||
*/
|
||||
export function writeRewrittenConfig(filePath: string, content: string): void {
|
||||
writeFileSync(filePath, content, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect legacy config fields in raw text content.
|
||||
* Returns migration items for each occurrence.
|
||||
*/
|
||||
export function detectLegacyConfigFields(
|
||||
content: string,
|
||||
filePath: string,
|
||||
): MigrationItem[] {
|
||||
const items: MigrationItem[] = [];
|
||||
const lines = content.split('\n');
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line === undefined) continue;
|
||||
|
||||
for (const [legacy, replacement] of Object.entries(LEGACY_CONFIG_MAPPINGS)) {
|
||||
// Match the field as a property key, avoiding matches inside strings/comments
|
||||
const regex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
|
||||
if (regex.test(line)) {
|
||||
items.push({
|
||||
type: 'config-field',
|
||||
file: filePath,
|
||||
line: i + 1,
|
||||
legacy,
|
||||
replacement,
|
||||
guidance: `Replace '${legacy}' with '${replacement}'`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect legacy fields that have no direct modern equivalent.
|
||||
* These emit human guidance instead of being auto-rewritten.
|
||||
*/
|
||||
export function detectLegacyFieldsNoEquivalent(
|
||||
content: string,
|
||||
filePath: string,
|
||||
): MigrationItem[] {
|
||||
const items: MigrationItem[] = [];
|
||||
const lines = content.split('\n');
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line === undefined) continue;
|
||||
|
||||
for (const [legacy, info] of Object.entries(LEGACY_FIELDS_NO_EQUIVALENT)) {
|
||||
const regex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
|
||||
if (regex.test(line)) {
|
||||
items.push({
|
||||
type: 'config-field',
|
||||
file: filePath,
|
||||
line: i + 1,
|
||||
legacy,
|
||||
replacement: '(removed — see guidance)',
|
||||
guidance: info.guidance,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect mixed legacy and modern config fields.
|
||||
* When both legacy and modern versions of the same field exist, report each clearly.
|
||||
*/
|
||||
export function detectMixedLegacyModernFields(
|
||||
content: string,
|
||||
filePath: string,
|
||||
): MixedFieldReport[] {
|
||||
const reports: MixedFieldReport[] = [];
|
||||
const lines = content.split('\n');
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line === undefined) continue;
|
||||
|
||||
for (const [legacy, modern] of Object.entries(LEGACY_CONFIG_MAPPINGS)) {
|
||||
// Check if this line contains the legacy field
|
||||
const legacyRegex = new RegExp(`\\b${escapeRegex(legacy)}\\s*:`);
|
||||
if (legacyRegex.test(line)) {
|
||||
// Check if the modern equivalent also exists somewhere in the file
|
||||
const modernRegex = new RegExp(`\\b${escapeRegex(modern)}\\s*:`);
|
||||
if (modernRegex.test(content)) {
|
||||
reports.push({
|
||||
legacy,
|
||||
modern,
|
||||
line: i + 1,
|
||||
guidance: `Both '${legacy}' (legacy) and '${modern}' (modern) found. Remove '${legacy}' to avoid conflicts.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return reports;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* Route rewriter for APOPHIS migrate command.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Rewrite route schema annotations (e.g., x-validate-runtime → runtime)
|
||||
* - Rewrite route schema annotations
|
||||
* - Preserve schema structure and formatting
|
||||
* - Handle annotations in Fastify route definitions
|
||||
* - Detect ambiguous annotations and require manual choice
|
||||
@@ -40,10 +40,12 @@ export interface AmbiguousRoutePattern {
|
||||
|
||||
/**
|
||||
* Mapping of deprecated route schema annotations to their modern equivalents.
|
||||
* x-validate-runtime is intentionally NOT here — it is the current, active annotation
|
||||
* used by contract.ts and hook-validator.ts for per-route runtime validation opt-out.
|
||||
* Rewriting it to 'runtime' would break it, since 'runtime' is a top-level plugin
|
||||
* config option, not a valid route schema annotation.
|
||||
*/
|
||||
export const LEGACY_ROUTE_ANNOTATIONS: Record<string, string> = {
|
||||
'x-validate-runtime': 'runtime',
|
||||
};
|
||||
export const LEGACY_ROUTE_ANNOTATIONS: Record<string, string> = {};
|
||||
|
||||
/**
|
||||
* Ambiguous route patterns that require manual choice.
|
||||
|
||||
@@ -1,328 +0,0 @@
|
||||
/**
|
||||
* S5: Observe thread - Observe command handler
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Load config and resolve profile
|
||||
* - Validate observe configuration
|
||||
* - Check reporting sink setup (logs, metrics, traces)
|
||||
* - Validate non-blocking semantics
|
||||
* - Environment safety checks (block blocking behavior in prod by default)
|
||||
* - Support --check-config (validate only, don't activate)
|
||||
* - Explain what would be checked and why it is safe
|
||||
* - Clear output about safety boundaries
|
||||
* - Exit 0 on valid config, 2 on safety violation
|
||||
*/
|
||||
|
||||
import type { CliContext } from '../../core/context.js';
|
||||
import { loadConfig } from '../../core/config-loader.js';
|
||||
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js';
|
||||
import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js';
|
||||
import { validateObserveConfig } from './validator.js';
|
||||
import { renderDoctorChecks } from '../../renderers/human.js';
|
||||
import { renderJson } from '../../renderers/json.js';
|
||||
import type { OutputContext } from '../../renderers/shared.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ObserveOptions {
|
||||
profile?: string;
|
||||
checkConfig?: boolean;
|
||||
config?: string;
|
||||
cwd?: string;
|
||||
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
|
||||
quiet?: boolean;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
export interface ObserveResult {
|
||||
exitCode: number;
|
||||
message?: string;
|
||||
checks?: Array<{
|
||||
name: string;
|
||||
status: 'pass' | 'fail' | 'warn';
|
||||
message: string;
|
||||
detail?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Command handler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Main observe command handler.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Load and resolve config
|
||||
* 2. Run policy engine checks
|
||||
* 3. Validate observe-specific configuration
|
||||
* 4. If --check-config, stop after validation
|
||||
* 5. Otherwise, report what would be activated and why it is safe
|
||||
* 6. Return appropriate exit code
|
||||
*/
|
||||
export async function observeCommand(
|
||||
options: ObserveOptions,
|
||||
ctx: CliContext,
|
||||
): Promise<ObserveResult> {
|
||||
const { profile, checkConfig, config: configPath, cwd } = options;
|
||||
const workingDir = cwd || ctx.cwd;
|
||||
|
||||
// Detect environment from context
|
||||
const env = detectEnvironment();
|
||||
|
||||
try {
|
||||
// 1. Load config
|
||||
const loadResult = await loadConfig({
|
||||
cwd: workingDir,
|
||||
configPath,
|
||||
profileName: profile,
|
||||
env,
|
||||
});
|
||||
|
||||
if (!loadResult.configPath) {
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: 'No config found. Run "apophis init" to create one.',
|
||||
};
|
||||
}
|
||||
|
||||
const config = loadResult.config;
|
||||
|
||||
// 2. Run policy engine checks
|
||||
const policyEngine = new PolicyEngine({
|
||||
config,
|
||||
env,
|
||||
mode: 'observe',
|
||||
profileName: profile || undefined,
|
||||
presetName: loadResult.presetName || undefined,
|
||||
});
|
||||
|
||||
const policyResult = policyEngine.check();
|
||||
|
||||
if (!policyResult.allowed) {
|
||||
const message = [
|
||||
'Policy check failed:',
|
||||
...policyResult.errors.map(e => ` ✗ ${e}`),
|
||||
].join('\n');
|
||||
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message,
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Validate observe-specific configuration
|
||||
const validationResult = validateObserveConfig(config, profile || undefined, env);
|
||||
|
||||
if (!validationResult.valid) {
|
||||
const message = formatValidationOutput(validationResult, { checkConfig, env, profile });
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message,
|
||||
checks: validationResult.checks,
|
||||
};
|
||||
}
|
||||
|
||||
// 4. If --check-config, stop after validation with success
|
||||
if (checkConfig) {
|
||||
const message = formatValidationOutput(validationResult, {
|
||||
checkConfig: true,
|
||||
env,
|
||||
profile,
|
||||
});
|
||||
return {
|
||||
exitCode: SUCCESS,
|
||||
message,
|
||||
checks: validationResult.checks,
|
||||
};
|
||||
}
|
||||
|
||||
// 5. Report what would be activated and why it is safe
|
||||
const activationMessage = formatActivationOutput(validationResult, {
|
||||
env,
|
||||
profile,
|
||||
configPath: loadResult.configPath,
|
||||
});
|
||||
|
||||
return {
|
||||
exitCode: SUCCESS,
|
||||
message: activationMessage,
|
||||
checks: validationResult.checks,
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: `Failed to run observe command: ${message}`,
|
||||
checks: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Output formatting
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface FormatOptions {
|
||||
checkConfig?: boolean;
|
||||
env: string;
|
||||
profile?: string;
|
||||
configPath?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format validation results for human-readable output.
|
||||
*/
|
||||
function formatValidationOutput(
|
||||
result: import('./validator.js').ObserveValidationResult,
|
||||
options: FormatOptions,
|
||||
): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
const mode = options.checkConfig ? 'Config validation' : 'Observe validation';
|
||||
lines.push(`${mode} for environment "${options.env}"`);
|
||||
if (options.profile) {
|
||||
lines.push(`Profile: ${options.profile}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Print each check
|
||||
for (const check of result.checks) {
|
||||
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
|
||||
lines.push(` ${icon} ${check.name}: ${check.message}`);
|
||||
if (check.detail) {
|
||||
lines.push(` ${check.detail}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
|
||||
// Summary
|
||||
if (result.errors.length > 0) {
|
||||
lines.push(`Failed with ${result.errors.length} error(s).`);
|
||||
lines.push('');
|
||||
lines.push('Safety boundaries:');
|
||||
lines.push(' - Observe mode is non-blocking by default');
|
||||
lines.push(' - Blocking behavior is prohibited in production');
|
||||
lines.push(' - Qualify-only features (chaos, stateful, etc.) are not allowed');
|
||||
lines.push(' - Sampling rate must be between 0.0 and 1.0');
|
||||
lines.push(' - Sinks must be configured when required by environment policy');
|
||||
} else if (result.warnings.length > 0) {
|
||||
lines.push(`Passed with ${result.warnings.length} warning(s).`);
|
||||
} else {
|
||||
lines.push('All checks passed.');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Format activation output explaining what would be checked and why it is safe.
|
||||
*/
|
||||
function formatActivationOutput(
|
||||
result: import('./validator.js').ObserveValidationResult,
|
||||
options: FormatOptions,
|
||||
): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
lines.push(`Observe mode ready for environment "${options.env}"`);
|
||||
if (options.profile) {
|
||||
lines.push(`Profile: ${options.profile}`);
|
||||
}
|
||||
if (options.configPath) {
|
||||
lines.push(`Config: ${options.configPath}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Print checks
|
||||
for (const check of result.checks) {
|
||||
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
|
||||
lines.push(` ${icon} ${check.name}: ${check.message}`);
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push('What would be checked:');
|
||||
lines.push(' - Request/response contracts are evaluated asynchronously');
|
||||
lines.push(' - Violations are logged to configured sinks without blocking');
|
||||
lines.push(' - Sampling controls the fraction of requests observed');
|
||||
lines.push(' - Metrics and traces provide runtime visibility into contract health');
|
||||
lines.push('');
|
||||
lines.push('Why this is safe:');
|
||||
lines.push(' - Non-blocking semantics guarantee observation does not affect latency');
|
||||
lines.push(' - No chaos injection or stateful sequences are activated in observe mode');
|
||||
lines.push(' - Production environments require explicit non-blocking configuration');
|
||||
lines.push(' - All qualify-only features are blocked by validation');
|
||||
|
||||
if (result.warnings.length > 0) {
|
||||
lines.push('');
|
||||
lines.push('Warnings:');
|
||||
for (const warning of result.warnings) {
|
||||
lines.push(` ⚠ ${warning}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push('To activate observation, run without --check-config.');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI adapter
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Adapter that bridges the CLI framework (cac) to the observe command handler.
|
||||
* This function signature matches what the CLI core expects.
|
||||
*/
|
||||
export async function handleObserve(
|
||||
_args: string[],
|
||||
ctx: CliContext,
|
||||
): Promise<number> {
|
||||
const options: ObserveOptions = {
|
||||
profile: ctx.options.profile || undefined,
|
||||
checkConfig: false,
|
||||
config: ctx.options.config || undefined,
|
||||
cwd: ctx.cwd,
|
||||
format: ctx.options.format as ObserveOptions['format'],
|
||||
quiet: ctx.options.quiet,
|
||||
verbose: ctx.options.verbose,
|
||||
};
|
||||
|
||||
// Parse command-specific flags from process.argv
|
||||
// cac passes these as parsed options, but we need to extract --check-config
|
||||
// Since cac doesn't expose parsed command-specific flags in the options object,
|
||||
// we scan process.argv directly for observe-specific flags
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.includes('--check-config')) {
|
||||
options.checkConfig = true;
|
||||
}
|
||||
|
||||
const result = await observeCommand(options, ctx);
|
||||
|
||||
// Output result based on format
|
||||
if (!ctx.options.quiet && result.message) {
|
||||
const format = options.format || ctx.options.format || 'human';
|
||||
if (format === 'json') {
|
||||
console.log(renderJson({
|
||||
exitCode: result.exitCode,
|
||||
checks: result.checks,
|
||||
message: result.message,
|
||||
}));
|
||||
} else if (format === 'ndjson') {
|
||||
process.stdout.write(JSON.stringify({
|
||||
type: 'run.completed',
|
||||
command: 'observe',
|
||||
exitCode: result.exitCode,
|
||||
checks: result.checks,
|
||||
message: result.message,
|
||||
}) + '\n');
|
||||
} else {
|
||||
console.log(result.message);
|
||||
}
|
||||
}
|
||||
|
||||
return result.exitCode;
|
||||
}
|
||||
@@ -1,539 +0,0 @@
|
||||
/**
|
||||
* S5: Observe thread - Observe config validation logic
|
||||
*
|
||||
* Validates observe-specific configuration including:
|
||||
* - Sink configuration checks (logs, metrics, traces)
|
||||
* - Sampling rate validation
|
||||
* - Feature restriction checks (no qualify-only features in observe)
|
||||
* - Non-blocking semantics validation
|
||||
*/
|
||||
|
||||
import type { Config, ProfileDefinition, PresetDefinition } from '../../core/config-loader.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ObserveValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
checks: ObserveCheck[];
|
||||
}
|
||||
|
||||
export interface ObserveCheck {
|
||||
name: string;
|
||||
status: 'pass' | 'fail' | 'warn';
|
||||
message: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface SinkConfig {
|
||||
logs?: boolean;
|
||||
metrics?: boolean;
|
||||
traces?: boolean;
|
||||
endpoint?: string;
|
||||
}
|
||||
|
||||
export interface ObserveProfileConfig {
|
||||
sampling?: number;
|
||||
blocking?: boolean;
|
||||
sinks?: SinkConfig;
|
||||
features?: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Features that are only valid in qualify mode */
|
||||
const QUALIFY_ONLY_FEATURES = new Set([
|
||||
'chaos',
|
||||
'stateful',
|
||||
'scenario',
|
||||
'outbound-mocks',
|
||||
'protocol-flow',
|
||||
]);
|
||||
|
||||
/** Valid sampling rate bounds */
|
||||
const SAMPLING_MIN = 0.0;
|
||||
const SAMPLING_MAX = 1.0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validation functions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Validate observe configuration for a given profile and environment.
|
||||
*/
|
||||
export function validateObserveConfig(
|
||||
config: Config,
|
||||
profileName: string | undefined,
|
||||
env: string,
|
||||
): ObserveValidationResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const checks: ObserveCheck[] = [];
|
||||
|
||||
// Resolve the effective profile config (preset + profile overrides)
|
||||
const profileConfig = resolveObserveProfileConfig(config, profileName);
|
||||
|
||||
// 1. Check profile exists and is observe mode
|
||||
const profileCheck = validateProfileMode(config, profileName);
|
||||
checks.push(profileCheck);
|
||||
if (profileCheck.status === 'fail') {
|
||||
errors.push(profileCheck.message);
|
||||
}
|
||||
|
||||
// 2. Check for qualify-only features (uses resolved profile config)
|
||||
const featureCheck = validateFeatures(profileConfig.features, profileName);
|
||||
checks.push(featureCheck);
|
||||
if (featureCheck.status === 'fail') {
|
||||
errors.push(featureCheck.message);
|
||||
}
|
||||
|
||||
// 3. Validate sampling rate (uses resolved profile config)
|
||||
const samplingCheck = validateSamplingRate(profileConfig.sampling);
|
||||
checks.push(samplingCheck);
|
||||
if (samplingCheck.status === 'fail') {
|
||||
errors.push(samplingCheck.message);
|
||||
}
|
||||
|
||||
// 4. Validate sink configuration (uses resolved profile config)
|
||||
const sinkCheck = validateSinkConfig(profileConfig.sinks, env, config);
|
||||
checks.push(sinkCheck);
|
||||
if (sinkCheck.status === 'fail') {
|
||||
errors.push(sinkCheck.message);
|
||||
} else if (sinkCheck.status === 'warn') {
|
||||
warnings.push(sinkCheck.message);
|
||||
}
|
||||
|
||||
// 5. Validate non-blocking semantics (uses resolved profile config)
|
||||
const blockingCheck = validateBlockingSemantics(profileConfig.blocking, env, config);
|
||||
checks.push(blockingCheck);
|
||||
if (blockingCheck.status === 'fail') {
|
||||
errors.push(blockingCheck.message);
|
||||
}
|
||||
|
||||
// 6. Environment policy check: must explicitly allow observe
|
||||
const envPolicyCheck = validateEnvironmentPolicy(config, env);
|
||||
checks.push(envPolicyCheck);
|
||||
if (envPolicyCheck.status === 'fail') {
|
||||
errors.push(envPolicyCheck.message);
|
||||
}
|
||||
|
||||
// 7. Environment safety check
|
||||
const envCheck = validateEnvironmentSafety(env, profileConfig);
|
||||
checks.push(envCheck);
|
||||
if (envCheck.status === 'warn') {
|
||||
warnings.push(envCheck.message);
|
||||
}
|
||||
|
||||
// 8. Profile must be configured for observe mode
|
||||
const profileObserveCheck = validateProfileObserveMode(config, profileName);
|
||||
checks.push(profileObserveCheck);
|
||||
if (profileObserveCheck.status === 'fail') {
|
||||
errors.push(profileObserveCheck.message);
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the observe-specific configuration from profile and preset.
|
||||
* Preset values are applied first, then profile overrides.
|
||||
*/
|
||||
function resolveObserveProfileConfig(
|
||||
config: Config,
|
||||
profileName: string | undefined,
|
||||
): ObserveProfileConfig {
|
||||
const result: ObserveProfileConfig = {};
|
||||
|
||||
if (!profileName || !config.profiles) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const profile = config.profiles[profileName];
|
||||
if (!profile) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Apply preset first if referenced
|
||||
if (profile.preset && config.presets) {
|
||||
const preset = config.presets[profile.preset];
|
||||
if (preset) {
|
||||
Object.assign(result, presetToObserveConfig(preset));
|
||||
}
|
||||
}
|
||||
|
||||
// Apply profile overrides
|
||||
Object.assign(result, profileToObserveConfig(profile));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert preset definition to observe config.
|
||||
*/
|
||||
function presetToObserveConfig(preset: PresetDefinition): ObserveProfileConfig {
|
||||
return {
|
||||
features: preset.features,
|
||||
sampling: (preset as Record<string, unknown>).sampling as number | undefined,
|
||||
blocking: (preset as Record<string, unknown>).blocking as boolean | undefined,
|
||||
sinks: (preset as Record<string, unknown>).sinks as SinkConfig | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert profile definition to observe config.
|
||||
*/
|
||||
function profileToObserveConfig(profile: ProfileDefinition): ObserveProfileConfig {
|
||||
return {
|
||||
features: profile.features,
|
||||
sampling: (profile as Record<string, unknown>).sampling as number | undefined,
|
||||
blocking: (profile as Record<string, unknown>).blocking as boolean | undefined,
|
||||
sinks: (profile as Record<string, unknown>).sinks as SinkConfig | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that the profile exists.
|
||||
* Note: mode validation is handled by validateProfileObserveMode.
|
||||
*/
|
||||
function validateProfileMode(
|
||||
config: Config,
|
||||
profileName: string | undefined,
|
||||
): ObserveCheck {
|
||||
if (!profileName) {
|
||||
return {
|
||||
name: 'profile-mode',
|
||||
status: 'pass',
|
||||
message: 'No profile specified, using default observe configuration',
|
||||
};
|
||||
}
|
||||
|
||||
if (!config.profiles || !config.profiles[profileName]) {
|
||||
const available = config.profiles ? Object.keys(config.profiles).join(', ') : 'none';
|
||||
return {
|
||||
name: 'profile-mode',
|
||||
status: 'fail',
|
||||
message: `Profile "${profileName}" not found. Available profiles: ${available}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'profile-mode',
|
||||
status: 'pass',
|
||||
message: `Profile "${profileName}" exists`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that the profile is explicitly configured for observe mode.
|
||||
*/
|
||||
function validateProfileObserveMode(
|
||||
config: Config,
|
||||
profileName: string | undefined,
|
||||
): ObserveCheck {
|
||||
if (!profileName) {
|
||||
return {
|
||||
name: 'profile-observe-mode',
|
||||
status: 'pass',
|
||||
message: 'No profile specified, mode will be determined by top-level config',
|
||||
};
|
||||
}
|
||||
|
||||
if (!config.profiles || !config.profiles[profileName]) {
|
||||
return {
|
||||
name: 'profile-observe-mode',
|
||||
status: 'pass',
|
||||
message: `Profile "${profileName}" not found — will be validated by profile-mode check`,
|
||||
};
|
||||
}
|
||||
|
||||
const profile = config.profiles[profileName];
|
||||
const profileMode = profile.mode;
|
||||
|
||||
if (profileMode && profileMode !== 'observe') {
|
||||
return {
|
||||
name: 'profile-observe-mode',
|
||||
status: 'fail',
|
||||
message: `Profile "${profileName}" is configured for "${profileMode}" mode but observe command requires "observe" mode`,
|
||||
detail: 'Change the profile mode to "observe" or use the appropriate command ' +
|
||||
`for "${profileMode}" mode (e.g., apophis ${profileMode}).`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'profile-observe-mode',
|
||||
status: 'pass',
|
||||
message: `Profile "${profileName}" is configured for observe mode`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that no qualify-only features are used in observe mode.
|
||||
*/
|
||||
function validateFeatures(
|
||||
features: string[] | undefined,
|
||||
profileName: string | undefined,
|
||||
): ObserveCheck {
|
||||
if (!features || features.length === 0) {
|
||||
return {
|
||||
name: 'feature-restrictions',
|
||||
status: 'pass',
|
||||
message: 'No features configured',
|
||||
};
|
||||
}
|
||||
|
||||
const invalidFeatures = features.filter(f => QUALIFY_ONLY_FEATURES.has(f));
|
||||
if (invalidFeatures.length > 0) {
|
||||
const profileRef = profileName ? `Profile "${profileName}"` : 'Configuration';
|
||||
return {
|
||||
name: 'feature-restrictions',
|
||||
status: 'fail',
|
||||
message: `${profileRef} references qualify-only features that cannot be used in observe mode: ${invalidFeatures.join(', ')}`,
|
||||
detail: `Remove these features from the profile or preset. Qualify-only features: ${Array.from(QUALIFY_ONLY_FEATURES).join(', ')}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'feature-restrictions',
|
||||
status: 'pass',
|
||||
message: `All features are valid for observe mode: ${features.join(', ')}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate sampling rate is within valid bounds [0.0, 1.0].
|
||||
*/
|
||||
export function validateSamplingRate(sampling: number | undefined): ObserveCheck {
|
||||
if (sampling === undefined || sampling === null) {
|
||||
return {
|
||||
name: 'sampling-rate',
|
||||
status: 'pass',
|
||||
message: 'No sampling rate configured, using default (1.0)',
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof sampling !== 'number' || Number.isNaN(sampling)) {
|
||||
return {
|
||||
name: 'sampling-rate',
|
||||
status: 'fail',
|
||||
message: `Sampling rate must be a number, got ${typeof sampling}`,
|
||||
detail: `Valid range: ${SAMPLING_MIN} to ${SAMPLING_MAX} (inclusive)`,
|
||||
};
|
||||
}
|
||||
|
||||
if (sampling < SAMPLING_MIN || sampling > SAMPLING_MAX) {
|
||||
return {
|
||||
name: 'sampling-rate',
|
||||
status: 'fail',
|
||||
message: `Sampling rate ${sampling} is out of bounds`,
|
||||
detail: `Set sampling to a value between ${SAMPLING_MIN} and ${SAMPLING_MAX} (inclusive). ` +
|
||||
`A rate of 0.0 disables observation, 1.0 observes all requests.`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'sampling-rate',
|
||||
status: 'pass',
|
||||
message: `Sampling rate ${sampling} is valid`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate sink configuration for the environment.
|
||||
*/
|
||||
function validateSinkConfig(
|
||||
sinks: SinkConfig | undefined,
|
||||
env: string,
|
||||
config: Config,
|
||||
): ObserveCheck {
|
||||
// Check if environment requires sinks
|
||||
const envPolicy = config.environments?.[env];
|
||||
const requireSink = envPolicy?.requireSink ?? false;
|
||||
|
||||
if (!sinks || Object.keys(sinks).length === 0) {
|
||||
if (requireSink) {
|
||||
return {
|
||||
name: 'sink-config',
|
||||
status: 'fail',
|
||||
message: `Environment "${env}" requires sink configuration but none is provided`,
|
||||
detail: 'Add sinks to your profile (e.g., sinks: { logs: true }) ' +
|
||||
'or set requireSink: false in the environment policy.',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'sink-config',
|
||||
status: 'warn',
|
||||
message: 'No sinks configured. Observation data will not be persisted.',
|
||||
detail: 'Configure at least one sink (logs, metrics, or traces) ' +
|
||||
'to capture observation data for analysis.',
|
||||
};
|
||||
}
|
||||
|
||||
const activeSinks = [];
|
||||
if (sinks.logs) activeSinks.push('logs');
|
||||
if (sinks.metrics) activeSinks.push('metrics');
|
||||
if (sinks.traces) activeSinks.push('traces');
|
||||
|
||||
if (activeSinks.length === 0) {
|
||||
return {
|
||||
name: 'sink-config',
|
||||
status: 'warn',
|
||||
message: 'Sinks are configured but none are enabled. Observation data will not be persisted.',
|
||||
detail: 'Set at least one of logs, metrics, or traces to true in your sink configuration.',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'sink-config',
|
||||
status: 'pass',
|
||||
message: `Active sinks: ${activeSinks.join(', ')}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate non-blocking semantics for the environment.
|
||||
* Blocking is NEVER allowed in production unless explicitly enabled by a break-glass policy.
|
||||
*/
|
||||
function validateBlockingSemantics(
|
||||
blocking: boolean | undefined,
|
||||
env: string,
|
||||
config: Config,
|
||||
): ObserveCheck {
|
||||
const isProd = env === 'production' || env === 'prod';
|
||||
|
||||
if (blocking === true && isProd) {
|
||||
// Check for break-glass policy override
|
||||
const envPolicy = config.environments?.[env];
|
||||
const allowBlocking = envPolicy?.allowBlocking ?? false;
|
||||
|
||||
if (!allowBlocking) {
|
||||
return {
|
||||
name: 'blocking-semantics',
|
||||
status: 'fail',
|
||||
message: `Blocking behavior is not allowed in production environment "${env}"`,
|
||||
detail: 'Set blocking: false in your profile, use a non-production environment, ' +
|
||||
'or set allowBlocking: true in the environment policy for break-glass scenarios.',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'blocking-semantics',
|
||||
status: 'pass',
|
||||
message: `Blocking behavior is enabled in production "${env}" via break-glass policy`,
|
||||
detail: 'WARNING: blocking observation can severely impact request latency. ' +
|
||||
'This should only be used during active incident response.',
|
||||
};
|
||||
}
|
||||
|
||||
if (blocking === true) {
|
||||
return {
|
||||
name: 'blocking-semantics',
|
||||
status: 'pass',
|
||||
message: `Blocking behavior is enabled in non-production environment "${env}"`,
|
||||
detail: 'Warning: blocking observation can increase request latency. ' +
|
||||
'Only enable in environments where latency impact is acceptable.',
|
||||
};
|
||||
}
|
||||
|
||||
// blocking is false or undefined (default to non-blocking)
|
||||
return {
|
||||
name: 'blocking-semantics',
|
||||
status: 'pass',
|
||||
message: `Non-blocking semantics confirmed for environment "${env}"`,
|
||||
detail: 'Observation will run asynchronously without blocking request handling.',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate environment policy explicitly allows observe mode.
|
||||
*/
|
||||
function validateEnvironmentPolicy(
|
||||
config: Config,
|
||||
env: string,
|
||||
): ObserveCheck {
|
||||
const envPolicy = config.environments?.[env];
|
||||
|
||||
if (!envPolicy) {
|
||||
// No explicit policy for this environment — warn but don't fail
|
||||
return {
|
||||
name: 'environment-policy',
|
||||
status: 'pass',
|
||||
message: `No environment policy defined for "${env}"`,
|
||||
detail: 'Observe mode is allowed by default when no policy is configured.',
|
||||
};
|
||||
}
|
||||
|
||||
const allowObserve = envPolicy.allowObserve;
|
||||
|
||||
if (allowObserve === false) {
|
||||
return {
|
||||
name: 'environment-policy',
|
||||
status: 'fail',
|
||||
message: `Environment policy for "${env}" explicitly blocks observe mode`,
|
||||
detail: 'Set allowObserve: true in the environment policy to enable observe mode, ' +
|
||||
'or run in an environment where observe is allowed.',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'environment-policy',
|
||||
status: 'pass',
|
||||
message: `Environment "${env}" explicitly allows observe mode`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate environment-specific safety constraints.
|
||||
*/
|
||||
function validateEnvironmentSafety(
|
||||
env: string,
|
||||
profileConfig: ObserveProfileConfig,
|
||||
): ObserveCheck {
|
||||
const isProd = env === 'production' || env === 'prod';
|
||||
|
||||
if (isProd) {
|
||||
const warnings = [];
|
||||
if (profileConfig.sampling === undefined) {
|
||||
warnings.push('sampling rate not configured (will use default 1.0)');
|
||||
}
|
||||
if (!profileConfig.sinks) {
|
||||
warnings.push('no sinks configured');
|
||||
}
|
||||
|
||||
if (warnings.length > 0) {
|
||||
return {
|
||||
name: 'environment-safety',
|
||||
status: 'warn',
|
||||
message: `Production environment "${env}" observe configuration has warnings: ${warnings.join(', ')}`,
|
||||
detail: 'In production, configure explicit sampling rate and sinks ' +
|
||||
'to control observation overhead and ensure data capture.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'environment-safety',
|
||||
status: 'pass',
|
||||
message: `Environment "${env}" safety checks passed`,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Exports for testing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
QUALIFY_ONLY_FEATURES,
|
||||
SAMPLING_MIN,
|
||||
SAMPLING_MAX,
|
||||
};
|
||||
@@ -11,7 +11,7 @@
|
||||
* - No optional imports — everything is passed via parameters
|
||||
*/
|
||||
|
||||
import { applyChaosToExecution, createChaosEventArbitrary, formatChaosEvents } from '../../../quality/chaos-v3.js'
|
||||
import { applyChaosToExecution, formatChaosEvents, extractDelays, sleep as chaosSleep } from '../../../quality/chaos-v3.js'
|
||||
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
|
||||
import type {
|
||||
RouteContract,
|
||||
@@ -23,6 +23,12 @@ import type { QualifyRunnerDeps, ChaosRunResult } from './runner.js'
|
||||
/**
|
||||
* Run a single route with chaos injection and collect traces.
|
||||
* Uses chaos-v3 pure functions for deterministic adversity.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Generate deterministic chaos events
|
||||
* 2. Apply transport-level delays (sleep before HTTP request)
|
||||
* 3. Execute the HTTP request
|
||||
* 4. Apply remaining chaos (error, dropout, corruption) to the response context
|
||||
*/
|
||||
export async function runChaosOnRoute(
|
||||
deps: QualifyRunnerDeps,
|
||||
@@ -31,12 +37,16 @@ export async function runChaosOnRoute(
|
||||
): Promise<{ ctx: EvalContext; chaosResult: ChaosRunResult }> {
|
||||
const started = Date.now()
|
||||
|
||||
// Generate chaos events using seeded RNG via fast-check
|
||||
// For CLI qualify, we use a deterministic subset
|
||||
const rng = new SeededRng(deps.seed)
|
||||
const contractNames: string[] = []
|
||||
// 1. Generate deterministic chaos events
|
||||
const events = generateDeterministicChaosEvents(chaosConfig, deps.seed)
|
||||
|
||||
// Build a minimal request for the route
|
||||
// 2. Apply transport-level delays BEFORE the HTTP request
|
||||
const { totalMs: delayMs } = extractDelays(events)
|
||||
if (delayMs > 0) {
|
||||
await chaosSleep(delayMs)
|
||||
}
|
||||
|
||||
// 3. Build a minimal request and execute it
|
||||
const request = {
|
||||
method: route.method,
|
||||
url: route.path,
|
||||
@@ -45,25 +55,57 @@ export async function runChaosOnRoute(
|
||||
body: undefined as unknown,
|
||||
}
|
||||
|
||||
// Execute the request
|
||||
const { executeHttp } = await import('../../../infrastructure/http-executor.js')
|
||||
const ctx = await executeHttp(deps.fastify, route, request, undefined, deps.timeout)
|
||||
|
||||
// Generate and apply chaos events
|
||||
const chaosArb = createChaosEventArbitrary(chaosConfig, contractNames)
|
||||
// For deterministic CLI runs, we generate a fixed small set of events
|
||||
// In practice, fast-check would be used in property tests; here we simulate
|
||||
const events = generateDeterministicChaosEvents(chaosConfig, deps.seed)
|
||||
// 4. Apply remaining chaos (error, dropout, corruption) to response context
|
||||
// Filter out delay events since they were already applied at the transport level
|
||||
const nonDelayEvents = events.filter(e => e.type !== 'inbound-delay')
|
||||
const application = applyChaosToExecution(ctx, nonDelayEvents)
|
||||
|
||||
const application = applyChaosToExecution(ctx, events)
|
||||
// Validate route contracts after chaos. Compose plugin contracts if available.
|
||||
let contractsPassed = true
|
||||
if (deps.pluginContractRegistry) {
|
||||
const composed = deps.pluginContractRegistry.composeContracts(route)
|
||||
if (!route.formulaSources) route.formulaSources = {}
|
||||
for (const phase of Object.values(composed.phases)) {
|
||||
for (const req of phase.requires) {
|
||||
if (!route.requires.includes(req.formula)) {
|
||||
route.requires = [...route.requires, req.formula]
|
||||
route.formulaSources[req.formula] = req.source
|
||||
}
|
||||
}
|
||||
for (const ens of phase.ensures) {
|
||||
if (!route.ensures.includes(ens.formula)) {
|
||||
route.ensures = [...route.ensures, ens.formula]
|
||||
route.formulaSources[ens.formula] = ens.source
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (application.applied && route.ensures.length > 0) {
|
||||
const { validatePostconditionsAsync } = await import('../../../domain/contract-validation.js')
|
||||
const result = await validatePostconditionsAsync(
|
||||
route.ensures,
|
||||
application.ctx,
|
||||
route,
|
||||
undefined as unknown as import('../../../extension/types.js').ExtensionRegistry
|
||||
)
|
||||
contractsPassed = result.success
|
||||
}
|
||||
|
||||
// Consider delay as "applied chaos" for reporting purposes
|
||||
const hadDelay = delayMs > 0
|
||||
const chaosApplied = application.applied || hadDelay
|
||||
|
||||
const chaosResult: ChaosRunResult = {
|
||||
applied: application.applied,
|
||||
applied: chaosApplied,
|
||||
events: application.events
|
||||
.filter(e => e.type !== 'none')
|
||||
.map(e => formatChaosEvents([e])),
|
||||
route: `${route.method} ${route.path}`,
|
||||
durationMs: Date.now() - started,
|
||||
contractsPassed,
|
||||
}
|
||||
|
||||
return { ctx: application.ctx, chaosResult }
|
||||
|
||||
+142
-172
@@ -19,7 +19,7 @@
|
||||
import type { CliContext } from '../../core/context.js'
|
||||
import { loadConfig } from '../../core/config-loader.js'
|
||||
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
||||
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
|
||||
|
||||
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
||||
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
|
||||
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
type StepTrace,
|
||||
type CleanupFailure,
|
||||
} from './runner.js'
|
||||
import { isGitRepo, filterChangedRoutes } from '../verify/runner.js'
|
||||
import { SeededRng } from '../../../infrastructure/seeded-rng.js'
|
||||
import type { ScenarioConfig, TestConfig, RouteContract, ChaosConfig } from '../../../types.js'
|
||||
import { renderHumanArtifact } from '../../renderers/human.js'
|
||||
@@ -37,7 +38,6 @@ import { renderJson, renderJsonArtifact, renderJsonSummaryArtifact } from '../..
|
||||
import { renderNdjsonArtifact, renderNdjsonSummaryArtifact } from '../../renderers/ndjson.js'
|
||||
import type { OutputContext } from '../../renderers/shared.js'
|
||||
import { resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const ROUTE_IDENTITY_PATTERN = /^[A-Z]+\s+\/\S*$/
|
||||
|
||||
@@ -54,13 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
|
||||
return ROUTE_IDENTITY_PATTERN.test(route)
|
||||
}
|
||||
|
||||
function coerceDepth(value: unknown): TestConfig['depth'] {
|
||||
if (value === 'quick' || value === 'standard' || value === 'thorough') {
|
||||
return value
|
||||
}
|
||||
return 'standard'
|
||||
}
|
||||
|
||||
function coerceTimeout(value: unknown): number | undefined {
|
||||
return typeof value === 'number' ? value : undefined
|
||||
}
|
||||
@@ -71,11 +64,11 @@ function coerceTimeout(value: unknown): number | undefined {
|
||||
|
||||
export interface QualifyOptions {
|
||||
profile?: string
|
||||
generationProfile?: string
|
||||
seed?: number
|
||||
changed?: boolean
|
||||
config?: string
|
||||
cwd?: string
|
||||
format?: 'human' | 'json' | 'ndjson'
|
||||
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary'
|
||||
quiet?: boolean
|
||||
verbose?: boolean
|
||||
artifactDir?: string
|
||||
@@ -106,24 +99,11 @@ export function generateSeed(): number {
|
||||
|
||||
/**
|
||||
* Discover routes from the Fastify app for chaos execution.
|
||||
* Injected fastify instance must have routes registered.
|
||||
* Uses the shared discovery module for capture + fallback support.
|
||||
*/
|
||||
async function discoverAppRoutes(fastify: unknown): Promise<RouteContract[]> {
|
||||
// Cast to access routes
|
||||
const app = fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> }> }
|
||||
if (!app.routes) return []
|
||||
|
||||
return app.routes.map(r => ({
|
||||
path: r.url,
|
||||
method: r.method as RouteContract['method'],
|
||||
category: 'observer',
|
||||
requires: [],
|
||||
ensures: [],
|
||||
invariants: [],
|
||||
regexPatterns: {},
|
||||
validateRuntime: false,
|
||||
schema: r.schema,
|
||||
}))
|
||||
const { discoverRoutes } = await import('../../../domain/discovery.js')
|
||||
return discoverRoutes(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -274,7 +254,8 @@ export function buildArtifact(
|
||||
|
||||
const totalTests =
|
||||
runResult.scenarioResults.reduce((sum, s) => sum + s.steps.length, 0) +
|
||||
(runResult.statefulResult?.tests.length ?? 0)
|
||||
(runResult.statefulResult?.summary.passed ?? 0) +
|
||||
(runResult.statefulResult?.summary.failed ?? 0)
|
||||
|
||||
const passedTests =
|
||||
runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0) +
|
||||
@@ -296,12 +277,55 @@ export function buildArtifact(
|
||||
// Build execution summary from runner result
|
||||
const executionSummary = runResult.executionSummary
|
||||
|
||||
// Build per-gate coverage breakdown for clear artifact interpretation
|
||||
const scenarioRoutes = [...new Set(
|
||||
runResult.stepTraces
|
||||
.filter(t => t.status === 'passed' || t.status === 'failed')
|
||||
.map(t => t.route)
|
||||
.filter(Boolean)
|
||||
)]
|
||||
const statefulRoutes = runResult.statefulResult
|
||||
? [...new Set(
|
||||
runResult.statefulResult.tests
|
||||
.filter(t => !t.directive)
|
||||
.map(t => normalizeRouteIdentity(t.name))
|
||||
)]
|
||||
: []
|
||||
const chaosRoutesPlanned: string[] = []
|
||||
const chaosRoutesExecuted: string[] = []
|
||||
for (const r of runResult.chaosResults ?? []) {
|
||||
if (r.applied) chaosRoutesExecuted.push(r.route)
|
||||
}
|
||||
// Infer planned from chaos results or executionSummary
|
||||
if (executionSummary.chaosRoutesPlanned > 0 && chaosRoutesExecuted.length > 0) {
|
||||
chaosRoutesPlanned.push(...chaosRoutesExecuted)
|
||||
}
|
||||
|
||||
const coverageBreakdown = {
|
||||
scenario: {
|
||||
routesCovered: scenarioRoutes,
|
||||
stepsTotal: executionSummary.totalSteps,
|
||||
stepsPassed: runResult.scenarioResults.reduce((sum, s) => sum + s.summary.passed, 0),
|
||||
},
|
||||
stateful: {
|
||||
routesCovered: statefulRoutes,
|
||||
testsTotal: executionSummary.statefulTestsRun,
|
||||
testsPassed: runResult.statefulResult?.summary.passed ?? 0,
|
||||
},
|
||||
chaos: {
|
||||
routesPlanned: chaosRoutesPlanned,
|
||||
routesExecuted: chaosRoutesExecuted,
|
||||
runsTotal: executionSummary.chaosRunsRun,
|
||||
runsPassed: (runResult.chaosResults ?? []).filter(r => r.contractsPassed).length,
|
||||
},
|
||||
}
|
||||
|
||||
// Build profile gates from the result context
|
||||
// We need to pass gates through or infer from results
|
||||
const profileGates = {
|
||||
scenario: runResult.scenarioResults.length > 0 || executionSummary.scenariosRun > 0,
|
||||
stateful: (runResult.statefulResult?.tests.length ?? 0) > 0 || executionSummary.statefulTestsRun > 0,
|
||||
chaos: (runResult.chaosResult !== undefined) || executionSummary.chaosRunsRun > 0,
|
||||
chaos: executionSummary.chaosRoutesPlanned > 0,
|
||||
}
|
||||
|
||||
// Deterministic parameters for audit
|
||||
@@ -326,8 +350,10 @@ export function buildArtifact(
|
||||
total: totalTests,
|
||||
passed: passedTests,
|
||||
failed: failures.length,
|
||||
skipped: (runResult.statefulResult?.summary.skipped ?? 0),
|
||||
},
|
||||
executionSummary,
|
||||
coverageBreakdown,
|
||||
executedRoutes: (runResult.executedRoutes || []).map(normalizeRouteIdentity),
|
||||
skippedRoutes: (runResult.skippedRoutes || []).map(sr => ({
|
||||
route: sr.route,
|
||||
@@ -394,117 +420,6 @@ async function emitArtifact(
|
||||
// Output formatting
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function formatHumanOutput(
|
||||
result: QualifyRunResult,
|
||||
options: { profile?: string; seed: number; env: string },
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
|
||||
lines.push(`Qualify run for environment "${options.env}"`)
|
||||
if (options.profile) {
|
||||
lines.push(`Profile: ${options.profile}`)
|
||||
}
|
||||
lines.push(`Seed: ${options.seed}`)
|
||||
lines.push('')
|
||||
|
||||
// Scenario results
|
||||
for (const scenario of result.scenarioResults) {
|
||||
lines.push(`Scenario: ${scenario.name}`)
|
||||
for (const step of scenario.steps) {
|
||||
const icon = step.ok ? '✓' : '✗'
|
||||
lines.push(` ${icon} ${step.name} (${step.statusCode ?? 'no-status'})`)
|
||||
if (!step.ok && step.diagnostics) {
|
||||
lines.push(` Expected: ${step.diagnostics.expected || 'success'}`)
|
||||
lines.push(` Observed: ${step.diagnostics.error || 'failure'}`)
|
||||
if (step.diagnostics.actual) {
|
||||
lines.push(` Actual: ${step.diagnostics.actual}`)
|
||||
}
|
||||
if (step.diagnostics.diff) {
|
||||
lines.push(` Diff:`)
|
||||
for (const line of String(step.diagnostics.diff).split('\n')) {
|
||||
lines.push(` ${line}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Stateful results
|
||||
if (result.statefulResult) {
|
||||
lines.push(`Stateful: ${result.statefulResult.summary.passed} passed, ${result.statefulResult.summary.failed} failed`)
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Chaos results
|
||||
if (result.chaosResult) {
|
||||
lines.push(`Chaos: ${result.chaosResult.applied ? 'applied' : 'none'}`)
|
||||
if (result.chaosResult.events.length > 0) {
|
||||
for (const event of result.chaosResult.events) {
|
||||
lines.push(` ${event}`)
|
||||
}
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Step traces
|
||||
if (result.stepTraces.length > 0) {
|
||||
lines.push('Step traces:')
|
||||
for (const trace of result.stepTraces.slice(0, 20)) {
|
||||
const icon = trace.status === 'passed' ? '✓' : trace.status === 'skipped' ? '⊘' : '✗'
|
||||
lines.push(` ${icon} ${trace.name} (${trace.durationMs}ms)`)
|
||||
}
|
||||
if (result.stepTraces.length > 20) {
|
||||
lines.push(` ... and ${result.stepTraces.length - 20} more`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Cleanup failures
|
||||
if (result.cleanupFailures.length > 0) {
|
||||
lines.push('Cleanup failures (reported separately):')
|
||||
for (const cf of result.cleanupFailures) {
|
||||
lines.push(` ⚠ ${cf.resource}: ${cf.error}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Per-profile gate execution counts
|
||||
lines.push('Profile gate execution counts:')
|
||||
lines.push(` Scenario: ${result.executionSummary.scenariosRun} run`)
|
||||
lines.push(` Stateful: ${result.executionSummary.statefulTestsRun} tests run`)
|
||||
lines.push(` Chaos: ${result.executionSummary.chaosRunsRun} runs run`)
|
||||
lines.push('')
|
||||
|
||||
// Executed routes
|
||||
if (result.executedRoutes.length > 0) {
|
||||
lines.push(`Executed routes (${result.executedRoutes.length}):`)
|
||||
for (const route of result.executedRoutes) {
|
||||
lines.push(` ${route}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Skipped routes
|
||||
if (result.skippedRoutes.length > 0) {
|
||||
lines.push(`Skipped routes (${result.skippedRoutes.length}):`)
|
||||
for (const sr of result.skippedRoutes) {
|
||||
lines.push(` ${sr.route}: ${sr.reason}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Summary
|
||||
if (result.passed) {
|
||||
lines.push('All qualifications passed.')
|
||||
} else {
|
||||
lines.push('Qualification failed.')
|
||||
lines.push(`Replay: apophis replay --artifact <artifact-path>`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main command handler
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -529,8 +444,8 @@ export async function qualifyCommand(
|
||||
): Promise<CommandResult> {
|
||||
const {
|
||||
profile,
|
||||
generationProfile,
|
||||
seed: explicitSeed,
|
||||
changed,
|
||||
config: configPath,
|
||||
cwd,
|
||||
artifactDir,
|
||||
@@ -558,7 +473,6 @@ export async function qualifyCommand(
|
||||
}
|
||||
|
||||
const config = loadResult.config
|
||||
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
|
||||
|
||||
// 2. Run policy engine checks
|
||||
const policyEngine = new PolicyEngine({
|
||||
@@ -593,19 +507,33 @@ export async function qualifyCommand(
|
||||
const profileDef = profile ? config.profiles?.[profile] : undefined
|
||||
const gates = resolveProfileGates(profileDef?.features)
|
||||
|
||||
// 5. Build scenario configs from profile routes
|
||||
// 5. Build scenario configs from profile routes and config
|
||||
const routes = profileDef?.routes ?? []
|
||||
const scenarios = buildScenarioConfigs(routes, seed)
|
||||
const fixtureScenarios = buildScenarioConfigs(routes, seed)
|
||||
|
||||
const configScenarios: ScenarioConfig[] = (config.scenarios ?? []).map(s => ({
|
||||
name: s.name,
|
||||
steps: s.steps.map(step => ({
|
||||
name: step.name,
|
||||
request: {
|
||||
method: step.request.method as ScenarioConfig['steps'][0]['request']['method'],
|
||||
url: step.request.url,
|
||||
body: step.request.body,
|
||||
headers: step.request.headers,
|
||||
},
|
||||
expect: step.expect,
|
||||
capture: step.capture,
|
||||
})),
|
||||
}))
|
||||
|
||||
const scenarios = [...fixtureScenarios, ...configScenarios]
|
||||
|
||||
// 6. Build stateful config
|
||||
const presetName = profileDef?.preset
|
||||
const preset = presetName ? config.presets?.[presetName] : undefined
|
||||
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
|
||||
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
|
||||
const statefulConfig: TestConfig | undefined = gates.stateful
|
||||
? {
|
||||
depth: presetDepth,
|
||||
generationProfile: resolvedGenerationProfile,
|
||||
seed,
|
||||
timeout: presetTimeout,
|
||||
routes: profileDef?.routes,
|
||||
@@ -616,6 +544,9 @@ export async function qualifyCommand(
|
||||
const chaosConfig: ChaosConfig | undefined = gates.chaos && preset?.chaos
|
||||
? {
|
||||
probability: 0.5,
|
||||
strategy: preset.chaosStrategy as ChaosConfig['strategy'],
|
||||
sampleSize: preset.chaosSampleSize,
|
||||
sampleRoutes: preset.chaosSampleRoutes,
|
||||
delay: { probability: 0.3, minMs: 100, maxMs: 500 },
|
||||
error: { probability: 0.2, statusCode: 503 },
|
||||
dropout: { probability: 0.2, statusCode: 504 },
|
||||
@@ -624,37 +555,72 @@ export async function qualifyCommand(
|
||||
: undefined
|
||||
|
||||
// 8. Load the Fastify app for execution
|
||||
// Try to import the app from the fixture
|
||||
let fastify: FastifyAppLike | undefined
|
||||
let appEntrypoint: string | undefined
|
||||
try {
|
||||
const appPath = resolve(workingDir, 'app.js')
|
||||
const appUrl = pathToFileURL(appPath)
|
||||
appUrl.searchParams.set('apophisRun', String(Date.now()))
|
||||
const appModule = await import(appUrl.href)
|
||||
fastify = (appModule.default || appModule) as FastifyAppLike
|
||||
const { loadApp } = await import('../../core/app-loader.js')
|
||||
const loaded = await loadApp(workingDir)
|
||||
fastify = loaded.fastify as FastifyAppLike
|
||||
appEntrypoint = loaded.entrypoint
|
||||
if (fastify && typeof fastify.ready === 'function') {
|
||||
await fastify.ready()
|
||||
}
|
||||
} catch (err) {
|
||||
// App not available — return a result indicating no app to test
|
||||
if (process.env.APOPHIS_DEBUG === '1') {
|
||||
console.error('Failed to load app:', err)
|
||||
}
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: 'No Fastify app found. Ensure app.js exports a Fastify instance.',
|
||||
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// 9. Discover routes for chaos
|
||||
const appRoutes = await discoverAppRoutes(fastify)
|
||||
// 9. Discover routes for chaos (with discovery metadata)
|
||||
const allRoutes = await discoverAppRoutes(fastify)
|
||||
let appRoutes = allRoutes
|
||||
|
||||
// 9a. Apply --changed filter
|
||||
if (changed) {
|
||||
const inGit = await isGitRepo(workingDir)
|
||||
if (!inGit) {
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: '--changed requires a git repository. Current directory is not inside a git repo.',
|
||||
}
|
||||
}
|
||||
appRoutes = await filterChangedRoutes(allRoutes, workingDir)
|
||||
const matched = appRoutes.length
|
||||
const total = allRoutes.length
|
||||
if (!ctx.options.quiet && format === 'human') {
|
||||
console.log(`${matched}/${total} routes match --changed filter`)
|
||||
}
|
||||
if (matched === 0) {
|
||||
return {
|
||||
exitCode: SUCCESS,
|
||||
message: `No routes changed since HEAD. All ${total} routes skipped.`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect discovery warnings
|
||||
const { discoverRouteDetails } = await import('../../../domain/discovery.js')
|
||||
const discoveryResult = discoverRouteDetails(fastify as { routes?: Array<{ method: string; url: string; schema?: Record<string, unknown> } >; hasRoute?: (opts: { method: string; url: string }) => boolean; printRoutes?: () => string })
|
||||
const discoveryWarnings: string[] = [
|
||||
...discoveryResult.warnings,
|
||||
]
|
||||
if (discoveryResult.source === 'print-routes' && !discoveryResult.hasSchemaMetadata) {
|
||||
discoveryWarnings.push(
|
||||
'Routes were discovered without schema metadata. Behavioral contracts and chaos injection may be limited. ' +
|
||||
'Register APOPHIS before defining routes or use createFastify().'
|
||||
)
|
||||
}
|
||||
|
||||
// 10. Run qualify execution
|
||||
const deps = {
|
||||
fastify: fastify as any,
|
||||
seed,
|
||||
timeout: presetTimeout,
|
||||
pluginContractRegistry: (fastify as unknown as Record<string, unknown>)?._apophisPluginContractRegistry as
|
||||
import('../../../domain/plugin-contracts.js').PluginContractRegistry | undefined,
|
||||
}
|
||||
|
||||
const runResult = await runQualify(deps, gates, scenarios, statefulConfig, chaosConfig, appRoutes)
|
||||
@@ -669,9 +635,18 @@ export async function qualifyCommand(
|
||||
seed,
|
||||
})
|
||||
|
||||
// Drain plugin contract registry warnings if app registered APOPHIS
|
||||
const pcr = (fastify as any)?._apophisPluginContractRegistry
|
||||
if (pcr?.drainWarnings) {
|
||||
const pcrWarnings = pcr.drainWarnings()
|
||||
if (pcrWarnings.length > 0) {
|
||||
artifact.warnings.push(...pcrWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
// 12. Signal quality guardrails — fail if zero checks executed
|
||||
const execSummary = runResult.executionSummary
|
||||
const warnings: string[] = [...artifact.warnings]
|
||||
const warnings: string[] = [...artifact.warnings, ...discoveryWarnings]
|
||||
|
||||
if (execSummary.totalExecuted === 0) {
|
||||
await emitArtifact(artifact, {
|
||||
@@ -693,7 +668,9 @@ export async function qualifyCommand(
|
||||
if (gates.scenario && execSummary.scenariosRun === 0) {
|
||||
warnings.push('WARNING: scenario gate enabled but zero scenarios executed. Check route configuration.')
|
||||
}
|
||||
if (gates.stateful && execSummary.statefulTestsRun === 0) {
|
||||
if (gates.stateful && execSummary.statefulTestsRun === 0 && execSummary.statefulSkipped > 0) {
|
||||
warnings.push(`WARNING: stateful gate enabled but all ${execSummary.statefulSkipped} commands were skipped (preconditions not met). Check required headers, scopes, or plugin contract settings.`)
|
||||
} else if (gates.stateful && execSummary.statefulTestsRun === 0) {
|
||||
warnings.push('WARNING: stateful gate enabled but zero stateful tests executed. Check app routes and schema.')
|
||||
}
|
||||
if (gates.chaos && execSummary.chaosRunsRun === 0) {
|
||||
@@ -752,12 +729,6 @@ export async function qualifyCommand(
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof GenerationProfileResolutionError) {
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: error.message,
|
||||
}
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return {
|
||||
exitCode: INTERNAL_ERROR,
|
||||
@@ -780,8 +751,8 @@ export async function handleQualify(
|
||||
): Promise<number> {
|
||||
const options: QualifyOptions = {
|
||||
profile: ctx.options.profile || undefined,
|
||||
generationProfile: ctx.options.generationProfile,
|
||||
seed: undefined,
|
||||
changed: false,
|
||||
config: ctx.options.config || undefined,
|
||||
cwd: ctx.cwd,
|
||||
format: ctx.options.format as QualifyOptions['format'],
|
||||
@@ -798,9 +769,8 @@ export async function handleQualify(
|
||||
}
|
||||
}
|
||||
|
||||
const generationProfileIdx = args.indexOf('--generation-profile')
|
||||
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
|
||||
options.generationProfile = args[generationProfileIdx + 1]
|
||||
if (args.includes('--changed')) {
|
||||
options.changed = true
|
||||
}
|
||||
|
||||
const result = await qualifyCommand(options, ctx)
|
||||
|
||||
@@ -29,6 +29,7 @@ import type {
|
||||
FastifyInjectInstance,
|
||||
} from '../../../types.js'
|
||||
import type { ExtensionRegistry } from '../../../extension/types.js'
|
||||
import type { PluginContractRegistry } from '../../../domain/plugin-contracts.js'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -48,6 +49,7 @@ export interface QualifyRunResult {
|
||||
scenarioResults: ScenarioResult[]
|
||||
statefulResult?: TestSuite
|
||||
chaosResult?: ChaosRunResult
|
||||
chaosResults: ChaosRunResult[]
|
||||
stepTraces: StepTrace[]
|
||||
cleanupFailures: CleanupFailure[]
|
||||
durationMs: number
|
||||
@@ -59,7 +61,10 @@ export interface QualifyRunResult {
|
||||
totalFailed: number
|
||||
scenariosRun: number
|
||||
statefulTestsRun: number
|
||||
statefulSkipped: number
|
||||
chaosRunsRun: number
|
||||
chaosRoutesPlanned: number
|
||||
chaosRoutesExecuted: number
|
||||
totalSteps: number
|
||||
}
|
||||
executedRoutes: string[]
|
||||
@@ -71,6 +76,7 @@ export interface ChaosRunResult {
|
||||
events: string[]
|
||||
route: string
|
||||
durationMs: number
|
||||
contractsPassed: boolean
|
||||
}
|
||||
|
||||
export interface CleanupFailure {
|
||||
@@ -81,6 +87,7 @@ export interface CleanupFailure {
|
||||
export interface QualifyRunnerDeps {
|
||||
fastify: FastifyInjectInstance
|
||||
extensionRegistry?: ExtensionRegistry
|
||||
pluginContractRegistry?: PluginContractRegistry
|
||||
seed: number
|
||||
timeout?: number
|
||||
}
|
||||
@@ -110,6 +117,45 @@ export function resolveProfileGates(features?: string[]): ProfileGates {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Select routes for chaos injection based on strategy.
|
||||
*
|
||||
* - 'one': picks one route deterministically (default, backward-compatible)
|
||||
* - 'all': every provided route
|
||||
* - 'sample': randomly sampled subset
|
||||
* - 'routes': explicit list matched against route strings "METHOD /path"
|
||||
*/
|
||||
export function selectChaosRoutes(
|
||||
allRoutes: RouteContract[],
|
||||
config: ChaosConfig,
|
||||
seed: number,
|
||||
): RouteContract[] {
|
||||
const strategy = config.strategy ?? 'one'
|
||||
const rng = new SeededRng(seed)
|
||||
|
||||
if (strategy === 'all') {
|
||||
return [...allRoutes]
|
||||
}
|
||||
|
||||
if (strategy === 'sample') {
|
||||
const size = Math.min(config.sampleSize ?? 3, allRoutes.length)
|
||||
const shuffled = [...allRoutes].sort(() => rng.next() - 0.5)
|
||||
return shuffled.slice(0, size)
|
||||
}
|
||||
|
||||
if (strategy === 'routes' && config.sampleRoutes) {
|
||||
return allRoutes.filter(r => {
|
||||
const id = `${r.method} ${r.path}`
|
||||
return config.sampleRoutes!.some(p => id === p)
|
||||
})
|
||||
}
|
||||
|
||||
// 'one' — default: pick one route deterministically
|
||||
const idx = Math.floor(rng.next() * allRoutes.length)
|
||||
const route = allRoutes[idx]
|
||||
return route ? [route] : []
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main qualify runner
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -125,13 +171,14 @@ export async function runQualify(
|
||||
statefulConfig?: TestConfig,
|
||||
chaosConfig?: ChaosConfig,
|
||||
routes?: RouteContract[],
|
||||
cleanupManager?: import('../../../infrastructure/cleanup-manager.js').CleanupManager,
|
||||
): Promise<QualifyRunResult> {
|
||||
const started = Date.now()
|
||||
const scenarioResults: ScenarioResult[] = []
|
||||
const allTraces: StepTrace[] = []
|
||||
const cleanupFailures: CleanupFailure[] = []
|
||||
let statefulResult: TestSuite | undefined
|
||||
let chaosResult: ChaosRunResult | undefined
|
||||
const chaosResults: ChaosRunResult[] = []
|
||||
|
||||
// Run scenarios
|
||||
if (gates.scenario) {
|
||||
@@ -149,39 +196,64 @@ export async function runQualify(
|
||||
allTraces.push(...traces)
|
||||
}
|
||||
|
||||
// Run chaos on routes
|
||||
// Run chaos on selected routes
|
||||
let chaosRoutesPlanned = 0
|
||||
let chaosRoutesExecuted = 0
|
||||
if (gates.chaos && chaosConfig && routes && routes.length > 0) {
|
||||
// Pick one route deterministically for CLI chaos demo
|
||||
const rng = new SeededRng(deps.seed)
|
||||
const route = routes[Math.floor(rng.next() * routes.length)]
|
||||
if (route) {
|
||||
const { chaosResult: cr } = await runChaosOnRoute(deps, route, chaosConfig)
|
||||
chaosResult = cr
|
||||
const selectedRoutes = selectChaosRoutes(routes, chaosConfig, deps.seed)
|
||||
chaosRoutesPlanned = selectedRoutes.length
|
||||
|
||||
for (const route of selectedRoutes) {
|
||||
try {
|
||||
const { chaosResult: cr } = await runChaosOnRoute(deps, route, chaosConfig)
|
||||
chaosResults.push(cr)
|
||||
if (cr.applied) {
|
||||
chaosRoutesExecuted++
|
||||
}
|
||||
} catch {
|
||||
// Individual chaos run failure should not abort the qualify run
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate cleanup tracking
|
||||
// In real usage, cleanupManager would be injected and tracked
|
||||
// For now, cleanup failures are empty unless injected by caller
|
||||
// Run cleanup and track failures
|
||||
if (cleanupManager) {
|
||||
try {
|
||||
const outcomes = await cleanupManager.cleanup()
|
||||
for (const outcome of outcomes) {
|
||||
if (outcome.error) {
|
||||
cleanupFailures.push({
|
||||
resource: `${outcome.resource.type}/${outcome.resource.id} (${outcome.resource.url})`,
|
||||
error: outcome.error,
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Cleanup failures are tracked but should not block the qualify result
|
||||
}
|
||||
}
|
||||
|
||||
const durationMs = Date.now() - started
|
||||
|
||||
// Determine overall pass/fail
|
||||
const scenarioPassed = scenarioResults.every(r => r.ok)
|
||||
const statefulPassed = !statefulResult || statefulResult.summary.failed === 0
|
||||
const chaosPassed = !chaosResult || chaosResult.applied // chaos "passes" if it applied
|
||||
const chaosPassed = chaosResults.every(r => !r.applied || r.contractsPassed)
|
||||
|
||||
// Count execution metrics
|
||||
// Count execution metrics — distinguish executed from skipped
|
||||
const scenariosRun = scenarioResults.length
|
||||
const statefulTestsRun = statefulResult?.tests.length ?? 0
|
||||
const chaosRunsRun = chaosResult ? 1 : 0
|
||||
const statefulPassedCount = statefulResult?.summary.passed ?? 0
|
||||
const statefulFailedCount = statefulResult?.summary.failed ?? 0
|
||||
const statefulSkippedCount = statefulResult?.summary.skipped ?? 0
|
||||
const statefulTestsRun = statefulPassedCount + statefulFailedCount
|
||||
const chaosRunsRun = chaosResults.length
|
||||
const totalSteps = allTraces.length
|
||||
const totalExecuted = scenariosRun + statefulTestsRun + chaosRunsRun
|
||||
const totalPassed = scenarioResults.reduce((sum, r) => sum + r.summary.passed, 0) +
|
||||
(statefulResult?.summary.passed ?? 0) +
|
||||
(chaosResult?.applied ? 1 : 0)
|
||||
statefulPassedCount +
|
||||
chaosResults.reduce((sum, r) => sum + (r.applied && r.contractsPassed ? 1 : 0), 0)
|
||||
const totalFailed = scenarioResults.reduce((sum, r) => sum + r.summary.failed, 0) +
|
||||
(statefulResult?.summary.failed ?? 0)
|
||||
statefulFailedCount
|
||||
|
||||
// Track executed and skipped routes for transparency
|
||||
const executedRoutes: string[] = []
|
||||
@@ -204,9 +276,9 @@ export async function runQualify(
|
||||
}
|
||||
}
|
||||
|
||||
// Track chaos route
|
||||
if (chaosResult) {
|
||||
executedRoutes.push(chaosResult.route)
|
||||
// Track chaos routes
|
||||
for (const cr of chaosResults) {
|
||||
executedRoutes.push(cr.route)
|
||||
}
|
||||
|
||||
// Track skipped routes from profile filters
|
||||
@@ -224,6 +296,9 @@ export async function runQualify(
|
||||
reason = 'Stateful config missing or invalid'
|
||||
} else if (gates.chaos && !chaosConfig) {
|
||||
reason = 'Chaos config missing or invalid'
|
||||
} else if (gates.chaos && chaosConfig) {
|
||||
const strategy = chaosConfig.strategy ?? 'one'
|
||||
reason = `Not selected by chaos strategy: ${strategy}`
|
||||
}
|
||||
skippedRoutes.push({ route: routeStr, reason })
|
||||
}
|
||||
@@ -234,19 +309,23 @@ export async function runQualify(
|
||||
passed: scenarioPassed && statefulPassed && chaosPassed,
|
||||
scenarioResults,
|
||||
statefulResult,
|
||||
chaosResult,
|
||||
chaosResult: chaosResults[0],
|
||||
chaosResults,
|
||||
stepTraces: allTraces,
|
||||
cleanupFailures,
|
||||
durationMs,
|
||||
seed: deps.seed,
|
||||
executionSummary: {
|
||||
totalPlanned: scenarios.length + (statefulConfig ? 1 : 0) + (chaosConfig && routes && routes.length > 0 ? 1 : 0),
|
||||
totalPlanned: scenarios.length + (statefulConfig ? 1 : 0) + chaosRoutesPlanned,
|
||||
totalExecuted,
|
||||
totalPassed,
|
||||
totalFailed,
|
||||
scenariosRun,
|
||||
statefulTestsRun,
|
||||
statefulSkipped: statefulSkippedCount,
|
||||
chaosRunsRun,
|
||||
chaosRoutesPlanned,
|
||||
chaosRoutesExecuted,
|
||||
totalSteps,
|
||||
},
|
||||
executedRoutes: [...new Set(executedRoutes)],
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
* - No optional imports — everything is passed via parameters
|
||||
*/
|
||||
|
||||
import { runScenario } from '../../../test/scenario-runner.js'
|
||||
import { runScenario } from '../../../quality/scenario-runner.js'
|
||||
import type {
|
||||
ScenarioConfig,
|
||||
ScenarioResult,
|
||||
@@ -27,7 +27,7 @@ export async function runScenarioWithTraces(
|
||||
): Promise<{ result: ScenarioResult; traces: StepTrace[] }> {
|
||||
const scopeHeaders: Record<string, string> = {}
|
||||
|
||||
const result = await runScenario(deps.fastify, config, scopeHeaders, deps.extensionRegistry)
|
||||
const result = await runScenario(deps.fastify, config, scopeHeaders, deps.extensionRegistry, deps.pluginContractRegistry)
|
||||
|
||||
const traces: StepTrace[] = result.steps.map((step, idx) => {
|
||||
const trace: StepTrace = {
|
||||
|
||||
@@ -10,13 +10,23 @@
|
||||
* - No optional imports — everything is passed via parameters
|
||||
*/
|
||||
|
||||
import { runStatefulTests } from '../../../test/stateful-runner.js'
|
||||
import { runStatefulTests } from '../../../quality/stateful-runner.js'
|
||||
import { CleanupManager } from '../../../infrastructure/cleanup-manager.js'
|
||||
import type {
|
||||
TestConfig,
|
||||
TestSuite,
|
||||
ScopeRegistry,
|
||||
} from '../../../types.js'
|
||||
import type { QualifyRunnerDeps, StepTrace } from './runner.js'
|
||||
|
||||
const minimalScopeRegistry: ScopeRegistry = {
|
||||
scopes: new Map(),
|
||||
defaultScope: { headers: {} },
|
||||
register() {},
|
||||
deriveFromRequest() { return { headers: {} } },
|
||||
getHeaders() { return {} },
|
||||
}
|
||||
|
||||
/**
|
||||
* Run stateful tests with the given config.
|
||||
* Wraps the existing stateful runner.
|
||||
@@ -27,13 +37,15 @@ export async function runStatefulWithTraces(
|
||||
): Promise<{ result: TestSuite; traces: StepTrace[] }> {
|
||||
const started = Date.now()
|
||||
|
||||
const cleanupManager = new CleanupManager(deps.fastify as any, minimalScopeRegistry, false)
|
||||
|
||||
const result = await runStatefulTests(
|
||||
deps.fastify,
|
||||
config,
|
||||
undefined, // cleanupManager — injected if needed by caller
|
||||
undefined, // scopeRegistry
|
||||
cleanupManager,
|
||||
minimalScopeRegistry,
|
||||
deps.extensionRegistry,
|
||||
undefined, // pluginContractRegistry
|
||||
deps.pluginContractRegistry,
|
||||
undefined, // outboundContractRegistry
|
||||
)
|
||||
|
||||
|
||||
@@ -73,33 +73,44 @@ function formatHumanOutput(result: ReplayResult, artifact: Artifact): string {
|
||||
w.includes('Artifact cwd no longer exists')
|
||||
)
|
||||
|
||||
function formatFailure(label: string, failure: FailureRecord | undefined): void {
|
||||
if (!failure) return
|
||||
lines.push(label)
|
||||
lines.push(` Route: ${failure.route}`)
|
||||
if (failure.source && failure.source !== 'route') {
|
||||
lines.push(` Source: ${failure.source}`)
|
||||
}
|
||||
if (failure.contract) {
|
||||
lines.push(` Contract: ${failure.contract}`)
|
||||
}
|
||||
if (failure.expected) {
|
||||
lines.push(` Expected: ${failure.expected}`)
|
||||
}
|
||||
if (failure.observed) {
|
||||
lines.push(` Observed: ${failure.observed}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (result.reproduced) {
|
||||
lines.push('Replay reproduced the original failure.')
|
||||
lines.push('')
|
||||
lines.push('Original failure')
|
||||
lines.push(` Route: ${result.originalFailure?.route}`)
|
||||
lines.push(` Contract: ${result.originalFailure?.contract}`)
|
||||
lines.push(` Expected: ${result.originalFailure?.expected}`)
|
||||
lines.push(` Observed: ${result.originalFailure?.observed}`)
|
||||
formatFailure('Original failure', result.originalFailure)
|
||||
lines.push(` Seed: ${artifact.seed}`)
|
||||
} else if (result.newFailure) {
|
||||
lines.push('Replay produced a different result.')
|
||||
lines.push('')
|
||||
lines.push('Original failure')
|
||||
lines.push(` Route: ${result.originalFailure?.route}`)
|
||||
lines.push(` Contract: ${result.originalFailure?.contract}`)
|
||||
formatFailure('Original failure', result.originalFailure)
|
||||
lines.push('')
|
||||
lines.push('New result')
|
||||
lines.push(` Route: ${result.newFailure.route}`)
|
||||
lines.push(` Contract: ${result.newFailure.contract}`)
|
||||
lines.push(` Expected: ${result.newFailure.expected}`)
|
||||
lines.push(` Observed: ${result.newFailure.observed}`)
|
||||
formatFailure('New result', result.newFailure)
|
||||
lines.push(` Seed: ${artifact.seed}`)
|
||||
} else {
|
||||
lines.push('Replay passed — failure no longer reproduces.')
|
||||
lines.push('')
|
||||
lines.push('Original failure')
|
||||
lines.push(` Route: ${result.originalFailure?.route}`)
|
||||
if (result.originalFailure?.source && result.originalFailure.source !== 'route') {
|
||||
lines.push(` Source: ${result.originalFailure.source}`)
|
||||
}
|
||||
lines.push(` Contract: ${result.originalFailure?.contract}`)
|
||||
lines.push(` Seed: ${artifact.seed}`)
|
||||
}
|
||||
@@ -172,9 +183,36 @@ async function executeContractDirect(
|
||||
validateRuntime: false,
|
||||
}
|
||||
|
||||
// Compose plugin contracts if the app registered the APOPHIS plugin
|
||||
const pluginContractRegistry = (fastify as any)?._apophisPluginContractRegistry
|
||||
if (pluginContractRegistry) {
|
||||
const composed = pluginContractRegistry.composeContracts(routeContract)
|
||||
if (!routeContract.formulaSources) routeContract.formulaSources = {}
|
||||
for (const phase of Object.values(composed.phases) as Array<{ requires: Array<{ formula: string; source: string }>; ensures: Array<{ formula: string; source: string }> }>) {
|
||||
for (const req of phase.requires) {
|
||||
if (!routeContract.requires.includes(req.formula)) {
|
||||
routeContract.requires.push(req.formula)
|
||||
routeContract.formulaSources[req.formula] = req.source as 'route' | `plugin:${string}`
|
||||
}
|
||||
}
|
||||
for (const ens of phase.ensures) {
|
||||
if (!routeContract.ensures.includes(ens.formula)) {
|
||||
routeContract.ensures.push(ens.formula)
|
||||
routeContract.formulaSources[ens.formula] = ens.source as 'route' | `plugin:${string}`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build request
|
||||
const headers: Record<string, string> = {}
|
||||
|
||||
// Auto-inject headers required by plugin contract preconditions
|
||||
for (const f of routeContract.requires) {
|
||||
const m = f.match(/request_headers\(this\)\.([\w-]+)\s*(!=\s*null|==\s*"([^"]+)")/)
|
||||
if (m) headers[m[1]!] = m[3] || 'test-value'
|
||||
}
|
||||
|
||||
// Execute request
|
||||
try {
|
||||
const ctx = await executeHttp(fastify, routeContract, {
|
||||
@@ -256,7 +294,7 @@ async function executeReplay(
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: `Cannot load Fastify app from ${workingDir}/app.js: ${errorMessage}`,
|
||||
message: `Cannot load Fastify app: ${errorMessage}`,
|
||||
warnings,
|
||||
reproduced: false,
|
||||
originalFailure: failure,
|
||||
@@ -264,12 +302,22 @@ async function executeReplay(
|
||||
}
|
||||
|
||||
// Try to run verify first (works if app has APOPHIS plugin)
|
||||
const pluginContractRegistry = (fastify as any)?._apophisPluginContractRegistry
|
||||
let runResult = await runVerify({
|
||||
fastify: fastify as any,
|
||||
seed: artifact.seed || 42,
|
||||
routeFilters: [failure.route],
|
||||
pluginContractRegistry,
|
||||
})
|
||||
|
||||
// Drain plugin contract registry warnings (missing extensions, etc.)
|
||||
if (pluginContractRegistry?.drainWarnings) {
|
||||
const pcrWarnings = pluginContractRegistry.drainWarnings()
|
||||
if (pcrWarnings.length > 0) {
|
||||
warnings.push(...pcrWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
// If no routes matched, or route found but no contracts (plugin not registered before routes),
|
||||
// try direct contract execution
|
||||
if (runResult.noRoutesMatched || runResult.noContractsFound) {
|
||||
@@ -346,21 +394,7 @@ async function executeReplay(
|
||||
|
||||
// Check if there are different failures
|
||||
if (runResult.failures.length > 0) {
|
||||
const newFailure = runResult.failures[0]
|
||||
if (!newFailure) {
|
||||
return {
|
||||
exitCode: SUCCESS,
|
||||
message: formatHumanOutput({
|
||||
exitCode: SUCCESS,
|
||||
reproduced: false,
|
||||
originalFailure: failure,
|
||||
warnings,
|
||||
}, artifact),
|
||||
warnings,
|
||||
reproduced: false,
|
||||
originalFailure: failure,
|
||||
}
|
||||
}
|
||||
const newFailure = runResult.failures[0]!
|
||||
return {
|
||||
exitCode: BEHAVIORAL_FAILURE,
|
||||
message: formatHumanOutput({
|
||||
@@ -374,6 +408,7 @@ async function executeReplay(
|
||||
observed: newFailure.observed,
|
||||
seed: artifact.seed || 42,
|
||||
replayCommand: `apophis replay --artifact ${artifactPath}`,
|
||||
source: newFailure.source,
|
||||
},
|
||||
warnings,
|
||||
}, artifact),
|
||||
@@ -387,6 +422,7 @@ async function executeReplay(
|
||||
observed: newFailure.observed,
|
||||
seed: artifact.seed || 42,
|
||||
replayCommand: `apophis replay --artifact ${artifactPath}`,
|
||||
source: newFailure.source,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import type { Artifact, FailureRecord } from '../../core/types.js';
|
||||
const SUPPORTED_ARTIFACT_VERSION = 'apophis-artifact/1';
|
||||
|
||||
/** Current CLI version for compatibility checks */
|
||||
const CLI_VERSION = '2.0.0';
|
||||
const CLI_VERSION = '2.7.0';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
import type { CliContext } from '../../core/context.js'
|
||||
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
|
||||
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
||||
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
|
||||
|
||||
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
||||
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
|
||||
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
||||
@@ -54,13 +54,12 @@ function isReplayCompatibleRoute(route: string): boolean {
|
||||
|
||||
export interface VerifyOptions {
|
||||
profile?: string
|
||||
generationProfile?: string
|
||||
routes?: string
|
||||
seed?: number
|
||||
changed?: boolean
|
||||
config?: string
|
||||
cwd?: string
|
||||
format?: 'human' | 'json' | 'ndjson'
|
||||
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary'
|
||||
quiet?: boolean
|
||||
verbose?: boolean
|
||||
artifactDir?: string
|
||||
@@ -118,27 +117,31 @@ function buildArtifact(
|
||||
}
|
||||
return {
|
||||
route,
|
||||
contract: f.contract,
|
||||
contract: f.formula || f.contract,
|
||||
expected: f.expected,
|
||||
observed: f.observed,
|
||||
seed: options.seed,
|
||||
replayCommand: `apophis replay --artifact ${f.artifactPath || '<artifact-path-unavailable>'}`,
|
||||
category: f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME,
|
||||
category: f.category ?? (f.observed ? classifyError(f.observed) : ErrorTaxonomy.RUNTIME),
|
||||
source: f.source,
|
||||
}
|
||||
})
|
||||
|
||||
if (runResult.noContractsFound) {
|
||||
warnings.push('No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.')
|
||||
if (runResult.discoveryWarnings && runResult.discoveryWarnings.length > 0) {
|
||||
warnings.push(...runResult.discoveryWarnings)
|
||||
}
|
||||
}
|
||||
if (runResult.noRoutesMatched) {
|
||||
warnings.push(`No routes matched the filter. Available routes: ${runResult.availableRoutes?.join(', ') || 'none'}`)
|
||||
if (runResult.discoveryWarnings && runResult.discoveryWarnings.length > 0) {
|
||||
warnings.push(...runResult.discoveryWarnings)
|
||||
}
|
||||
}
|
||||
if (runResult.notGitRepo) {
|
||||
warnings.push('--changed requires a git repository. Current directory is not inside a git repo.')
|
||||
}
|
||||
if (runResult.noRelevantChanges) {
|
||||
warnings.push('No relevant changes detected. Git shows no modified files that match any route.')
|
||||
}
|
||||
if (runResult.failures.length > 0) {
|
||||
const profileFlag = options.profile ? ` --profile ${options.profile}` : ''
|
||||
const routesFlag = options.routeFilters && options.routeFilters.length > 0
|
||||
@@ -150,7 +153,7 @@ function buildArtifact(
|
||||
|
||||
return {
|
||||
version: 'apophis-artifact/1',
|
||||
cliVersion: '2.0.0',
|
||||
cliVersion: '2.7.0',
|
||||
command: 'verify',
|
||||
mode: 'verify',
|
||||
cwd: options.cwd,
|
||||
@@ -165,6 +168,7 @@ function buildArtifact(
|
||||
total: runResult.total,
|
||||
passed: runResult.passedCount,
|
||||
failed: runResult.failed,
|
||||
skipped: runResult.skipped,
|
||||
},
|
||||
deterministicParams: {
|
||||
seed: options.seed,
|
||||
@@ -256,106 +260,6 @@ function formatHumanFailure(failure: FailureRecord, profile?: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Format human-readable output for verify results.
|
||||
*/
|
||||
function formatHumanOutput(
|
||||
runResult: VerifyRunResult,
|
||||
options: { profile?: string; seed: number; env: string; routeFilters?: string[] },
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
|
||||
if (runResult.notGitRepo) {
|
||||
lines.push(`--changed requires a git repository.`)
|
||||
lines.push(`Current directory is not inside a git repo.`)
|
||||
lines.push('')
|
||||
lines.push('Next:')
|
||||
lines.push(` Initialize git with \`git init\`, or run verify without --changed.`)
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
if (runResult.noRelevantChanges) {
|
||||
lines.push(`No relevant changes detected.`)
|
||||
lines.push(`Git shows no modified files that match any route.`)
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
if (runResult.noRoutesMatched) {
|
||||
lines.push(`No routes matched the filter.`)
|
||||
lines.push(`Filters applied: ${options.routeFilters?.join(', ') || 'none'}`)
|
||||
lines.push(`Available routes:`)
|
||||
for (const r of runResult.availableRoutes || []) {
|
||||
lines.push(` ${r}`)
|
||||
}
|
||||
lines.push('')
|
||||
lines.push('Next:')
|
||||
lines.push(` Adjust --routes filter or add routes to your app.`)
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
if (runResult.noContractsFound) {
|
||||
lines.push('No behavioral contracts found.')
|
||||
lines.push('')
|
||||
lines.push('APOPHIS discovered routes, but none have behavioral contracts.')
|
||||
lines.push('Schema-only routes (with response schemas) are not enough.')
|
||||
lines.push('You must add x-ensures or x-requires clauses that check behavior.')
|
||||
lines.push('')
|
||||
lines.push('Example — add this to your route schema:')
|
||||
lines.push(' "x-ensures": [')
|
||||
lines.push(' "response_code(GET /users/{response_body(this).id}) == 200"')
|
||||
lines.push(' ]')
|
||||
lines.push('')
|
||||
lines.push('Next steps:')
|
||||
lines.push(' 1. Open your route file (e.g., app.js or src/routes/users.js)')
|
||||
lines.push(' 2. Find the route you want to test')
|
||||
lines.push(' 3. Add an "x-ensures" array inside the schema object')
|
||||
lines.push(' 4. Run: apophis verify --profile quick --routes "POST /users"')
|
||||
lines.push('')
|
||||
lines.push('For more examples, see docs/getting-started.md')
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
// Print failures using canonical format
|
||||
for (const failure of runResult.failures) {
|
||||
const failureRecord: FailureRecord = {
|
||||
route: failure.route,
|
||||
contract: failure.contract,
|
||||
expected: failure.expected,
|
||||
observed: failure.observed,
|
||||
seed: options.seed,
|
||||
replayCommand: `apophis replay --artifact ${failure.artifactPath || 'reports/apophis/failure-*.json'}`,
|
||||
}
|
||||
lines.push(formatHumanFailure(failureRecord, options.profile))
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
// Summary
|
||||
if (runResult.passed) {
|
||||
lines.push(`All ${runResult.total} contract(s) passed.`)
|
||||
} else {
|
||||
lines.push(`Failed: ${runResult.failed} of ${runResult.total} contract(s) failed.`)
|
||||
}
|
||||
lines.push(`Seed: ${options.seed}`)
|
||||
|
||||
// Replay command on failure
|
||||
if (!runResult.passed && runResult.failures.length > 0) {
|
||||
lines.push('')
|
||||
lines.push('Replay')
|
||||
lines.push(` apophis replay --artifact <path-to-artifact>`)
|
||||
lines.push('')
|
||||
lines.push('Determinism')
|
||||
lines.push(` This run used seed ${options.seed}.`)
|
||||
lines.push(` Same seed + same app state = same results.`)
|
||||
lines.push(` If results differ on re-run, the app has nondeterministic behavior.`)
|
||||
lines.push(` Stabilize: reset app state, mock external services, avoid time-dependent logic.`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main command handler
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -381,7 +285,6 @@ export async function verifyCommand(
|
||||
): Promise<CommandResult> {
|
||||
const {
|
||||
profile,
|
||||
generationProfile,
|
||||
routes: routesFlag,
|
||||
seed: explicitSeed,
|
||||
changed,
|
||||
@@ -412,7 +315,6 @@ export async function verifyCommand(
|
||||
}
|
||||
|
||||
const config = loadResult.config
|
||||
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
|
||||
|
||||
// 2a. Resolve profile — if explicitly requested but missing, list available ones
|
||||
if (profile && !config.profiles?.[profile]) {
|
||||
@@ -457,10 +359,12 @@ export async function verifyCommand(
|
||||
|
||||
// 5. Load the Fastify app
|
||||
let fastify: unknown
|
||||
let appEntrypoint: string | undefined
|
||||
try {
|
||||
const { loadApp } = await import('../../core/app-loader.js')
|
||||
const loaded = await loadApp(workingDir)
|
||||
fastify = loaded.fastify
|
||||
appEntrypoint = loaded.entrypoint
|
||||
if (fastify && typeof (fastify as any).ready === 'function') {
|
||||
await (fastify as any).ready()
|
||||
}
|
||||
@@ -468,7 +372,7 @@ export async function verifyCommand(
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: `No Fastify app found. Ensure app.js exports a Fastify instance.\n\nError: ${errorMessage}\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
||||
message: `Failed to load Fastify app.\n\nError: ${errorMessage}\n\nApp file candidates (searched in order):\n app.js, app.ts, server.js, server.ts, index.js, index.ts\n src/app.js, src/app.ts, src/server.js, src/server.ts, src/index.js, src/index.ts\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -476,13 +380,16 @@ export async function verifyCommand(
|
||||
const runResult = await runVerify({
|
||||
fastify: fastify as any,
|
||||
seed,
|
||||
generationProfile: resolvedGenerationProfile,
|
||||
timeout: typeof config.presets?.[loadResult.presetName || '']?.timeout === 'number'
|
||||
? (config.presets[loadResult.presetName || ''] as { timeout?: number }).timeout
|
||||
: undefined,
|
||||
routeFilters,
|
||||
changed,
|
||||
profileRoutes: config.profiles?.[profile || '']?.routes,
|
||||
runs: typeof config.presets?.[loadResult.presetName || '']?.runs === 'number'
|
||||
? (config.presets[loadResult.presetName || ''] as { runs?: number }).runs
|
||||
: undefined,
|
||||
pluginContractRegistry: (fastify as any)?._apophisPluginContractRegistry,
|
||||
})
|
||||
|
||||
// 7. Build artifact
|
||||
@@ -496,6 +403,15 @@ export async function verifyCommand(
|
||||
routeFilters,
|
||||
})
|
||||
|
||||
// Drain plugin contract registry warnings if app registered APOPHIS
|
||||
const pcr = (fastify as any)?._apophisPluginContractRegistry
|
||||
if (pcr?.drainWarnings) {
|
||||
const pcrWarnings = pcr.drainWarnings()
|
||||
if (pcrWarnings.length > 0) {
|
||||
artifact.warnings.push(...pcrWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
// 8. Write artifact if configured or on failure
|
||||
const shouldEmitArtifact = Boolean(artifactDir || config.artifactDir || !runResult.passed)
|
||||
await emitArtifact(artifact, {
|
||||
@@ -552,12 +468,7 @@ export async function verifyCommand(
|
||||
message: `Config validation failed: ${message}`,
|
||||
}
|
||||
}
|
||||
if (error instanceof GenerationProfileResolutionError) {
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
exitCode: INTERNAL_ERROR,
|
||||
message: `Internal error in verify command: ${message}`,
|
||||
@@ -579,7 +490,6 @@ export async function handleVerify(
|
||||
): Promise<number> {
|
||||
const options: VerifyOptions = {
|
||||
profile: ctx.options.profile || undefined,
|
||||
generationProfile: ctx.options.generationProfile,
|
||||
routes: undefined,
|
||||
seed: undefined,
|
||||
changed: false,
|
||||
@@ -611,11 +521,6 @@ export async function handleVerify(
|
||||
options.changed = true
|
||||
}
|
||||
|
||||
const generationProfileIdx = args.indexOf('--generation-profile')
|
||||
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
|
||||
options.generationProfile = args[generationProfileIdx + 1]
|
||||
}
|
||||
|
||||
const workspaceMode = args.includes('--workspace')
|
||||
|
||||
if (workspaceMode) {
|
||||
@@ -742,54 +647,27 @@ export async function handleVerify(
|
||||
const format = options.format || ctx.options.format || 'human'
|
||||
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
|
||||
|
||||
if (!ctx.options.quiet) {
|
||||
// Machine-readable formats always emit regardless of --quiet
|
||||
if (machineMode) {
|
||||
if (format === 'json') {
|
||||
if (result.artifact) {
|
||||
console.log(renderJsonArtifact(result.artifact))
|
||||
} else {
|
||||
console.log(renderJson({
|
||||
exitCode: result.exitCode,
|
||||
message: result.message,
|
||||
warnings: result.warnings,
|
||||
}))
|
||||
}
|
||||
console.log(result.artifact ? renderJsonArtifact(result.artifact) : renderJson({ exitCode: result.exitCode, message: result.message, warnings: result.warnings }))
|
||||
} else if (format === 'json-summary') {
|
||||
if (result.artifact) {
|
||||
console.log(renderJsonSummaryArtifact(result.artifact))
|
||||
} else {
|
||||
console.log(renderJson({
|
||||
exitCode: result.exitCode,
|
||||
message: result.message,
|
||||
warnings: result.warnings,
|
||||
}))
|
||||
}
|
||||
console.log(result.artifact ? renderJsonSummaryArtifact(result.artifact) : renderJson({ exitCode: result.exitCode, message: result.message, warnings: result.warnings }))
|
||||
} else if (format === 'ndjson') {
|
||||
if (result.artifact) {
|
||||
renderNdjsonArtifact(result.artifact)
|
||||
} else {
|
||||
process.stdout.write(JSON.stringify({
|
||||
type: 'run.completed',
|
||||
command: 'verify',
|
||||
exitCode: result.exitCode,
|
||||
message: result.message,
|
||||
warnings: result.warnings,
|
||||
}) + '\n')
|
||||
process.stdout.write(JSON.stringify({ type: 'run.completed', command: 'verify', exitCode: result.exitCode, message: result.message, warnings: result.warnings }) + '\n')
|
||||
}
|
||||
} else if (format === 'ndjson-summary') {
|
||||
if (result.artifact) {
|
||||
renderNdjsonSummaryArtifact(result.artifact)
|
||||
} else {
|
||||
process.stdout.write(JSON.stringify({
|
||||
type: 'run.completed',
|
||||
command: 'verify',
|
||||
exitCode: result.exitCode,
|
||||
message: result.message,
|
||||
warnings: result.warnings,
|
||||
}) + '\n')
|
||||
process.stdout.write(JSON.stringify({ type: 'run.completed', command: 'verify', exitCode: result.exitCode, message: result.message, warnings: result.warnings }) + '\n')
|
||||
}
|
||||
} else if (result.message) {
|
||||
console.log(result.message)
|
||||
}
|
||||
} else if (!ctx.options.quiet && result.message) {
|
||||
console.log(result.message)
|
||||
}
|
||||
|
||||
// Print warnings in human mode only
|
||||
|
||||
@@ -20,8 +20,13 @@ import { executeHttp } from '../../../infrastructure/http-executor.js'
|
||||
import { parse } from '../../../formula/parser.js'
|
||||
import { evaluateAsync } from '../../../formula/evaluator.js'
|
||||
import { createOperationResolver } from '../../../formula/runtime.js'
|
||||
import { buildRequest } from '../../../domain/request-builder.js'
|
||||
import { convertSchema } from '../../../domain/schema-to-arbitrary.js'
|
||||
import * as fc from 'fast-check'
|
||||
import { resolveRuns } from '../../../types.js'
|
||||
import type { EvalContext, RouteContract, FastifyInjectInstance } from '../../../types.js'
|
||||
import type { RouteResult } from '../../core/types.js'
|
||||
import type { PluginContractRegistry } from '../../../domain/plugin-contracts.js'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -33,6 +38,10 @@ export interface VerifyFailure {
|
||||
expected: string
|
||||
observed: string
|
||||
artifactPath?: string
|
||||
formula?: string
|
||||
category?: string
|
||||
/** Source of the contract: 'route' or 'plugin:name' */
|
||||
source?: string
|
||||
}
|
||||
|
||||
export interface VerifyRunResult {
|
||||
@@ -40,24 +49,27 @@ export interface VerifyRunResult {
|
||||
total: number
|
||||
passedCount: number
|
||||
failed: number
|
||||
skipped: number
|
||||
failures: VerifyFailure[]
|
||||
durationMs: number
|
||||
noRoutesMatched: boolean
|
||||
noContractsFound: boolean
|
||||
notGitRepo?: boolean
|
||||
noRelevantChanges?: boolean
|
||||
availableRoutes?: string[]
|
||||
artifactPaths: string[]
|
||||
discoveryWarnings?: string[]
|
||||
runs: number
|
||||
}
|
||||
|
||||
export interface VerifyRunnerDeps {
|
||||
fastify: FastifyInjectInstance
|
||||
seed: number
|
||||
generationProfile?: 'quick' | 'standard' | 'thorough'
|
||||
timeout?: number
|
||||
routeFilters?: string[]
|
||||
changed?: boolean
|
||||
profileRoutes?: string[]
|
||||
runs?: number
|
||||
pluginContractRegistry?: PluginContractRegistry
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -95,26 +107,22 @@ export async function discoverSpecificRoutes(
|
||||
|
||||
// For exact routes (no wildcards), check if route exists
|
||||
if (!pattern.includes('*') && !pattern.includes('?')) {
|
||||
try {
|
||||
if (fastify.hasRoute({ url: path, method })) {
|
||||
const key = `${method} ${path}`
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key)
|
||||
routes.push({
|
||||
method: method as RouteContract['method'],
|
||||
path,
|
||||
category: 'observer',
|
||||
schema: {},
|
||||
requires: [],
|
||||
ensures: [],
|
||||
invariants: [],
|
||||
regexPatterns: {},
|
||||
validateRuntime: false,
|
||||
})
|
||||
}
|
||||
if (fastify.hasRoute({ url: path, method })) {
|
||||
const key = `${method} ${path}`
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key)
|
||||
routes.push({
|
||||
method: method as RouteContract['method'],
|
||||
path,
|
||||
category: 'observer',
|
||||
schema: {},
|
||||
requires: [],
|
||||
ensures: [],
|
||||
invariants: [],
|
||||
regexPatterns: {},
|
||||
validateRuntime: false,
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
// Route doesn't exist
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -153,7 +161,7 @@ function filterRoutesByPatterns(routes: RouteContract[], patterns: string[]): Ro
|
||||
/**
|
||||
* Check if cwd is inside a git repository.
|
||||
*/
|
||||
async function isGitRepo(cwd: string): Promise<boolean> {
|
||||
export async function isGitRepo(cwd: string): Promise<boolean> {
|
||||
try {
|
||||
const { execSync } = await import('node:child_process')
|
||||
execSync('git rev-parse --git-dir', { cwd, encoding: 'utf-8', stdio: 'pipe' })
|
||||
@@ -166,7 +174,7 @@ async function isGitRepo(cwd: string): Promise<boolean> {
|
||||
/**
|
||||
* Get git-modified files for --changed filtering.
|
||||
*/
|
||||
async function getGitChangedFiles(cwd: string): Promise<string[]> {
|
||||
export async function getGitChangedFiles(cwd: string): Promise<string[]> {
|
||||
try {
|
||||
const { execSync } = await import('node:child_process')
|
||||
const output = execSync('git diff --name-only HEAD', { cwd, encoding: 'utf-8' })
|
||||
@@ -177,23 +185,29 @@ async function getGitChangedFiles(cwd: string): Promise<string[]> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter routes to only those modified in git.
|
||||
* Filter routes to only those whose source file was modified in git.
|
||||
* Uses sourceFile captured from stack traces during route registration.
|
||||
* Falls back to path-segment heuristic when sourceFile is unavailable.
|
||||
*/
|
||||
async function filterChangedRoutes(
|
||||
export async function filterChangedRoutes(
|
||||
routes: RouteContract[],
|
||||
cwd: string,
|
||||
): Promise<RouteContract[]> {
|
||||
const changedFiles = await getGitChangedFiles(cwd)
|
||||
|
||||
// Map route paths to potential file paths (heuristic)
|
||||
|
||||
return routes.filter(route => {
|
||||
// Prefer precise source file match
|
||||
if (route.sourceFile) {
|
||||
return changedFiles.some(file =>
|
||||
route.sourceFile!.endsWith(file) || file.endsWith(route.sourceFile!)
|
||||
)
|
||||
}
|
||||
// Fallback: heuristic path-segment match
|
||||
const routePath = route.path
|
||||
// Check if any changed file might contain this route
|
||||
return changedFiles.some(file => {
|
||||
// Simple heuristic: check if route path segments appear in file path
|
||||
const segments = routePath.split('/').filter(Boolean)
|
||||
return segments.some(segment => file.includes(segment))
|
||||
})
|
||||
const segments = routePath.split('/').filter(Boolean)
|
||||
return changedFiles.some(file =>
|
||||
segments.some(segment => file.includes(segment))
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -204,28 +218,58 @@ async function filterChangedRoutes(
|
||||
/**
|
||||
* Build a request for a route.
|
||||
*/
|
||||
function buildRouteRequest(route: RouteContract): {
|
||||
function buildRouteRequest(
|
||||
route: RouteContract,
|
||||
seed?: number,
|
||||
runIndex?: number,
|
||||
variant?: { name: string; headers?: Record<string, string> },
|
||||
): {
|
||||
method: string
|
||||
url: string
|
||||
body?: unknown
|
||||
query?: Record<string, string>
|
||||
headers: Record<string, string>
|
||||
} {
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
let generatedData: Record<string, unknown> = {}
|
||||
|
||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||
if (bodySchema && seed !== undefined) {
|
||||
try {
|
||||
const bodyArb = convertSchema(bodySchema, { context: 'request' })
|
||||
const bodySeed = seed + (runIndex ?? 0) * 31
|
||||
const samples = fc.sample(bodyArb, { numRuns: 1, seed: bodySeed })
|
||||
const bodySample = samples[0]
|
||||
if (bodySample !== null && typeof bodySample === 'object') {
|
||||
generatedData = bodySample as Record<string, unknown>
|
||||
}
|
||||
} catch {
|
||||
// fall through to example-based generation
|
||||
}
|
||||
}
|
||||
|
||||
// Build body from schema if available
|
||||
let body: unknown = undefined
|
||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||
if (bodySchema && route.method === 'POST') {
|
||||
body = buildExampleBody(bodySchema)
|
||||
if (!generatedData || Object.keys(generatedData).length === 0) {
|
||||
generatedData = buildExampleBody(bodySchema ?? {}) as Record<string, unknown> ?? {}
|
||||
}
|
||||
|
||||
const request = buildRequest(
|
||||
route,
|
||||
generatedData,
|
||||
variant?.headers ?? ({} as Record<string, string>),
|
||||
{ resources: new Map(), counters: new Map() },
|
||||
)
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
...request.headers,
|
||||
...(variant?.headers ?? {}),
|
||||
}
|
||||
|
||||
return {
|
||||
method: route.method,
|
||||
url: route.path,
|
||||
body,
|
||||
method: request.method,
|
||||
url: request.url,
|
||||
body: request.body,
|
||||
headers,
|
||||
query: request.query,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,18 +313,62 @@ function buildExampleValue(schema: Record<string, unknown>): unknown {
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single contract for a route.
|
||||
* Returns the evaluation context and any failure.
|
||||
*/
|
||||
function buildFailureDiagnostic(
|
||||
contract: string,
|
||||
evalCtx: EvalContext,
|
||||
evalError?: string,
|
||||
): Pick<VerifyFailure, 'expected' | 'observed' | 'formula' | 'category'> {
|
||||
if (evalError) {
|
||||
return { expected: 'true', observed: evalError, formula: contract, category: 'runtime' }
|
||||
}
|
||||
const status = evalCtx.response.statusCode
|
||||
|
||||
const statusMatch = contract.match(/^status:(\d+)$/i)
|
||||
if (statusMatch) {
|
||||
return { expected: `HTTP ${statusMatch[1]}`, observed: `HTTP ${status}`, formula: contract, category: 'runtime' }
|
||||
}
|
||||
|
||||
const fieldMatch = contract.match(/response_body\(this\)\.([\w.]+)\s*(!=|==)\s*null/i)
|
||||
if (fieldMatch) {
|
||||
const path = fieldMatch[1]!
|
||||
const negated = fieldMatch[2] === '!='
|
||||
const parts = path.split('.')
|
||||
let value: unknown = evalCtx.response.body
|
||||
for (const p of parts) {
|
||||
if (value != null && typeof value === 'object') {
|
||||
value = (value as Record<string, unknown>)[p]
|
||||
} else { value = undefined; break }
|
||||
}
|
||||
const actual = value === undefined ? 'undefined' : value === null ? 'null' : `"${String(value)}"`
|
||||
return {
|
||||
expected: negated ? `${path} != null` : `${path} == null`,
|
||||
observed: `${path} is ${actual}`,
|
||||
formula: contract, category: 'runtime',
|
||||
}
|
||||
}
|
||||
|
||||
const codeMatch = contract.match(/response_code\(this\)\s*(==|!=|<|>|<=|>=)\s*(\d+)/i)
|
||||
if (codeMatch) {
|
||||
return {
|
||||
expected: `response_code == ${codeMatch[2]}`,
|
||||
observed: `response_code is ${status}`,
|
||||
formula: contract, category: 'runtime',
|
||||
}
|
||||
}
|
||||
|
||||
return { expected: contract, observed: `false`, formula: contract, category: 'runtime' }
|
||||
}
|
||||
|
||||
async function executeContract(
|
||||
fastify: FastifyInjectInstance,
|
||||
route: RouteContract,
|
||||
contract: string,
|
||||
timeout?: number,
|
||||
variant?: { name: string; headers?: Record<string, string> },
|
||||
seed?: number,
|
||||
runIndex?: number,
|
||||
): Promise<{ ctx: EvalContext; failure?: VerifyFailure }> {
|
||||
const request = buildRouteRequest(route)
|
||||
const request = buildRouteRequest(route, seed, runIndex, variant)
|
||||
|
||||
// Merge variant headers if provided
|
||||
const headers = variant?.headers
|
||||
@@ -308,6 +396,7 @@ async function executeContract(
|
||||
const result = await evaluateAsync(parsed.ast, evalCtx)
|
||||
|
||||
if (!result.success || !result.value) {
|
||||
const diagnostic = buildFailureDiagnostic(contract, evalCtx, result.success ? undefined : result.error)
|
||||
return {
|
||||
ctx: evalCtx,
|
||||
failure: {
|
||||
@@ -315,14 +404,17 @@ async function executeContract(
|
||||
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
||||
: `${route.method} ${route.path}`,
|
||||
contract,
|
||||
expected: 'true',
|
||||
observed: result.success ? String(result.value) : result.error,
|
||||
expected: diagnostic.expected,
|
||||
observed: diagnostic.observed,
|
||||
formula: diagnostic.formula,
|
||||
category: diagnostic.category,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return { ctx: evalCtx }
|
||||
} catch (error) {
|
||||
const diagnostic = buildFailureDiagnostic(contract, evalCtx, error instanceof Error ? error.message : String(error))
|
||||
return {
|
||||
ctx: evalCtx,
|
||||
failure: {
|
||||
@@ -330,8 +422,10 @@ async function executeContract(
|
||||
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
||||
: `${route.method} ${route.path}`,
|
||||
contract,
|
||||
expected: 'true',
|
||||
observed: error instanceof Error ? error.message : String(error),
|
||||
expected: diagnostic.expected,
|
||||
observed: diagnostic.observed,
|
||||
formula: diagnostic.formula,
|
||||
category: diagnostic.category,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -393,14 +487,16 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
|
||||
total: 0,
|
||||
passedCount: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
failures: [],
|
||||
durationMs: Date.now() - started,
|
||||
noRoutesMatched: false,
|
||||
noContractsFound: false,
|
||||
availableRoutes,
|
||||
artifactPaths: [],
|
||||
notGitRepo: true,
|
||||
}
|
||||
artifactPaths: [],
|
||||
notGitRepo: true,
|
||||
runs: 0,
|
||||
}
|
||||
}
|
||||
routes = await filterChangedRoutes(routes, cwd)
|
||||
}
|
||||
@@ -412,12 +508,36 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
|
||||
total: 0,
|
||||
passedCount: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
failures: [],
|
||||
durationMs: Date.now() - started,
|
||||
noRoutesMatched: true,
|
||||
noContractsFound: false,
|
||||
availableRoutes,
|
||||
artifactPaths: [],
|
||||
runs: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// 3a. Compose plugin contracts into route contracts
|
||||
if (deps.pluginContractRegistry) {
|
||||
for (const route of routes) {
|
||||
const composed = deps.pluginContractRegistry.composeContracts(route)
|
||||
if (!route.formulaSources) route.formulaSources = {}
|
||||
for (const phase of Object.values(composed.phases)) {
|
||||
for (const req of phase.requires) {
|
||||
if (!route.requires.includes(req.formula)) {
|
||||
route.requires = [...route.requires, req.formula]
|
||||
route.formulaSources[req.formula] = req.source
|
||||
}
|
||||
}
|
||||
for (const ens of phase.ensures) {
|
||||
if (!route.ensures.includes(ens.formula)) {
|
||||
route.ensures = [...route.ensures, ens.formula]
|
||||
route.formulaSources[ens.formula] = ens.source
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -432,35 +552,131 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
|
||||
total: 0,
|
||||
passedCount: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
failures: [],
|
||||
durationMs: Date.now() - started,
|
||||
noRoutesMatched: false,
|
||||
noContractsFound: true,
|
||||
availableRoutes,
|
||||
artifactPaths: [],
|
||||
runs: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Execute contracts (with variant expansion)
|
||||
// Execute one HTTP request per (route, variant, runIndex), then validate
|
||||
// all contracts against the same request context.
|
||||
// Requires (preconditions) that fail → SKIP the ensures and count as skipped.
|
||||
// Ensures (postconditions) that fail → count as failures.
|
||||
const runConfig = resolveRuns(deps.runs)
|
||||
const runs = runConfig.contractRuns
|
||||
const failures: VerifyFailure[] = []
|
||||
let total = 0
|
||||
let passedCount = 0
|
||||
let skippedCount = 0
|
||||
|
||||
// Extract required headers from precondition formulas and auto-inject them
|
||||
// so that plugin contracts (e.g., x-tenant-id != null) are satisfied.
|
||||
function extractRequiredHeaders(formulas: string[]): Record<string, string> {
|
||||
const headers: Record<string, string> = {}
|
||||
for (const f of formulas) {
|
||||
const nullCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*!=\s*null/i)
|
||||
if (nullCheck) { headers[nullCheck[1]!] = 'test-value'; continue }
|
||||
const eqCheck = f.match(/request_headers\(this\)\.([\w-]+)\s*==\s*"([^"]+)"/i)
|
||||
if (eqCheck) { headers[eqCheck[1]!] = eqCheck[2]!; continue }
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
for (const route of routesWithContracts) {
|
||||
const contracts = [...route.requires, ...route.ensures]
|
||||
const defaultHeaders = extractRequiredHeaders(route.requires)
|
||||
const variants = route.variants && route.variants.length > 0
|
||||
? route.variants
|
||||
: [{ name: 'default' }]
|
||||
|
||||
for (const variant of variants) {
|
||||
for (const contract of contracts) {
|
||||
total++
|
||||
const result = await executeContract(fastify, route, contract, deps.timeout, variant)
|
||||
for (let runIndex = 0; runIndex < runs; runIndex++) {
|
||||
// Execute one HTTP request per run
|
||||
const request = buildRouteRequest(route, deps.seed, runIndex, variant)
|
||||
const headers = {
|
||||
...defaultHeaders,
|
||||
...request.headers,
|
||||
...(variant.headers ?? {}),
|
||||
}
|
||||
const ctx = await executeHttp(fastify, route, {
|
||||
method: request.method,
|
||||
url: request.url,
|
||||
body: request.body,
|
||||
headers,
|
||||
query: {},
|
||||
}, undefined, deps.timeout)
|
||||
const evalCtx: EvalContext = {
|
||||
...ctx,
|
||||
operationResolver: createOperationResolver(fastify, headers, ctx),
|
||||
}
|
||||
|
||||
if (result.failure) {
|
||||
failures.push(result.failure)
|
||||
// Evaluate requires first. If any fail, skip ensures for this run.
|
||||
let requiresPassed = true
|
||||
for (const formula of route.requires) {
|
||||
total++
|
||||
try {
|
||||
const parsed = parse(formula)
|
||||
const result = await evaluateAsync(parsed.ast, evalCtx)
|
||||
if (!result.success || !result.value) {
|
||||
requiresPassed = false
|
||||
skippedCount++
|
||||
} else {
|
||||
passedCount++
|
||||
}
|
||||
} catch {
|
||||
requiresPassed = false
|
||||
skippedCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Only evaluate ensures if all requires passed
|
||||
if (requiresPassed) {
|
||||
for (const formula of route.ensures) {
|
||||
total++
|
||||
try {
|
||||
const parsed = parse(formula)
|
||||
const result = await evaluateAsync(parsed.ast, evalCtx)
|
||||
if (!result.success || !result.value) {
|
||||
const diagnostic = buildFailureDiagnostic(formula, evalCtx, result.success ? undefined : result.error)
|
||||
failures.push({
|
||||
route: variant.name && variant.name !== 'default'
|
||||
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
||||
: `${route.method} ${route.path}`,
|
||||
contract: formula,
|
||||
expected: diagnostic.expected,
|
||||
observed: diagnostic.observed,
|
||||
formula: diagnostic.formula,
|
||||
category: diagnostic.category,
|
||||
source: route.formulaSources?.[formula],
|
||||
})
|
||||
} else {
|
||||
passedCount++
|
||||
}
|
||||
} catch (error) {
|
||||
const diagnostic = buildFailureDiagnostic(formula, evalCtx, error instanceof Error ? error.message : String(error))
|
||||
failures.push({
|
||||
route: variant.name && variant.name !== 'default'
|
||||
? `[variant:${variant.name}] ${route.method} ${route.path}`
|
||||
: `${route.method} ${route.path}`,
|
||||
contract: formula,
|
||||
expected: diagnostic.expected,
|
||||
observed: diagnostic.observed,
|
||||
formula: diagnostic.formula,
|
||||
category: diagnostic.category,
|
||||
source: route.formulaSources?.[formula],
|
||||
})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
passedCount++
|
||||
// Preconditions failed: ensures are skipped but not evaluated.
|
||||
// We still count each ensure as "total" and "skipped" for transparency.
|
||||
total += route.ensures.length
|
||||
skippedCount += route.ensures.length
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -480,11 +696,13 @@ export async function runVerify(deps: VerifyRunnerDeps): Promise<VerifyRunResult
|
||||
total,
|
||||
passedCount,
|
||||
failed: failures.length,
|
||||
skipped: skippedCount,
|
||||
failures: sortedFailures,
|
||||
durationMs,
|
||||
noRoutesMatched: false,
|
||||
noContractsFound: false,
|
||||
availableRoutes,
|
||||
artifactPaths: [],
|
||||
runs,
|
||||
}
|
||||
}
|
||||
|
||||
+157
-45
@@ -1,99 +1,211 @@
|
||||
/**
|
||||
* App loader utility for CLI commands.
|
||||
* Handles various app export patterns and module systems.
|
||||
*
|
||||
* Supports auto-detection of entrypoint files:
|
||||
* app.js, app.ts, server.js, server.ts, index.js, index.ts,
|
||||
* src/app.js, src/app.ts, src/server.js, src/server.ts,
|
||||
* src/index.js, src/index.ts
|
||||
*/
|
||||
|
||||
import { resolve } from 'node:path'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { resolve, extname } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
function isTsxAvailable(): boolean {
|
||||
return process.execArgv.some(arg => arg.includes('tsx') || arg.includes('ts-node')) ||
|
||||
process.argv[0]?.includes('tsx') ||
|
||||
!!process.env.TSX_TSCONFIG_PATH
|
||||
}
|
||||
|
||||
export const APP_CANDIDATES = [
|
||||
'app.js',
|
||||
'app.ts',
|
||||
'server.js',
|
||||
'server.ts',
|
||||
'index.js',
|
||||
'index.ts',
|
||||
'src/app.js',
|
||||
'src/app.ts',
|
||||
'src/server.js',
|
||||
'src/server.ts',
|
||||
'src/index.js',
|
||||
'src/index.ts',
|
||||
]
|
||||
|
||||
export interface LoadedApp {
|
||||
fastify: unknown
|
||||
source: 'default' | 'named' | 'commonjs'
|
||||
entrypoint: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a Fastify app from app.js in the given directory.
|
||||
* Supports:
|
||||
* Find the first existing app entrypoint file in the given directory.
|
||||
* Returns the resolved absolute path, or null if no candidate exists.
|
||||
*/
|
||||
export function findAppFile(cwd: string): string | null {
|
||||
for (const candidate of APP_CANDIDATES) {
|
||||
const fullPath = resolve(cwd, candidate)
|
||||
if (existsSync(fullPath)) {
|
||||
return fullPath
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a Fastify app from the given directory.
|
||||
*
|
||||
* Auto-detects the entrypoint by searching APP_CANDIDATES for the first
|
||||
* existing file. Supports:
|
||||
* - ESM default export: export default fastifyInstance
|
||||
* - ESM named export: export const createApp = () => fastifyInstance
|
||||
* - CommonJS: module.exports = fastifyInstance
|
||||
* - CommonJS named: exports.createApp = () => fastifyInstance
|
||||
*
|
||||
* If an explicit entrypoint is provided, it takes precedence over auto-detection.
|
||||
* For .ts entrypoints, a clear error message suggests installing tsx.
|
||||
*
|
||||
* After loading the app, this function ensures route discovery works even if the
|
||||
* APOPHIS plugin is not registered or was registered after routes.
|
||||
*/
|
||||
export async function loadApp(cwd: string): Promise<LoadedApp> {
|
||||
const appPath = resolve(cwd, 'app.js')
|
||||
const appUrl = pathToFileURL(appPath).href + '?t=' + Date.now()
|
||||
export async function loadApp(cwd: string, entrypoint?: string): Promise<LoadedApp> {
|
||||
const resolvedEntrypoint = entrypoint
|
||||
? resolve(cwd, entrypoint)
|
||||
: findAppFile(cwd)
|
||||
|
||||
if (!resolvedEntrypoint) {
|
||||
throw new AppLoadError(
|
||||
`No Fastify app entrypoint found. Searched for: ${APP_CANDIDATES.join(', ')}.\n` +
|
||||
'Create an app.js, server.js, or similar that exports a Fastify instance.',
|
||||
'not_found',
|
||||
)
|
||||
}
|
||||
|
||||
const ext = extname(resolvedEntrypoint)
|
||||
const isTypeScript = ext === '.ts'
|
||||
|
||||
if (isTypeScript && !isTsxAvailable()) {
|
||||
throw new AppLoadError(
|
||||
`TypeScript entrypoint ${resolvedEntrypoint} requires a TS loader.\n` +
|
||||
'Install tsx and run:\n' +
|
||||
' npm install -D tsx\n' +
|
||||
' npx tsx ./node_modules/.bin/apophis verify\n\n' +
|
||||
'Or convert your entrypoint to JavaScript.',
|
||||
'import_failed',
|
||||
)
|
||||
}
|
||||
|
||||
const appUrl = pathToFileURL(resolvedEntrypoint).href + '?t=' + Date.now()
|
||||
|
||||
let appModule: Record<string, unknown>
|
||||
try {
|
||||
appModule = await import(appUrl) as Record<string, unknown>
|
||||
} catch (err) {
|
||||
const errMessage = err instanceof Error ? err.message : String(err)
|
||||
if (isTypeScript) {
|
||||
throw new AppLoadError(
|
||||
`Cannot load TypeScript entrypoint ${resolvedEntrypoint}: ${errMessage}\n\n` +
|
||||
'TypeScript entrypoints require tsx. Install tsx and run:\n' +
|
||||
' npm install -D tsx\n' +
|
||||
' npx tsx ./node_modules/.bin/apophis verify\n\n' +
|
||||
'Or convert your entrypoint to JavaScript.',
|
||||
'import_failed',
|
||||
)
|
||||
}
|
||||
throw new AppLoadError(
|
||||
`Cannot load app.js: ${err instanceof Error ? err.message : String(err)}`,
|
||||
`Cannot load ${resolvedEntrypoint}: ${errMessage}`,
|
||||
'import_failed',
|
||||
)
|
||||
}
|
||||
|
||||
// Try default export first
|
||||
let fastify: unknown
|
||||
let source: 'default' | 'named' | 'commonjs' = 'default'
|
||||
|
||||
if (appModule.default && isFastifyInstance(appModule.default)) {
|
||||
return { fastify: appModule.default, source: 'default' }
|
||||
}
|
||||
fastify = appModule.default
|
||||
source = 'default'
|
||||
} else {
|
||||
let found = false
|
||||
for (const [key, value] of Object.entries(appModule)) {
|
||||
if (key === 'default') continue
|
||||
|
||||
// Try named exports that look like Fastify instances or factory functions
|
||||
for (const [key, value] of Object.entries(appModule)) {
|
||||
if (key === 'default') continue
|
||||
if (isFastifyInstance(value)) {
|
||||
fastify = value
|
||||
source = 'named'
|
||||
found = true
|
||||
break
|
||||
}
|
||||
|
||||
if (isFastifyInstance(value)) {
|
||||
return { fastify: value, source: 'named' }
|
||||
}
|
||||
|
||||
// Try calling factory functions
|
||||
if (typeof value === 'function' && !isClass(value)) {
|
||||
try {
|
||||
const result = await value()
|
||||
if (isFastifyInstance(result)) {
|
||||
return { fastify: result, source: 'named' }
|
||||
if (typeof value === 'function' && !isClass(value)) {
|
||||
try {
|
||||
const result = await value()
|
||||
if (isFastifyInstance(result)) {
|
||||
fastify = result
|
||||
source = 'named'
|
||||
found = true
|
||||
break
|
||||
}
|
||||
} catch {
|
||||
// Factory function failed, try next
|
||||
}
|
||||
} catch {
|
||||
// Factory function failed, try next
|
||||
}
|
||||
}
|
||||
if (!found && isFastifyInstance(appModule)) {
|
||||
fastify = appModule
|
||||
source = 'commonjs'
|
||||
found = true
|
||||
}
|
||||
if (!found) {
|
||||
throw new AppLoadError(
|
||||
`No Fastify instance found in ${resolvedEntrypoint}. ` +
|
||||
'Ensure the file exports a Fastify instance or a factory function.\n\n' +
|
||||
'Supported patterns:\n' +
|
||||
' export default app\n' +
|
||||
' export { app }\n' +
|
||||
' export const createApp = () => app\n' +
|
||||
' module.exports = app',
|
||||
'no_fastify',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// If module itself is a Fastify instance (CommonJS)
|
||||
if (isFastifyInstance(appModule)) {
|
||||
return { fastify: appModule, source: 'commonjs' }
|
||||
}
|
||||
await ensureRouteDiscovery(fastify as Record<string, unknown>)
|
||||
|
||||
throw new AppLoadError(
|
||||
'No Fastify instance found in app.js. Ensure app.js exports a Fastify instance or a factory function.',
|
||||
'no_fastify',
|
||||
)
|
||||
return { fastify, source, entrypoint: resolvedEntrypoint }
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a value looks like a Fastify instance.
|
||||
* Install a lightweight onRoute hook for route discovery.
|
||||
* This runs before avvio's boot phase, so routes registered
|
||||
* inside deferred plugins will be captured even if apophis
|
||||
* is not registered first.
|
||||
*/
|
||||
async function ensureRouteDiscovery(f: Record<string, unknown>): Promise<void> {
|
||||
if (typeof f.addHook !== 'function') return
|
||||
|
||||
try {
|
||||
const { installRouteDiscovery } = await import('../../infrastructure/discovery-hook.js')
|
||||
installRouteDiscovery(f as { addHook: Function })
|
||||
} catch {
|
||||
// discovery module not importable, skip auto-capture
|
||||
}
|
||||
}
|
||||
|
||||
function isFastifyInstance(value: unknown): boolean {
|
||||
return value !== null &&
|
||||
typeof value === 'object' &&
|
||||
typeof (value as Record<string, unknown>).ready === 'function'
|
||||
if (value === null || typeof value !== 'object') return false
|
||||
const obj = value as Record<string, unknown>
|
||||
return typeof obj.ready === 'function'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a function is a class constructor.
|
||||
*/
|
||||
function isClass(fn: unknown): boolean {
|
||||
return typeof fn === 'function' &&
|
||||
fn.toString().startsWith('class ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Error type for app loading failures.
|
||||
*/
|
||||
export class AppLoadError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly code: 'import_failed' | 'no_fastify',
|
||||
public readonly code: 'import_failed' | 'no_fastify' | 'not_found',
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'AppLoadError'
|
||||
|
||||
@@ -30,7 +30,7 @@ export interface Config {
|
||||
environments?: Record<string, EnvironmentPolicy>;
|
||||
profiles?: Record<string, ProfileDefinition>;
|
||||
presets?: Record<string, PresetDefinition>;
|
||||
generationProfiles?: Record<string, 'quick' | 'standard' | 'thorough' | { base: 'quick' | 'standard' | 'thorough' }>;
|
||||
scenarios?: ScenarioConfigDef[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
@@ -38,23 +38,67 @@ export interface EnvironmentPolicy {
|
||||
allowedModes?: ('verify' | 'observe' | 'qualify')[];
|
||||
blockQualify?: boolean;
|
||||
allowChaosOnProtected?: boolean;
|
||||
allowVerify?: boolean;
|
||||
allowObserve?: boolean;
|
||||
allowQualify?: boolean;
|
||||
allowChaos?: boolean;
|
||||
allowBlocking?: boolean;
|
||||
requireSink?: boolean;
|
||||
sinks?: Record<string, unknown>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface ProfileDefinition {
|
||||
name?: string;
|
||||
mode?: 'verify' | 'observe' | 'qualify';
|
||||
preset?: string;
|
||||
routes?: string[];
|
||||
seed?: number;
|
||||
artifactDir?: string;
|
||||
environment?: string;
|
||||
observe?: boolean;
|
||||
sampling?: number;
|
||||
blocking?: boolean;
|
||||
sinks?: Record<string, unknown>;
|
||||
features?: string[];
|
||||
[key: string]: unknown;
|
||||
chaos?: boolean;
|
||||
chaosStrategy?: 'one' | 'all' | 'sample' | 'routes';
|
||||
chaosSampleSize?: number;
|
||||
chaosSampleRoutes?: string[];
|
||||
}
|
||||
|
||||
export interface PresetDefinition {
|
||||
name?: string;
|
||||
mode?: 'verify' | 'observe' | 'qualify';
|
||||
routes?: string[];
|
||||
seed?: number;
|
||||
timeout?: number;
|
||||
parallel?: boolean;
|
||||
chaos?: boolean;
|
||||
observe?: boolean;
|
||||
runs?: number;
|
||||
sampling?: number;
|
||||
blocking?: boolean;
|
||||
sinks?: Record<string, unknown>;
|
||||
features?: string[];
|
||||
[key: string]: unknown;
|
||||
chaosStrategy?: 'one' | 'all' | 'sample' | 'routes';
|
||||
chaosSampleSize?: number;
|
||||
chaosSampleRoutes?: string[];
|
||||
}
|
||||
|
||||
export interface ScenarioConfigDef {
|
||||
name: string;
|
||||
steps: Array<{
|
||||
name: string;
|
||||
request: {
|
||||
method: string;
|
||||
url: string;
|
||||
body?: unknown;
|
||||
headers?: Record<string, string>;
|
||||
};
|
||||
expect: string[];
|
||||
capture?: Record<string, string>;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface LoadConfigOptions {
|
||||
@@ -107,16 +151,25 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
|
||||
optional: true,
|
||||
properties: {},
|
||||
},
|
||||
generationProfiles: {
|
||||
type: 'object',
|
||||
optional: true,
|
||||
properties: {},
|
||||
},
|
||||
packs: {
|
||||
type: 'array',
|
||||
optional: true,
|
||||
items: { type: 'string' },
|
||||
},
|
||||
metadata: {
|
||||
type: 'object',
|
||||
optional: true,
|
||||
},
|
||||
scenarios: {
|
||||
type: 'array',
|
||||
optional: true,
|
||||
items: { type: 'object' },
|
||||
},
|
||||
pluginContracts: {
|
||||
type: 'object',
|
||||
optional: true,
|
||||
properties: {},
|
||||
},
|
||||
};
|
||||
|
||||
// Schema for EnvironmentPolicy values (inside environments.<name>)
|
||||
@@ -146,26 +199,29 @@ const PROFILE_SCHEMA: Record<string, SchemaField> = {
|
||||
sampling: { type: 'number', optional: true },
|
||||
blocking: { type: 'boolean', optional: true },
|
||||
sinks: { type: 'object', optional: true },
|
||||
chaos: { type: 'object', optional: true },
|
||||
chaosStrategy: { type: 'string', optional: true, enumValues: ['one', 'all', 'sample', 'routes'] },
|
||||
chaosSampleSize: { type: 'number', optional: true, min: 1 },
|
||||
chaosSampleRoutes: { type: 'array', optional: true, items: { type: 'string' } },
|
||||
};
|
||||
|
||||
// Schema for PresetDefinition values (inside presets.<name>)
|
||||
const PRESET_SCHEMA: Record<string, SchemaField> = {
|
||||
name: { type: 'string', optional: false },
|
||||
depth: { type: 'string', optional: true, enumValues: ['quick', 'standard', 'deep'] },
|
||||
timeout: { type: 'number', optional: true, min: 0 },
|
||||
parallel: { type: 'boolean', optional: true },
|
||||
chaos: { type: 'boolean', optional: true },
|
||||
chaosStrategy: { type: 'string', optional: true, enumValues: ['one', 'all', 'sample', 'routes'] },
|
||||
chaosSampleSize: { type: 'number', optional: true, min: 1 },
|
||||
chaosSampleRoutes: { type: 'array', optional: true, items: { type: 'string' } },
|
||||
observe: { type: 'boolean', optional: true },
|
||||
features: { type: 'array', optional: true, items: { type: 'string' } },
|
||||
sampling: { type: 'number', optional: true },
|
||||
blocking: { type: 'boolean', optional: true },
|
||||
sinks: { type: 'object', optional: true },
|
||||
runs: { type: 'number', optional: true, min: 1 },
|
||||
};
|
||||
|
||||
const GENERATION_PROFILE_ALIAS_SCHEMA: Record<string, SchemaField> = {
|
||||
base: { type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config discovery
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -211,7 +267,7 @@ export function loadPackageJsonConfig(cwd: string): { config: Config | null; pat
|
||||
|
||||
/**
|
||||
* Load a config file by path.
|
||||
* Supports .js, .ts (via dynamic import, assumes tsx available), and .json.
|
||||
* Supports .js, .ts (via dynamic import when tsx is available), and .json.
|
||||
*/
|
||||
export async function loadConfigFile(configPath: string): Promise<Config> {
|
||||
if (configPath.endsWith('.json')) {
|
||||
@@ -219,6 +275,14 @@ export async function loadConfigFile(configPath: string): Promise<Config> {
|
||||
return JSON.parse(content) as Config;
|
||||
}
|
||||
|
||||
if (configPath.endsWith('.ts') && !process.execArgv.some(a => a.includes('tsx') || a.includes('ts-node'))) {
|
||||
throw new Error(
|
||||
`TypeScript config file ${configPath} requires a TS loader.\n` +
|
||||
'Convert to apophis.config.js or apophis.config.json, or run through tsx:\n' +
|
||||
' npx tsx ./node_modules/.bin/apophis verify'
|
||||
);
|
||||
}
|
||||
|
||||
// For .js and .ts, use dynamic import.
|
||||
// tsx handles .ts files in dev environments.
|
||||
const fileUrl = pathToFileURL(configPath).href;
|
||||
@@ -259,7 +323,6 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
|
||||
if (path === 'profiles') return PROFILE_SCHEMA;
|
||||
if (path === 'presets') return PRESET_SCHEMA;
|
||||
if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA;
|
||||
if (path === 'generationProfiles') return GENERATION_PROFILE_ALIAS_SCHEMA;
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -267,7 +330,16 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
|
||||
* Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar).
|
||||
*/
|
||||
function isInsideDynamicContainer(path: string): boolean {
|
||||
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.') || path.startsWith('generationProfiles.');
|
||||
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.');
|
||||
}
|
||||
|
||||
/**
|
||||
* User-managed extension keys that pass through schema validation.
|
||||
* APOPHIS will never interpret these fields; they are reserved for team metadata.
|
||||
* Keys starting with 'x-' are always allowed at any level.
|
||||
*/
|
||||
function isUserExtensionKey(key: string): boolean {
|
||||
return key.startsWith('x-')
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -379,18 +451,11 @@ export function validateConfigAgainstSchema(
|
||||
|
||||
// Handle dynamic containers: profiles, presets, environments
|
||||
// The keys are user-defined names; their values have specific schemas
|
||||
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments' || path === 'generationProfiles';
|
||||
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments';
|
||||
if (!fieldSchema && isDynamicContainer) {
|
||||
const childSchema = getDynamicContainerSchema(path);
|
||||
const fieldValue = obj[key];
|
||||
if (path === 'generationProfiles' && typeof fieldValue === 'string') {
|
||||
validateType(
|
||||
fieldValue,
|
||||
{ type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
|
||||
currentPath,
|
||||
key,
|
||||
);
|
||||
} else if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
|
||||
if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
|
||||
// Validate the dynamic container value against its specific schema
|
||||
validateConfigAgainstSchema(fieldValue, childSchema, currentPath);
|
||||
} else if (childSchema) {
|
||||
@@ -424,13 +489,14 @@ export function validateConfigAgainstSchema(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (isUserExtensionKey(key)) continue
|
||||
// Unknown key inside a profile/preset/environment object
|
||||
throw new ConfigValidationError(
|
||||
`Unknown config key at ${currentPath}`,
|
||||
currentPath,
|
||||
key,
|
||||
obj[key],
|
||||
`Valid keys for ${parentContainer} entries: ${Object.keys(childSchema || {}).join(', ')}.`,
|
||||
`Valid keys for ${parentContainer} entries: ${Object.keys(childSchema || {}).join(', ')}. Use x- prefixed keys for team metadata.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -438,12 +504,13 @@ export function validateConfigAgainstSchema(
|
||||
}
|
||||
|
||||
if (!fieldSchema) {
|
||||
if (isUserExtensionKey(key)) continue
|
||||
throw new ConfigValidationError(
|
||||
`Unknown config key at ${currentPath}`,
|
||||
currentPath,
|
||||
key,
|
||||
obj[key],
|
||||
`Valid top-level keys: ${Object.keys(CONFIG_SCHEMA).join(', ')}.`,
|
||||
`Valid top-level keys: ${Object.keys(CONFIG_SCHEMA).join(', ')}. Use the metadata field or x- prefixed keys for team data.`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -633,19 +700,6 @@ export function validateConfigSemantics(config: Config): void {
|
||||
);
|
||||
}
|
||||
}
|
||||
if (preset.depth !== undefined) {
|
||||
const validDepths = ['quick', 'standard', 'deep'];
|
||||
const depthValue = preset.depth;
|
||||
if (typeof depthValue === 'string' && !validDepths.includes(depthValue as string)) {
|
||||
throw new ConfigValidationError(
|
||||
`Preset "${presetName}" has invalid depth: "${depthValue}"`,
|
||||
`presets.${presetName}.depth`,
|
||||
'depth',
|
||||
depthValue,
|
||||
`Must be one of: ${validDepths.join(', ')}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
|
||||
const packageManager = detectPackageManager(cwd);
|
||||
|
||||
// Normalize options
|
||||
const format = options.format === 'json' || options.format === 'ndjson'
|
||||
const format = options.format === 'json' || options.format === 'ndjson' || options.format === 'json-summary' || options.format === 'ndjson-summary'
|
||||
? options.format
|
||||
: 'human';
|
||||
|
||||
@@ -101,10 +101,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
|
||||
? options.color
|
||||
: 'auto';
|
||||
|
||||
const generationProfile = typeof options.generationProfile === 'string'
|
||||
? options.generationProfile
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
cwd,
|
||||
env: {
|
||||
@@ -119,7 +115,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
|
||||
options: {
|
||||
config: typeof options.config === 'string' ? options.config : undefined,
|
||||
profile: typeof options.profile === 'string' ? options.profile : undefined,
|
||||
generationProfile,
|
||||
format,
|
||||
color,
|
||||
quiet: options.quiet === true,
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
import type { Config } from './config-loader.js'
|
||||
|
||||
export type ResolvedGenerationProfile = 'quick' | 'standard' | 'thorough'
|
||||
|
||||
export class GenerationProfileResolutionError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'GenerationProfileResolutionError'
|
||||
}
|
||||
}
|
||||
|
||||
function isBuiltInProfile(value: string): value is ResolvedGenerationProfile {
|
||||
return value === 'quick' || value === 'standard' || value === 'thorough'
|
||||
}
|
||||
|
||||
export function resolveGenerationProfileOverride(
|
||||
rawProfile: string | undefined,
|
||||
config: Config,
|
||||
): ResolvedGenerationProfile | undefined {
|
||||
if (!rawProfile) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
if (isBuiltInProfile(rawProfile)) {
|
||||
return rawProfile
|
||||
}
|
||||
|
||||
const aliases = config.generationProfiles
|
||||
if (!aliases) {
|
||||
throw new GenerationProfileResolutionError(
|
||||
`Unknown generation profile "${rawProfile}". Use one of: quick, standard, thorough, or define an alias in config.generationProfiles.`,
|
||||
)
|
||||
}
|
||||
|
||||
const alias = aliases[rawProfile]
|
||||
if (!alias) {
|
||||
const available = Object.keys(aliases).join(', ') || 'none'
|
||||
throw new GenerationProfileResolutionError(
|
||||
`Unknown generation profile "${rawProfile}". Built-ins: quick, standard, thorough. Config aliases: ${available}.`,
|
||||
)
|
||||
}
|
||||
|
||||
const target = typeof alias === 'string' ? alias : alias.base
|
||||
if (!isBuiltInProfile(target)) {
|
||||
throw new GenerationProfileResolutionError(
|
||||
`Invalid generation profile alias "${rawProfile}". Alias must resolve to quick, standard, or thorough.`,
|
||||
)
|
||||
}
|
||||
|
||||
return target
|
||||
}
|
||||
+24
-46
@@ -2,7 +2,7 @@ import { cac } from 'cac';
|
||||
import pc from 'picocolors';
|
||||
import { createContext, type CliContext } from './context.js';
|
||||
|
||||
const CLI_VERSION = '2.0.0';
|
||||
const CLI_VERSION = '2.7.0';
|
||||
|
||||
const HELP_HEADER = `
|
||||
${pc.bold('apophis')} — Contract-driven API testing for Fastify
|
||||
@@ -13,7 +13,6 @@ const HELP_HEADER = `
|
||||
${pc.dim('Commands:')}
|
||||
init Scaffold config, scripts, and example usage
|
||||
verify Run deterministic contract verification
|
||||
observe Validate runtime observe configuration and reporting setup
|
||||
qualify Run scenario, stateful, protocol, or chaos-driven qualification
|
||||
replay Replay a failure using seed and stored trace
|
||||
doctor Validate config, environment safety, docs/example correctness
|
||||
@@ -22,9 +21,8 @@ const HELP_HEADER = `
|
||||
${pc.dim('Global Options:')}
|
||||
--config <path> Config file path
|
||||
--profile <name> Profile name from config
|
||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
||||
--cwd <path> Working directory override
|
||||
--format <mode> Output format: human | json | ndjson (default: human)
|
||||
--format <mode> Output format: human | json | ndjson | json-summary | ndjson-summary (default: human)
|
||||
--color <mode> Color mode: auto | always | never (default: auto)
|
||||
--quiet Suppress non-error output
|
||||
--verbose Enable verbose logging
|
||||
@@ -38,7 +36,6 @@ const HELP_HEADER = `
|
||||
${pc.dim('Examples:')}
|
||||
apophis init --preset safe-ci
|
||||
apophis verify --profile quick --routes "POST /users"
|
||||
apophis observe --profile staging-observe --check-config
|
||||
apophis qualify --profile oauth-nightly --seed 42
|
||||
apophis replay --artifact reports/apophis/failure-*.json
|
||||
apophis doctor
|
||||
@@ -55,9 +52,9 @@ function getCommandHelp(command: string): string {
|
||||
apophis init [options]
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--preset <name> Preset name (e.g. safe-ci, full)
|
||||
--force Overwrite existing files
|
||||
--noninteractive Skip all prompts, require explicit flags
|
||||
--preset, -p <name> Preset name (e.g. safe-ci, full)
|
||||
--force, -f Overwrite existing files
|
||||
--noninteractive Skip all prompts, require explicit flags
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
apophis init --preset safe-ci
|
||||
@@ -71,29 +68,16 @@ function getCommandHelp(command: string): string {
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--profile <name> Profile name from config
|
||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
||||
--routes <filter> Route filter pattern
|
||||
--seed <number> Deterministic seed
|
||||
--changed Filter to git-modified routes
|
||||
--workspace Run across all monorepo packages
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
apophis verify --profile quick
|
||||
apophis verify --routes "POST /users" --seed 42
|
||||
apophis verify --changed
|
||||
`,
|
||||
observe: `
|
||||
${pc.bold('apophis observe')} — Validate runtime observe configuration and reporting setup
|
||||
|
||||
${pc.dim('Usage:')}
|
||||
apophis observe [options]
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--profile <name> Profile name from config
|
||||
--check-config Only validate, do not activate
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
apophis observe --profile staging-observe
|
||||
apophis observe --check-config
|
||||
apophis verify --workspace
|
||||
`,
|
||||
qualify: `
|
||||
${pc.bold('apophis qualify')} — Run scenario, stateful, protocol, or chaos-driven qualification
|
||||
@@ -103,11 +87,12 @@ function getCommandHelp(command: string): string {
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--profile <name> Profile name from config
|
||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
||||
--seed <number> Deterministic seed
|
||||
--changed Filter to git-modified routes
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
apophis qualify --profile oauth-nightly --seed 42
|
||||
apophis qualify --changed
|
||||
`,
|
||||
replay: `
|
||||
${pc.bold('apophis replay')} — Replay a failure using seed and stored trace
|
||||
@@ -117,9 +102,11 @@ function getCommandHelp(command: string): string {
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--artifact <path> Path to failure artifact
|
||||
--route <filter> Select failure by route pattern
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
apophis replay --artifact reports/apophis/failure-*.json
|
||||
apophis replay --artifact failure-1.json --route "POST /users"
|
||||
`,
|
||||
doctor: `
|
||||
${pc.bold('apophis doctor')} — Validate config, environment safety, docs/example correctness
|
||||
@@ -130,11 +117,12 @@ function getCommandHelp(command: string): string {
|
||||
${pc.dim('Options:')}
|
||||
--mode <mode> Focus checks on a mode: verify | observe | qualify
|
||||
--strict Treat warnings as failures
|
||||
--workspace Run across all monorepo packages
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
apophis doctor
|
||||
apophis doctor --mode verify
|
||||
apophis doctor --strict
|
||||
apophis doctor --mode observe
|
||||
apophis doctor --strict --workspace
|
||||
`,
|
||||
migrate: `
|
||||
${pc.bold('apophis migrate')} — Check and rewrite deprecated config or API usage
|
||||
@@ -170,18 +158,18 @@ function printInternalError(error: unknown): void {
|
||||
console.error();
|
||||
}
|
||||
|
||||
function resolveRequestedFormat(argv: string[]): 'human' | 'json' | 'ndjson' {
|
||||
function resolveRequestedFormat(argv: string[]): 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary' {
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (!arg) continue;
|
||||
if (arg === '--format' && argv[i + 1]) {
|
||||
const value = argv[i + 1];
|
||||
if (value === 'json' || value === 'ndjson') return value;
|
||||
if (value === 'json' || value === 'ndjson' || value === 'json-summary' || value === 'ndjson-summary') return value as 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
|
||||
return 'human';
|
||||
}
|
||||
if (arg.startsWith('--format=')) {
|
||||
const value = arg.slice('--format='.length);
|
||||
if (value === 'json' || value === 'ndjson') return value;
|
||||
if (value === 'json' || value === 'ndjson' || value === 'json-summary' || value === 'ndjson-summary') return value as 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
|
||||
return 'human';
|
||||
}
|
||||
}
|
||||
@@ -199,13 +187,12 @@ function writeMachineRecord(
|
||||
process.stdout.write(JSON.stringify(payload) + '\n');
|
||||
}
|
||||
|
||||
type CommandName = 'init' | 'verify' | 'observe' | 'qualify' | 'replay' | 'doctor' | 'migrate';
|
||||
type CommandName = 'init' | 'verify' | 'qualify' | 'replay' | 'doctor' | 'migrate';
|
||||
type CommandHandler = (args: string[], ctx: CliContext) => Promise<number>;
|
||||
|
||||
const commandLoaders: Record<CommandName, () => Promise<CommandHandler>> = {
|
||||
init: async () => (await import('../commands/init/index.js')).handleInit,
|
||||
verify: async () => (await import('../commands/verify/index.js')).handleVerify,
|
||||
observe: async () => (await import('../commands/observe/index.js')).handleObserve,
|
||||
qualify: async () => (await import('../commands/qualify/index.js')).handleQualify,
|
||||
replay: async () => (await import('../commands/replay/index.js')).handleReplay,
|
||||
doctor: async () => (await import('../commands/doctor/index.js')).handleDoctor,
|
||||
@@ -225,9 +212,8 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
// Global flags
|
||||
cli.option('--config <path>', 'Config file path');
|
||||
cli.option('--profile <name>', 'Profile name from config');
|
||||
cli.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
||||
cli.option('--cwd <path>', 'Working directory override');
|
||||
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
|
||||
cli.option('--format <mode>', 'Output format: human | json | ndjson | json-summary | ndjson-summary', { default: 'human' });
|
||||
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
|
||||
cli.option('--quiet', 'Suppress non-error output');
|
||||
cli.option('--verbose', 'Enable verbose logging');
|
||||
@@ -251,7 +237,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
const commands = [
|
||||
'init',
|
||||
'verify',
|
||||
'observe',
|
||||
'qualify',
|
||||
'replay',
|
||||
'doctor',
|
||||
@@ -270,19 +255,14 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
break;
|
||||
case 'verify':
|
||||
cmd.option('--profile <name>', 'Profile name from config');
|
||||
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
||||
cmd.option('--routes <filter>', 'Route filter pattern');
|
||||
cmd.option('--seed <number>', 'Deterministic seed');
|
||||
cmd.option('--changed', 'Filter to git-modified routes');
|
||||
break;
|
||||
case 'observe':
|
||||
cmd.option('--profile <name>', 'Profile name from config');
|
||||
cmd.option('--check-config', 'Only validate, do not activate');
|
||||
break;
|
||||
case 'qualify':
|
||||
cmd.option('--profile <name>', 'Profile name from config');
|
||||
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
||||
cmd.option('--seed <number>', 'Deterministic seed');
|
||||
cmd.option('--changed', 'Filter to git-modified routes');
|
||||
break;
|
||||
case 'replay':
|
||||
cmd.option('--artifact <path>', 'Path to failure artifact');
|
||||
@@ -373,17 +353,15 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
// Handle unknown flags
|
||||
const knownGlobalFlags = new Set([
|
||||
'--config', '--profile', '--cwd', '--format', '--color',
|
||||
'--generation-profile',
|
||||
'--quiet', '--verbose', '--artifact-dir', '--workspace',
|
||||
'-v', '--version', '-h', '--help',
|
||||
]);
|
||||
|
||||
const commandSpecificFlags: Record<string, Set<string>> = {
|
||||
init: new Set(['--preset', '--force', '--noninteractive']),
|
||||
verify: new Set(['--profile', '--generation-profile', '--routes', '--seed', '--changed', '--workspace']),
|
||||
observe: new Set(['--profile', '--check-config', '--workspace']),
|
||||
qualify: new Set(['--profile', '--generation-profile', '--seed', '--workspace']),
|
||||
replay: new Set(['--artifact']),
|
||||
init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']),
|
||||
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
|
||||
qualify: new Set(['--profile', '--seed', '--changed']),
|
||||
replay: new Set(['--artifact', '--route']),
|
||||
doctor: new Set(['--mode', '--strict', '--workspace']),
|
||||
migrate: new Set(['--check', '--dry-run', '--write']),
|
||||
};
|
||||
|
||||
@@ -145,13 +145,9 @@ export class PolicyEngine {
|
||||
}
|
||||
warnings.push(...comboCheck.warnings);
|
||||
|
||||
// 4. Check observe-specific safety
|
||||
// 4. Observe-specific safety
|
||||
if (this.mode === 'observe') {
|
||||
const observeCheck = this.checkObserveSafety();
|
||||
if (!observeCheck.allowed) {
|
||||
errors.push(...observeCheck.errors);
|
||||
}
|
||||
warnings.push(...observeCheck.warnings);
|
||||
warnings.push(...this.checkObserveSafety().warnings);
|
||||
}
|
||||
|
||||
// 5. Check qualify-specific safety
|
||||
|
||||
+31
-5
@@ -29,7 +29,6 @@ export interface CliContext {
|
||||
options: {
|
||||
config: string | undefined;
|
||||
profile: string | undefined;
|
||||
generationProfile?: string;
|
||||
format: OutputFormat;
|
||||
color: ColorMode;
|
||||
quiet: boolean;
|
||||
@@ -96,6 +95,7 @@ export interface EnvironmentPolicy {
|
||||
allowChaos?: boolean;
|
||||
allowBlocking?: boolean;
|
||||
requireSink?: boolean;
|
||||
sinks?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -116,13 +116,22 @@ export interface EnvironmentPolicy {
|
||||
* }
|
||||
*/
|
||||
export interface ProfileDefinition {
|
||||
name: string;
|
||||
name?: string;
|
||||
mode?: ApophisMode;
|
||||
preset?: string;
|
||||
routes?: string[];
|
||||
seed?: number;
|
||||
artifactDir?: string;
|
||||
environment?: string;
|
||||
observe?: boolean;
|
||||
sampling?: number;
|
||||
blocking?: boolean;
|
||||
sinks?: Record<string, unknown>;
|
||||
features?: string[];
|
||||
chaos?: boolean;
|
||||
chaosStrategy?: 'one' | 'all' | 'sample' | 'routes';
|
||||
chaosSampleSize?: number;
|
||||
chaosSampleRoutes?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -132,7 +141,6 @@ export interface ProfileDefinition {
|
||||
* required: ["name"],
|
||||
* properties: {
|
||||
* name: { type: "string" },
|
||||
* depth: { type: "string", enum: ["quick", "standard", "deep"] },
|
||||
* timeout: { type: "number" },
|
||||
* parallel: { type: "boolean" },
|
||||
* chaos: { type: "boolean" },
|
||||
@@ -142,12 +150,22 @@ export interface ProfileDefinition {
|
||||
* }
|
||||
*/
|
||||
export interface PresetDefinition {
|
||||
name: string;
|
||||
depth?: "quick" | "standard" | "deep";
|
||||
name?: string;
|
||||
mode?: 'verify' | 'observe' | 'qualify';
|
||||
routes?: string[];
|
||||
seed?: number;
|
||||
timeout?: number;
|
||||
parallel?: boolean;
|
||||
chaos?: boolean;
|
||||
observe?: boolean;
|
||||
runs?: number;
|
||||
sampling?: number;
|
||||
blocking?: boolean;
|
||||
sinks?: Record<string, unknown>;
|
||||
features?: string[];
|
||||
chaosStrategy?: 'one' | 'all' | 'sample' | 'routes';
|
||||
chaosSampleSize?: number;
|
||||
chaosSampleRoutes?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -198,6 +216,8 @@ export interface FailureRecord {
|
||||
category?: string;
|
||||
diff?: string;
|
||||
actual?: string;
|
||||
/** Source of the contract: 'route' or 'plugin:name' */
|
||||
source?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -300,8 +320,14 @@ export interface Artifact {
|
||||
total: number;
|
||||
passed: number;
|
||||
failed: number;
|
||||
skipped?: number;
|
||||
};
|
||||
executionSummary?: ExecutionSummary;
|
||||
coverageBreakdown?: {
|
||||
scenario: { routesCovered: string[]; stepsTotal: number; stepsPassed: number }
|
||||
stateful: { routesCovered: string[]; testsTotal: number; testsPassed: number }
|
||||
chaos: { routesPlanned: string[]; routesExecuted: string[]; runsTotal: number; runsPassed: number }
|
||||
}
|
||||
executedRoutes?: string[];
|
||||
skippedRoutes?: RouteExecutionInfo[];
|
||||
stepTraces?: StepTrace[];
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user