apophis/apophis-fastify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: remove observe CLI command — redundant with doctor + programmatic plugin

Browse Source
This commit is contained in:
John Dvorak
2026-05-22 13:47:18 -07:00
parent 04359613b5
commit fe8e94fbaa
9 changed files with 3 additions and 1923 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/commands/observe/index.ts
-351
View File
@@ -1,351 +0,0 @@
/**
* S5: Observe thread - Observe command handler
*
* Responsibilities:
* - Load config and resolve profile
* - Validate observe configuration
* - Check reporting sink setup (logs, metrics, traces)
* - Validate non-blocking semantics
* - Environment safety checks (block blocking behavior in prod by default)
* - Support --check-config (validate only, don't activate)
* - Explain what would be checked and why it is safe
* - Clear output about safety boundaries
* - Exit 0 on valid config, 2 on safety violation
*/
import type { CliContext } from '../../core/context.js';
import { loadConfig } from '../../core/config-loader.js';
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js';
import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js';
import { validateObserveConfig } from './validator.js';
import { renderDoctorChecks } from '../../renderers/human.js';
import { renderJson } from '../../renderers/json.js';
import type { OutputContext } from '../../renderers/shared.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveOptions {
profile?: string;
checkConfig?: boolean;
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
quiet?: boolean;
verbose?: boolean;
}
export interface ObserveResult {
exitCode: number;
message?: string;
checks?: Array<{
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}>;
}
// ---------------------------------------------------------------------------
// Command handler
// ---------------------------------------------------------------------------
/**
* Main observe command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks
* 3. Validate observe-specific configuration
* 4. If --check-config, stop after validation
* 5. Otherwise, report what would be activated and why it is safe
* 6. Return appropriate exit code
*/
export async function observeCommand(
options: ObserveOptions,
ctx: CliContext,
): Promise<ObserveResult> {
const { profile, checkConfig, config: configPath, cwd } = options;
const workingDir = cwd || ctx.cwd;
// Detect environment from context
const env = detectEnvironment();
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
});
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
};
}
const config = loadResult.config;
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'observe',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
});
const policyResult = policyEngine.check();
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => ` ✗ ${e}`),
].join('\n');
return {
exitCode: USAGE_ERROR,
message,
};
}
// 3. Validate observe-specific configuration
const validationResult = validateObserveConfig(config, profile || undefined, env);
if (!validationResult.valid) {
const message = formatValidationOutput(validationResult, { checkConfig, env, profile });
return {
exitCode: USAGE_ERROR,
message,
checks: validationResult.checks,
};
}
// 4. If --check-config, stop after validation with success
if (checkConfig) {
const message = formatValidationOutput(validationResult, {
checkConfig: true,
env,
profile,
});
return {
exitCode: SUCCESS,
message,
checks: validationResult.checks,
};
}
// 5. Report what would be activated and why it is safe
const activationMessage = formatActivationOutput(validationResult, {
env,
profile,
configPath: loadResult.configPath,
});
return {
exitCode: SUCCESS,
message: activationMessage,
checks: validationResult.checks,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
message: `Failed to run observe command: ${message}`,
checks: [],
};
}
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
interface FormatOptions {
checkConfig?: boolean;
env: string;
profile?: string;
configPath?: string;
}
/**
* Format validation results for human-readable output.
*/
function formatValidationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
const mode = options.checkConfig ? 'Config validation' : 'Observe validation';
lines.push(`${mode} for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
lines.push('');
// Print each check
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
}
lines.push('');
// Summary
if (result.errors.length > 0) {
lines.push(`Failed with ${result.errors.length} error(s).`);
lines.push('');
lines.push('Safety boundaries:');
lines.push(' - Observe mode is non-blocking by default');
lines.push(' - Blocking behavior is prohibited in production');
lines.push(' - Qualify-only features (chaos, stateful, etc.) are not allowed');
lines.push(' - Sampling rate must be between 0.0 and 1.0');
lines.push(' - Sinks must be configured when required by environment policy');
} else if (result.warnings.length > 0) {
lines.push(`Passed with ${result.warnings.length} warning(s).`);
} else {
lines.push('All checks passed.');
}
return lines.join('\n');
}
/**
* Format activation output explaining what would be checked and why it is safe.
*/
function formatActivationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
lines.push(`Observe mode ready for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
if (options.configPath) {
lines.push(`Config: ${options.configPath}`);
}
lines.push('');
// Print checks
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
}
lines.push('');
lines.push('What would be checked:');
lines.push(' - Request/response contracts are evaluated asynchronously');
lines.push(' - Violations are logged to configured sinks without blocking');
lines.push(' - Sampling controls the fraction of requests observed');
lines.push(' - Metrics and traces provide runtime visibility into contract health');
lines.push('');
lines.push('Why this is safe:');
lines.push(' - Non-blocking semantics guarantee observation does not affect latency');
lines.push(' - No chaos injection or stateful sequences are activated in observe mode');
lines.push(' - Production environments require explicit non-blocking configuration');
lines.push(' - All qualify-only features are blocked by validation');
if (result.warnings.length > 0) {
lines.push('');
lines.push('Warnings:');
for (const warning of result.warnings) {
lines.push(` ⚠ ${warning}`);
}
}
lines.push('');
lines.push('The CLI validates configuration and reports readiness.');
lines.push('To activate runtime observation, register the APOPHIS plugin in your');
lines.push('application with observe options:');
lines.push('');
lines.push(' await fastify.register(apophisPlugin, {');
lines.push(' runtime: \'warn\',');
lines.push(' observe: { enabled: true, sinks: [...] }');
lines.push(' });');
lines.push('');
lines.push('See docs/observe.md for the full programmatic activation guide.');
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the observe command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleObserve(
_args: string[],
ctx: CliContext,
): Promise<number> {
const options: ObserveOptions = {
profile: ctx.options.profile || undefined,
checkConfig: false,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as ObserveOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
};
// Parse command-specific flags from process.argv
// cac passes these as parsed options, but we need to extract --check-config
// Since cac doesn't expose parsed command-specific flags in the options object,
// we scan process.argv directly for observe-specific flags
const argv = process.argv.slice(2);
if (argv.includes('--check-config')) {
options.checkConfig = true;
}
const result = await observeCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'observe',
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}) + '\n');
} else if (format === 'json-summary') {
console.log(renderJson({
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}));
} else if (format === 'ndjson-summary') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'observe',
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
src/cli/commands/observe/validator.ts
-539
View File
@@ -1,539 +0,0 @@
/**
* S5: Observe thread - Observe config validation logic
*
* Validates observe-specific configuration including:
* - Sink configuration checks (logs, metrics, traces)
* - Sampling rate validation
* - Feature restriction checks (no qualify-only features in observe)
* - Non-blocking semantics validation
*/
import type { Config, ProfileDefinition, PresetDefinition } from '../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveValidationResult {
valid: boolean;
errors: string[];
warnings: string[];
checks: ObserveCheck[];
}
export interface ObserveCheck {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}
export interface SinkConfig {
logs?: boolean;
metrics?: boolean;
traces?: boolean;
endpoint?: string;
}
export interface ObserveProfileConfig {
sampling?: number;
blocking?: boolean;
sinks?: SinkConfig;
features?: string[];
}
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Features that are only valid in qualify mode */
const QUALIFY_ONLY_FEATURES = new Set([
'chaos',
'stateful',
'scenario',
'outbound-mocks',
'protocol-flow',
]);
/** Valid sampling rate bounds */
const SAMPLING_MIN = 0.0;
const SAMPLING_MAX = 1.0;
// ---------------------------------------------------------------------------
// Validation functions
// ---------------------------------------------------------------------------
/**
* Validate observe configuration for a given profile and environment.
*/
export function validateObserveConfig(
config: Config,
profileName: string | undefined,
env: string,
): ObserveValidationResult {
const errors: string[] = [];
const warnings: string[] = [];
const checks: ObserveCheck[] = [];
// Resolve the effective profile config (preset + profile overrides)
const profileConfig = resolveObserveProfileConfig(config, profileName);
// 1. Check profile exists and is observe mode
const profileCheck = validateProfileMode(config, profileName);
checks.push(profileCheck);
if (profileCheck.status === 'fail') {
errors.push(profileCheck.message);
}
// 2. Check for qualify-only features (uses resolved profile config)
const featureCheck = validateFeatures(profileConfig.features, profileName);
checks.push(featureCheck);
if (featureCheck.status === 'fail') {
errors.push(featureCheck.message);
}
// 3. Validate sampling rate (uses resolved profile config)
const samplingCheck = validateSamplingRate(profileConfig.sampling);
checks.push(samplingCheck);
if (samplingCheck.status === 'fail') {
errors.push(samplingCheck.message);
}
// 4. Validate sink configuration (uses resolved profile config)
const sinkCheck = validateSinkConfig(profileConfig.sinks, env, config);
checks.push(sinkCheck);
if (sinkCheck.status === 'fail') {
errors.push(sinkCheck.message);
} else if (sinkCheck.status === 'warn') {
warnings.push(sinkCheck.message);
}
// 5. Validate non-blocking semantics (uses resolved profile config)
const blockingCheck = validateBlockingSemantics(profileConfig.blocking, env, config);
checks.push(blockingCheck);
if (blockingCheck.status === 'fail') {
errors.push(blockingCheck.message);
}
// 6. Environment policy check: must explicitly allow observe
const envPolicyCheck = validateEnvironmentPolicy(config, env);
checks.push(envPolicyCheck);
if (envPolicyCheck.status === 'fail') {
errors.push(envPolicyCheck.message);
}
// 7. Environment safety check
const envCheck = validateEnvironmentSafety(env, profileConfig);
checks.push(envCheck);
if (envCheck.status === 'warn') {
warnings.push(envCheck.message);
}
// 8. Profile must be configured for observe mode
const profileObserveCheck = validateProfileObserveMode(config, profileName);
checks.push(profileObserveCheck);
if (profileObserveCheck.status === 'fail') {
errors.push(profileObserveCheck.message);
}
return {
valid: errors.length === 0,
errors,
warnings,
checks,
};
}
/**
* Resolve the observe-specific configuration from profile and preset.
* Preset values are applied first, then profile overrides.
*/
function resolveObserveProfileConfig(
config: Config,
profileName: string | undefined,
): ObserveProfileConfig {
const result: ObserveProfileConfig = {};
if (!profileName || !config.profiles) {
return result;
}
const profile = config.profiles[profileName];
if (!profile) {
return result;
}
// Apply preset first if referenced
if (profile.preset && config.presets) {
const preset = config.presets[profile.preset];
if (preset) {
Object.assign(result, presetToObserveConfig(preset));
}
}
// Apply profile overrides
Object.assign(result, profileToObserveConfig(profile));
return result;
}
/**
* Convert preset definition to observe config.
*/
function presetToObserveConfig(preset: PresetDefinition): ObserveProfileConfig {
return {
features: preset.features,
sampling: (preset as Record<string, unknown>).sampling as number | undefined,
blocking: (preset as Record<string, unknown>).blocking as boolean | undefined,
sinks: (preset as Record<string, unknown>).sinks as SinkConfig | undefined,
};
}
/**
* Convert profile definition to observe config.
*/
function profileToObserveConfig(profile: ProfileDefinition): ObserveProfileConfig {
return {
features: profile.features,
sampling: profile.sampling,
blocking: profile.blocking,
sinks: profile.sinks as SinkConfig | undefined,
};
}
/**
* Validate that the profile exists.
* Note: mode validation is handled by validateProfileObserveMode.
*/
function validateProfileMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-mode',
status: 'pass',
message: 'No profile specified, using default observe configuration',
};
}
if (!config.profiles || !config.profiles[profileName]) {
const available = config.profiles ? Object.keys(config.profiles).join(', ') : 'none';
return {
name: 'profile-mode',
status: 'fail',
message: `Profile "${profileName}" not found. Available profiles: ${available}`,
};
}
return {
name: 'profile-mode',
status: 'pass',
message: `Profile "${profileName}" exists`,
};
}
/**
* Validate that the profile is explicitly configured for observe mode.
*/
function validateProfileObserveMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: 'No profile specified, mode will be determined by top-level config',
};
}
if (!config.profiles || !config.profiles[profileName]) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" not found — will be validated by profile-mode check`,
};
}
const profile = config.profiles[profileName];
const profileMode = profile.mode;
if (profileMode && profileMode !== 'observe') {
return {
name: 'profile-observe-mode',
status: 'fail',
message: `Profile "${profileName}" is configured for "${profileMode}" mode but observe command requires "observe" mode`,
detail: 'Change the profile mode to "observe" or use the appropriate command ' +
`for "${profileMode}" mode (e.g., apophis ${profileMode}).`,
};
}
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" is configured for observe mode`,
};
}
/**
* Validate that no qualify-only features are used in observe mode.
*/
function validateFeatures(
features: string[] | undefined,
profileName: string | undefined,
): ObserveCheck {
if (!features || features.length === 0) {
return {
name: 'feature-restrictions',
status: 'pass',
message: 'No features configured',
};
}
const invalidFeatures = features.filter(f => QUALIFY_ONLY_FEATURES.has(f));
if (invalidFeatures.length > 0) {
const profileRef = profileName ? `Profile "${profileName}"` : 'Configuration';
return {
name: 'feature-restrictions',
status: 'fail',
message: `${profileRef} references qualify-only features that cannot be used in observe mode: ${invalidFeatures.join(', ')}`,
detail: `Remove these features from the profile or preset. Qualify-only features: ${Array.from(QUALIFY_ONLY_FEATURES).join(', ')}`,
};
}
return {
name: 'feature-restrictions',
status: 'pass',
message: `All features are valid for observe mode: ${features.join(', ')}`,
};
}
/**
* Validate sampling rate is within valid bounds [0.0, 1.0].
*/
export function validateSamplingRate(sampling: number | undefined): ObserveCheck {
if (sampling === undefined || sampling === null) {
return {
name: 'sampling-rate',
status: 'pass',
message: 'No sampling rate configured, using default (1.0)',
};
}
if (typeof sampling !== 'number' || Number.isNaN(sampling)) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate must be a number, got ${typeof sampling}`,
detail: `Valid range: ${SAMPLING_MIN} to ${SAMPLING_MAX} (inclusive)`,
};
}
if (sampling < SAMPLING_MIN || sampling > SAMPLING_MAX) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate ${sampling} is out of bounds`,
detail: `Set sampling to a value between ${SAMPLING_MIN} and ${SAMPLING_MAX} (inclusive). ` +
`A rate of 0.0 disables observation, 1.0 observes all requests.`,
};
}
return {
name: 'sampling-rate',
status: 'pass',
message: `Sampling rate ${sampling} is valid`,
};
}
/**
* Validate sink configuration for the environment.
*/
function validateSinkConfig(
sinks: SinkConfig | undefined,
env: string,
config: Config,
): ObserveCheck {
// Check if environment requires sinks
const envPolicy = config.environments?.[env];
const requireSink = envPolicy?.requireSink ?? false;
if (!sinks || Object.keys(sinks).length === 0) {
if (requireSink) {
return {
name: 'sink-config',
status: 'fail',
message: `Environment "${env}" requires sink configuration but none is provided`,
detail: 'Add sinks to your profile (e.g., sinks: { logs: true }) ' +
'or set requireSink: false in the environment policy.',
};
}
return {
name: 'sink-config',
status: 'warn',
message: 'No sinks configured. Observation data will not be persisted.',
detail: 'Configure at least one sink (logs, metrics, or traces) ' +
'to capture observation data for analysis.',
};
}
const activeSinks = [];
if (sinks.logs) activeSinks.push('logs');
if (sinks.metrics) activeSinks.push('metrics');
if (sinks.traces) activeSinks.push('traces');
if (activeSinks.length === 0) {
return {
name: 'sink-config',
status: 'warn',
message: 'Sinks are configured but none are enabled. Observation data will not be persisted.',
detail: 'Set at least one of logs, metrics, or traces to true in your sink configuration.',
};
}
return {
name: 'sink-config',
status: 'pass',
message: `Active sinks: ${activeSinks.join(', ')}`,
};
}
/**
* Validate non-blocking semantics for the environment.
* Blocking is NEVER allowed in production unless explicitly enabled by a break-glass policy.
*/
function validateBlockingSemantics(
blocking: boolean | undefined,
env: string,
config: Config,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (blocking === true && isProd) {
// Check for break-glass policy override
const envPolicy = config.environments?.[env];
const allowBlocking = envPolicy?.allowBlocking ?? false;
if (!allowBlocking) {
return {
name: 'blocking-semantics',
status: 'fail',
message: `Blocking behavior is not allowed in production environment "${env}"`,
detail: 'Set blocking: false in your profile, use a non-production environment, ' +
'or set allowBlocking: true in the environment policy for break-glass scenarios.',
};
}
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in production "${env}" via break-glass policy`,
detail: 'WARNING: blocking observation can severely impact request latency. ' +
'This should only be used during active incident response.',
};
}
if (blocking === true) {
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in non-production environment "${env}"`,
detail: 'Warning: blocking observation can increase request latency. ' +
'Only enable in environments where latency impact is acceptable.',
};
}
// blocking is false or undefined (default to non-blocking)
return {
name: 'blocking-semantics',
status: 'pass',
message: `Non-blocking semantics confirmed for environment "${env}"`,
detail: 'Observation will run asynchronously without blocking request handling.',
};
}
/**
* Validate environment policy explicitly allows observe mode.
*/
function validateEnvironmentPolicy(
config: Config,
env: string,
): ObserveCheck {
const envPolicy = config.environments?.[env];
if (!envPolicy) {
// No explicit policy for this environment — warn but don't fail
return {
name: 'environment-policy',
status: 'pass',
message: `No environment policy defined for "${env}"`,
detail: 'Observe mode is allowed by default when no policy is configured.',
};
}
const allowObserve = envPolicy.allowObserve;
if (allowObserve === false) {
return {
name: 'environment-policy',
status: 'fail',
message: `Environment policy for "${env}" explicitly blocks observe mode`,
detail: 'Set allowObserve: true in the environment policy to enable observe mode, ' +
'or run in an environment where observe is allowed.',
};
}
return {
name: 'environment-policy',
status: 'pass',
message: `Environment "${env}" explicitly allows observe mode`,
};
}
/**
* Validate environment-specific safety constraints.
*/
function validateEnvironmentSafety(
env: string,
profileConfig: ObserveProfileConfig,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (isProd) {
const warnings = [];
if (profileConfig.sampling === undefined) {
warnings.push('sampling rate not configured (will use default 1.0)');
}
if (!profileConfig.sinks) {
warnings.push('no sinks configured');
}
if (warnings.length > 0) {
return {
name: 'environment-safety',
status: 'warn',
message: `Production environment "${env}" observe configuration has warnings: ${warnings.join(', ')}`,
detail: 'In production, configure explicit sampling rate and sinks ' +
'to control observation overhead and ensure data capture.',
};
}
}
return {
name: 'environment-safety',
status: 'pass',
message: `Environment "${env}" safety checks passed`,
};
}
// ---------------------------------------------------------------------------
// Exports for testing
// ---------------------------------------------------------------------------
export {
QUALIFY_ONLY_FEATURES,
SAMPLING_MIN,
SAMPLING_MAX,
};
src/cli/core/index.ts
+1 -24
View File
@@ -13,7 +13,6 @@ const HELP_HEADER = `
${pc.dim('Commands:')} ${pc.dim('Commands:')}
init Scaffold config, scripts, and example usage init Scaffold config, scripts, and example usage
verify Run deterministic contract verification verify Run deterministic contract verification
observe Validate runtime observe configuration and reporting setup
qualify Run scenario, stateful, protocol, or chaos-driven qualification qualify Run scenario, stateful, protocol, or chaos-driven qualification
replay Replay a failure using seed and stored trace replay Replay a failure using seed and stored trace
doctor Validate config, environment safety, docs/example correctness doctor Validate config, environment safety, docs/example correctness
@@ -37,7 +36,6 @@ const HELP_HEADER = `
${pc.dim('Examples:')} ${pc.dim('Examples:')}
apophis init --preset safe-ci apophis init --preset safe-ci
apophis verify --profile quick --routes "POST /users" apophis verify --profile quick --routes "POST /users"
apophis observe --profile staging-observe --check-config
apophis qualify --profile oauth-nightly --seed 42 apophis qualify --profile oauth-nightly --seed 42
apophis replay --artifact reports/apophis/failure-*.json apophis replay --artifact reports/apophis/failure-*.json
apophis doctor apophis doctor
@@ -80,20 +78,6 @@ function getCommandHelp(command: string): string {
apophis verify --routes "POST /users" --seed 42 apophis verify --routes "POST /users" --seed 42
apophis verify --changed apophis verify --changed
apophis verify --workspace apophis verify --workspace
`,
observe: `
${pc.bold('apophis observe')} — Validate runtime observe configuration and reporting setup
${pc.dim('Usage:')}
apophis observe [options]
${pc.dim('Options:')}
--profile <name> Profile name from config
--check-config Only validate, do not activate
${pc.dim('Examples:')}
apophis observe --profile staging-observe
apophis observe --check-config
`, `,
qualify: ` qualify: `
${pc.bold('apophis qualify')} — Run scenario, stateful, protocol, or chaos-driven qualification ${pc.bold('apophis qualify')} — Run scenario, stateful, protocol, or chaos-driven qualification
@@ -203,13 +187,12 @@ function writeMachineRecord(
process.stdout.write(JSON.stringify(payload) + '\n'); process.stdout.write(JSON.stringify(payload) + '\n');
} }
type CommandName = 'init' | 'verify' | 'observe' | 'qualify' | 'replay' | 'doctor' | 'migrate'; type CommandName = 'init' | 'verify' | 'qualify' | 'replay' | 'doctor' | 'migrate';
type CommandHandler = (args: string[], ctx: CliContext) => Promise<number>; type CommandHandler = (args: string[], ctx: CliContext) => Promise<number>;
const commandLoaders: Record<CommandName, () => Promise<CommandHandler>> = { const commandLoaders: Record<CommandName, () => Promise<CommandHandler>> = {
init: async () => (await import('../commands/init/index.js')).handleInit, init: async () => (await import('../commands/init/index.js')).handleInit,
verify: async () => (await import('../commands/verify/index.js')).handleVerify, verify: async () => (await import('../commands/verify/index.js')).handleVerify,
observe: async () => (await import('../commands/observe/index.js')).handleObserve,
qualify: async () => (await import('../commands/qualify/index.js')).handleQualify, qualify: async () => (await import('../commands/qualify/index.js')).handleQualify,
replay: async () => (await import('../commands/replay/index.js')).handleReplay, replay: async () => (await import('../commands/replay/index.js')).handleReplay,
doctor: async () => (await import('../commands/doctor/index.js')).handleDoctor, doctor: async () => (await import('../commands/doctor/index.js')).handleDoctor,
@@ -254,7 +237,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
const commands = [ const commands = [
'init', 'init',
'verify', 'verify',
'observe',
'qualify', 'qualify',
'replay', 'replay',
'doctor', 'doctor',
@@ -277,10 +259,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
cmd.option('--seed <number>', 'Deterministic seed'); cmd.option('--seed <number>', 'Deterministic seed');
cmd.option('--changed', 'Filter to git-modified routes'); cmd.option('--changed', 'Filter to git-modified routes');
break; break;
case 'observe':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--check-config', 'Only validate, do not activate');
break;
case 'qualify': case 'qualify':
cmd.option('--profile <name>', 'Profile name from config'); cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--seed <number>', 'Deterministic seed'); cmd.option('--seed <number>', 'Deterministic seed');
@@ -382,7 +360,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
const commandSpecificFlags: Record<string, Set<string>> = { const commandSpecificFlags: Record<string, Set<string>> = {
init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']), init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']),
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']), verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config']),
qualify: new Set(['--profile', '--seed', '--changed']), qualify: new Set(['--profile', '--seed', '--changed']),
replay: new Set(['--artifact', '--route']), replay: new Set(['--artifact', '--route']),
doctor: new Set(['--mode', '--strict', '--workspace']), doctor: new Set(['--mode', '--strict', '--workspace']),
src/test/cli/acceptance.test.ts
-6
View File
@@ -83,12 +83,6 @@ test('acceptance matrix routes through CLI main entrypoint', async () => {
exitClass: 'success', exitClass: 'success',
requiredSignals: ['Summary', 'Seed: 42'], requiredSignals: ['Summary', 'Seed: 42'],
}, },
{
name: 'observe profile succeeds on observe-config',
args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe'],
exitClass: 'success',
requiredSignals: ['Observe mode ready for environment'],
},
{ {
name: 'qualify runs protocol-lab scenario flow', name: 'qualify runs protocol-lab scenario flow',
args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42'], args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42'],
src/test/cli/dispatch.test.ts
-1
View File
@@ -33,7 +33,6 @@ function captureOutput<T>(fn: () => Promise<T>): Promise<{ result: T; stdout: st
const commands = [ const commands = [
'init', 'init',
'verify', 'verify',
'observe',
'qualify', 'qualify',
'replay', 'replay',
'doctor', 'doctor',
src/test/cli/docs-smoke.test.ts
+1 -1
View File
@@ -190,7 +190,7 @@ test('docs contain expected CLI commands', () => {
const docsDir = resolve(process.cwd(), 'docs'); const docsDir = resolve(process.cwd(), 'docs');
const files = findMarkdownFiles(docsDir); const files = findMarkdownFiles(docsDir);
const commandNames = ['init', 'verify', 'observe', 'qualify', 'replay', 'doctor', 'migrate']; const commandNames = ['init', 'verify', 'qualify', 'replay', 'doctor', 'migrate'];
const foundCommands = new Set<string>(); const foundCommands = new Set<string>();
for (const file of files) { for (const file of files) {
src/test/cli/goldens.test.ts
-20
View File
@@ -49,7 +49,6 @@ test('global --help matches golden snapshot', async () => {
assert.ok(output.includes('apophis'), 'Should include apophis'); assert.ok(output.includes('apophis'), 'Should include apophis');
assert.ok(output.includes('init'), 'Should include init command'); assert.ok(output.includes('init'), 'Should include init command');
assert.ok(output.includes('verify'), 'Should include verify command'); assert.ok(output.includes('verify'), 'Should include verify command');
assert.ok(output.includes('observe'), 'Should include observe command');
assert.ok(output.includes('qualify'), 'Should include qualify command'); assert.ok(output.includes('qualify'), 'Should include qualify command');
assert.ok(output.includes('replay'), 'Should include replay command'); assert.ok(output.includes('replay'), 'Should include replay command');
assert.ok(output.includes('doctor'), 'Should include doctor command'); assert.ok(output.includes('doctor'), 'Should include doctor command');
@@ -75,24 +74,6 @@ test('verify --help matches golden snapshot', async () => {
assert.ok(output.includes('--seed'), 'Should include --seed'); assert.ok(output.includes('--seed'), 'Should include --seed');
}); });
test('observe --help matches golden snapshot', async () => {
const golden = readGolden('observe-help.txt');
const originalLog = console.log;
let output = '';
console.log = (msg: string) => { output += msg + '\n'; };
try {
await main(['observe', '--help']);
} finally {
console.log = originalLog;
}
assert.ok(output.includes('apophis observe'), 'Should include observe header');
assert.ok(output.includes('--profile'), 'Should include --profile');
assert.ok(output.includes('--check-config'), 'Should include --check-config');
});
test('qualify --help matches golden snapshot', async () => { test('qualify --help matches golden snapshot', async () => {
const golden = readGolden('qualify-help.txt'); const golden = readGolden('qualify-help.txt');
@@ -199,7 +180,6 @@ test('all golden files are accounted for', () => {
'help.txt', 'help.txt',
'verify-help.txt', 'verify-help.txt',
'verify-failure.txt', 'verify-failure.txt',
'observe-help.txt',
'qualify-help.txt', 'qualify-help.txt',
'replay-help.txt', 'replay-help.txt',
'doctor-help.txt', 'doctor-help.txt',
src/test/cli/observe-safety.test.ts
-980
View File
@@ -1,980 +0,0 @@
/**
* WS3: Observe mode safety hardening tests
*
* Comprehensive boundary and edge-case tests for observe mode safety.
* Every safety violation must return exit code 2 with clear boundary explanation.
*/
import { test } from 'node:test';
import assert from 'node:assert';
import { observeCommand } from '../../cli/commands/observe/index.js';
import { USAGE_ERROR } from '../../cli/core/exit-codes.js';
import { createTestContext, writeTempConfig, cleanupTempDir } from './helpers.js';
// ---------------------------------------------------------------------------
// Test 1: Blocking in production without break-glass
// ---------------------------------------------------------------------------
test('blocking in production without break-glass fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-prod`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
blocking: true,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
allowBlocking: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('Blocking'), 'Should mention blocking boundary');
assert.ok(result.message!.includes('production'), 'Should mention production');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 2: Blocking in production WITH break-glass (should pass)
// ---------------------------------------------------------------------------
test('blocking in production with break-glass policy passes', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-breakglass`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
blocking: true,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
allowBlocking: true,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('break-glass') || result.message!.includes('Blocking'), 'Should mention break-glass');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 3: Blocking in staging (should pass)
// ---------------------------------------------------------------------------
test('blocking in staging passes without break-glass', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'staging';
const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-staging`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'staging-observe': {
name: 'staging-observe',
mode: 'observe' as const,
blocking: true,
},
},
environments: {
staging: {
name: 'staging',
allowObserve: true,
allowBlocking: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'staging', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'staging-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('staging'), 'Should mention staging environment');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 4: Sampling rate = -0.1 (out of bounds)
// ---------------------------------------------------------------------------
test('sampling rate -0.1 fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-neg`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'bad-sampling': {
name: 'bad-sampling',
mode: 'observe' as const,
sampling: -0.1,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'bad-sampling', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sampling') || result.message!.includes('Sampling'), 'Should mention sampling boundary');
assert.ok(result.message!.includes('-0.1') || result.message!.includes('out of bounds'), 'Should mention the invalid value');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 5: Sampling rate = 1.5 (out of bounds)
// ---------------------------------------------------------------------------
test('sampling rate 1.5 fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-high`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'bad-sampling': {
name: 'bad-sampling',
mode: 'observe' as const,
sampling: 1.5,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'bad-sampling', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sampling') || result.message!.includes('Sampling'), 'Should mention sampling boundary');
assert.ok(result.message!.includes('1.5') || result.message!.includes('out of bounds'), 'Should mention the invalid value');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 6: Sampling rate = 1.0 (boundary, should pass)
// ---------------------------------------------------------------------------
test('sampling rate 1.0 at boundary passes', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-boundary`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'boundary-sampling': {
name: 'boundary-sampling',
mode: 'observe' as const,
sampling: 1.0,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'boundary-sampling', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
const samplingCheck = result.checks!.find(c => c.name === 'sampling-rate');
assert.ok(samplingCheck, 'Should have sampling-rate check');
assert.strictEqual(samplingCheck!.status, 'pass', 'Sampling rate 1.0 should pass');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 7: Missing sink in production (should fail)
// ---------------------------------------------------------------------------
test('missing sink in production fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-sink-prod`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
requireSink: true,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sink') || result.message!.includes('Sink'), 'Should mention sink boundary');
assert.ok(result.message!.includes('production') || result.message!.includes('requires'), 'Should mention production requirement');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 8: Missing sink in staging (should fail)
// ---------------------------------------------------------------------------
test('missing sink in staging fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'staging';
const tmpDir = `${process.cwd()}/tmp-observe-test-sink-staging`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'staging-observe': {
name: 'staging-observe',
mode: 'observe' as const,
},
},
environments: {
staging: {
name: 'staging',
allowObserve: true,
requireSink: true,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'staging', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'staging-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sink') || result.message!.includes('Sink'), 'Should mention sink boundary');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 9: Missing sink in local (should pass or warn)
// ---------------------------------------------------------------------------
test('missing sink in local passes with warning', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'local';
const tmpDir = `${process.cwd()}/tmp-observe-test-sink-local`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'local-observe': {
name: 'local-observe',
mode: 'observe' as const,
},
},
environments: {
local: {
name: 'local',
allowObserve: true,
requireSink: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'local', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'local-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
const sinkCheck = result.checks!.find(c => c.name === 'sink-config');
assert.ok(sinkCheck, 'Should have sink-config check');
assert.ok(sinkCheck!.status === 'warn' || sinkCheck!.status === 'pass', 'Should warn or pass about missing sinks in local');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 10: Profile with chaos feature
// ---------------------------------------------------------------------------
test('profile with chaos feature fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-chaos`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'chaos-observe': {
name: 'chaos-observe',
mode: 'observe' as const,
features: ['chaos'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'chaos-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('chaos'), 'Should mention chaos feature');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 11: Profile with stateful feature
// ---------------------------------------------------------------------------
test('profile with stateful feature fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-stateful`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'stateful-observe': {
name: 'stateful-observe',
mode: 'observe' as const,
features: ['stateful'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'stateful-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('stateful'), 'Should mention stateful feature');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 12: Profile with scenario feature
// ---------------------------------------------------------------------------
test('profile with scenario feature fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-scenario`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'scenario-observe': {
name: 'scenario-observe',
mode: 'observe' as const,
features: ['scenario'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'scenario-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('scenario'), 'Should mention scenario feature');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 13: Profile configured for verify mode
// ---------------------------------------------------------------------------
test('profile configured for verify mode fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-verify-mode`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'verify-profile': {
name: 'verify-profile',
mode: 'verify' as const,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'verify-profile', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('verify'), 'Should mention verify mode');
assert.ok(result.message!.includes('observe'), 'Should mention observe mode requirement');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 14: Environment policy blocking observe
// ---------------------------------------------------------------------------
test('environment policy blocking observe fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-policy-block`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
},
},
environments: {
production: {
name: 'production',
allowObserve: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('observe'), 'Should mention observe mode');
assert.ok(result.message!.includes('blocks') || result.message!.includes('not allowed') || result.message!.includes('Policy'), 'Should mention policy blocking');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 15: No config found
// ---------------------------------------------------------------------------
test('no config found fails with exit code 2 and suggests init', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-noconfig`;
const fs = await import('node:fs');
fs.mkdirSync(tmpDir, { recursive: true });
try {
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('No config') || result.message!.includes('init'), 'Should mention missing config and suggest init');
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
// ---------------------------------------------------------------------------
// Test 16: Preset features are resolved and validated
// ---------------------------------------------------------------------------
test('preset with qualify-only features is caught via profile resolution', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-preset-features`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'preset-observe': {
name: 'preset-observe',
mode: 'observe' as const,
preset: 'bad-preset',
},
},
presets: {
'bad-preset': {
name: 'bad-preset',
features: ['chaos', 'stateful'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'preset-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('chaos') || result.message!.includes('stateful'), 'Should mention preset features');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 17: Observe output explains what would be observed and why safe
// ---------------------------------------------------------------------------
test('observe activation output explains what and why safe', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-safe-output`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'safe-observe': {
name: 'safe-observe',
mode: 'observe' as const,
sampling: 0.5,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'safe-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('checked') || result.message!.includes('observe'), 'Should explain what would be observed');
assert.ok(result.message!.includes('safe') || result.message!.includes('safety') || result.message!.includes('non-blocking'), 'Should explain why it is safe');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 18: Every safety violation returns exit code 2 with boundary name
// ---------------------------------------------------------------------------
test('every safety violation returns exit code 2 with exact boundary', async () => {
const violations = [
{
name: 'blocking-prod',
config: {
mode: 'observe',
profiles: { 'blocking-prod': { name: 'blocking-prod', mode: 'observe', blocking: true } },
environments: { production: { name: 'production', allowObserve: true, allowBlocking: false } },
},
env: 'production',
profile: 'blocking-prod',
expectedBoundary: 'blocking',
},
{
name: 'sampling-neg',
config: {
mode: 'observe',
profiles: { 'sampling-neg': { name: 'sampling-neg', mode: 'observe', sampling: -0.1 } },
},
env: 'test',
profile: 'sampling-neg',
expectedBoundary: 'sampling',
},
{
name: 'missing-sink-prod',
config: {
mode: 'observe',
profiles: { 'missing-sink-prod': { name: 'missing-sink-prod', mode: 'observe' } },
environments: { production: { name: 'production', allowObserve: true, requireSink: true } },
},
env: 'production',
profile: 'missing-sink-prod',
expectedBoundary: 'sink',
},
];
for (const violation of violations) {
const tmpDir = `${process.cwd()}/tmp-observe-test-${violation.name}`;
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = violation.env;
try {
const configPath = await writeTempConfig(tmpDir, violation.config);
const ctx = createTestContext({ env: { nodeEnv: violation.env, apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: violation.profile, config: configPath },
ctx,
);
assert.strictEqual(
result.exitCode,
USAGE_ERROR,
`${violation.name}: Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`,
);
assert.ok(
result.message!.toLowerCase().includes(violation.expectedBoundary),
`${violation.name}: Should mention "${violation.expectedBoundary}" boundary: ${result.message}`,
);
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
}
});
// ---------------------------------------------------------------------------
// Test 19: No silent passes — every failure has a message
// ---------------------------------------------------------------------------
test('no silent passes — every failure includes a message', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-nosilent`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'bad-profile': {
name: 'bad-profile',
mode: 'observe' as const,
features: ['chaos'],
sampling: -0.5,
blocking: true,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
requireSink: true,
allowBlocking: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
try {
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'bad-profile', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail with usage error');
assert.ok(result.message && result.message.length > 0, 'Should have a non-empty error message');
// Note: result.checks may be undefined when caught by policy engine before validator runs
if (result.checks) {
const failedChecks = result.checks.filter(c => c.status === 'fail');
assert.ok(failedChecks.length >= 3, `Should have at least 3 failed checks, got ${failedChecks.length}`);
for (const check of failedChecks) {
assert.ok(check.message.length > 0, `Check "${check.name}" should have a message`);
}
}
} finally {
process.env.NODE_ENV = originalNodeEnv;
}
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Fixture-based scenarios preserved from observe acceptance coverage
// ---------------------------------------------------------------------------
test('fixture profile validates successfully with expected checks', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'staging-observe',
},
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success but got: ${result.message}`);
assert.ok(result.checks && result.checks.length > 0, 'Should return validation checks');
const checkNames = result.checks!.map(c => c.name);
assert.ok(checkNames.includes('profile-mode'), 'Should check profile mode');
assert.ok(checkNames.includes('feature-restrictions'), 'Should check feature restrictions');
assert.ok(checkNames.includes('sink-config'), 'Should check sink config');
assert.ok(checkNames.includes('blocking-semantics'), 'Should check blocking semantics');
});
test('fixture check-config validates only without activation semantics', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'staging-observe',
checkConfig: true,
},
ctx,
);
assert.strictEqual(result.exitCode, 0, `Should pass validation: ${result.message}`);
assert.ok(result.message, 'Should include validation output');
assert.ok(
result.message!.includes('Config validation') || result.message!.includes('check'),
`Should indicate check mode: ${result.message}`,
);
assert.ok(
!result.message!.includes('activated') || result.message!.includes('validate'),
'Should be validation-only output',
);
});
test('fixture unknown profile lists available profiles', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'nonexistent-profile',
},
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail with unknown profile');
assert.ok(
result.message!.includes('not found') || result.message!.includes('Available'),
`Should list available profiles: ${result.message}`,
);
});
test('fixture output explains safety boundaries clearly', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'staging-observe',
},
ctx,
);
assert.strictEqual(result.exitCode, 0);
assert.ok(result.message, 'Should include output message');
assert.ok(
result.message!.includes('safe') || result.message!.includes('safety') || result.message!.includes('non-blocking'),
`Should mention safety boundaries: ${result.message}`,
);
assert.ok(
result.message!.includes('checked') || result.message!.includes('check'),
`Should mention checks: ${result.message}`,
);
});
test('environment allowedModes restriction blocks observe mode', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-allowed-modes`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
},
},
environments: {
production: {
name: 'production',
allowedModes: ['verify'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{
cwd: tmpDir,
profile: 'prod-observe',
config: configPath,
},
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail when observe is not in allowedModes');
assert.ok(
result.message!.includes('not allowed') || result.message!.includes('Policy'),
`Should explain policy violation: ${result.message}`,
);
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 20: Suggest how to fix for each boundary violation
// ---------------------------------------------------------------------------
test('each boundary violation suggests how to fix it', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-fix-suggestions`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'fixme': {
name: 'fixme',
mode: 'observe' as const,
features: ['chaos'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'fixme', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR);
assert.ok(
result.message!.includes('Set') ||
result.message!.includes('Change') ||
result.message!.includes('remove') ||
result.message!.includes('Remove') ||
result.message!.includes('qualify-only'),
`Should suggest how to fix: ${result.message}`
);
} finally {
await cleanupTempDir(tmpDir);
}
});
src/test/cli/packaging.test.ts
+1 -1
View File
@@ -82,7 +82,7 @@ describe('packaging', () => {
}); });
// For each of the 7 commands, verify they do NOT print "Not yet implemented" // For each of the 7 commands, verify they do NOT print "Not yet implemented"
const commands = ['init', 'verify', 'observe', 'qualify', 'replay', 'doctor', 'migrate']; const commands = ['init', 'verify', 'qualify', 'replay', 'doctor', 'migrate'];
for (const cmd of commands) { for (const cmd of commands) {
it(`${cmd} does not print "Not yet implemented"`, () => { it(`${cmd} does not print "Not yet implemented"`, () => {
// Some commands may fail for config reasons; we just assert they don't say "Not yet implemented" // Some commands may fail for config reasons; we just assert they don't say "Not yet implemented"