apophis/apophis-fastify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: remove observe CLI command — redundant with doctor + programmatic plugin

Browse Source
This commit is contained in:
John Dvorak
2026-05-22 13:47:18 -07:00
parent 04359613b5
commit fe8e94fbaa
9 changed files with 3 additions and 1923 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/commands/observe/index.ts
-351
View File
@@ -1,351 +0,0 @@
/**
* S5: Observe thread - Observe command handler
*
* Responsibilities:
* - Load config and resolve profile
* - Validate observe configuration
* - Check reporting sink setup (logs, metrics, traces)
* - Validate non-blocking semantics
* - Environment safety checks (block blocking behavior in prod by default)
* - Support --check-config (validate only, don't activate)
* - Explain what would be checked and why it is safe
* - Clear output about safety boundaries
* - Exit 0 on valid config, 2 on safety violation
*/
import type { CliContext } from '../../core/context.js';
import { loadConfig } from '../../core/config-loader.js';
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js';
import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js';
import { validateObserveConfig } from './validator.js';
import { renderDoctorChecks } from '../../renderers/human.js';
import { renderJson } from '../../renderers/json.js';
import type { OutputContext } from '../../renderers/shared.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveOptions {
profile?: string;
checkConfig?: boolean;
config?: string;
cwd?: string;
format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary';
quiet?: boolean;
verbose?: boolean;
}
export interface ObserveResult {
exitCode: number;
message?: string;
checks?: Array<{
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}>;
}
// ---------------------------------------------------------------------------
// Command handler
// ---------------------------------------------------------------------------
/**
* Main observe command handler.
*
* Flow:
* 1. Load and resolve config
* 2. Run policy engine checks
* 3. Validate observe-specific configuration
* 4. If --check-config, stop after validation
* 5. Otherwise, report what would be activated and why it is safe
* 6. Return appropriate exit code
*/
export async function observeCommand(
options: ObserveOptions,
ctx: CliContext,
): Promise<ObserveResult> {
const { profile, checkConfig, config: configPath, cwd } = options;
const workingDir = cwd || ctx.cwd;
// Detect environment from context
const env = detectEnvironment();
try {
// 1. Load config
const loadResult = await loadConfig({
cwd: workingDir,
configPath,
profileName: profile,
env,
});
if (!loadResult.configPath) {
return {
exitCode: USAGE_ERROR,
message: 'No config found. Run "apophis init" to create one.',
};
}
const config = loadResult.config;
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
config,
env,
mode: 'observe',
profileName: profile || undefined,
presetName: loadResult.presetName || undefined,
});
const policyResult = policyEngine.check();
if (!policyResult.allowed) {
const message = [
'Policy check failed:',
...policyResult.errors.map(e => ` ✗ ${e}`),
].join('\n');
return {
exitCode: USAGE_ERROR,
message,
};
}
// 3. Validate observe-specific configuration
const validationResult = validateObserveConfig(config, profile || undefined, env);
if (!validationResult.valid) {
const message = formatValidationOutput(validationResult, { checkConfig, env, profile });
return {
exitCode: USAGE_ERROR,
message,
checks: validationResult.checks,
};
}
// 4. If --check-config, stop after validation with success
if (checkConfig) {
const message = formatValidationOutput(validationResult, {
checkConfig: true,
env,
profile,
});
return {
exitCode: SUCCESS,
message,
checks: validationResult.checks,
};
}
// 5. Report what would be activated and why it is safe
const activationMessage = formatActivationOutput(validationResult, {
env,
profile,
configPath: loadResult.configPath,
});
return {
exitCode: SUCCESS,
message: activationMessage,
checks: validationResult.checks,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
exitCode: USAGE_ERROR,
message: `Failed to run observe command: ${message}`,
checks: [],
};
}
}
// ---------------------------------------------------------------------------
// Output formatting
// ---------------------------------------------------------------------------
interface FormatOptions {
checkConfig?: boolean;
env: string;
profile?: string;
configPath?: string;
}
/**
* Format validation results for human-readable output.
*/
function formatValidationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
const mode = options.checkConfig ? 'Config validation' : 'Observe validation';
lines.push(`${mode} for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
lines.push('');
// Print each check
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
if (check.detail) {
lines.push(` ${check.detail}`);
}
}
lines.push('');
// Summary
if (result.errors.length > 0) {
lines.push(`Failed with ${result.errors.length} error(s).`);
lines.push('');
lines.push('Safety boundaries:');
lines.push(' - Observe mode is non-blocking by default');
lines.push(' - Blocking behavior is prohibited in production');
lines.push(' - Qualify-only features (chaos, stateful, etc.) are not allowed');
lines.push(' - Sampling rate must be between 0.0 and 1.0');
lines.push(' - Sinks must be configured when required by environment policy');
} else if (result.warnings.length > 0) {
lines.push(`Passed with ${result.warnings.length} warning(s).`);
} else {
lines.push('All checks passed.');
}
return lines.join('\n');
}
/**
* Format activation output explaining what would be checked and why it is safe.
*/
function formatActivationOutput(
result: import('./validator.js').ObserveValidationResult,
options: FormatOptions,
): string {
const lines: string[] = [];
lines.push(`Observe mode ready for environment "${options.env}"`);
if (options.profile) {
lines.push(`Profile: ${options.profile}`);
}
if (options.configPath) {
lines.push(`Config: ${options.configPath}`);
}
lines.push('');
// Print checks
for (const check of result.checks) {
const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗';
lines.push(` ${icon} ${check.name}: ${check.message}`);
}
lines.push('');
lines.push('What would be checked:');
lines.push(' - Request/response contracts are evaluated asynchronously');
lines.push(' - Violations are logged to configured sinks without blocking');
lines.push(' - Sampling controls the fraction of requests observed');
lines.push(' - Metrics and traces provide runtime visibility into contract health');
lines.push('');
lines.push('Why this is safe:');
lines.push(' - Non-blocking semantics guarantee observation does not affect latency');
lines.push(' - No chaos injection or stateful sequences are activated in observe mode');
lines.push(' - Production environments require explicit non-blocking configuration');
lines.push(' - All qualify-only features are blocked by validation');
if (result.warnings.length > 0) {
lines.push('');
lines.push('Warnings:');
for (const warning of result.warnings) {
lines.push(` ⚠ ${warning}`);
}
}
lines.push('');
lines.push('The CLI validates configuration and reports readiness.');
lines.push('To activate runtime observation, register the APOPHIS plugin in your');
lines.push('application with observe options:');
lines.push('');
lines.push(' await fastify.register(apophisPlugin, {');
lines.push(' runtime: \'warn\',');
lines.push(' observe: { enabled: true, sinks: [...] }');
lines.push(' });');
lines.push('');
lines.push('See docs/observe.md for the full programmatic activation guide.');
return lines.join('\n');
}
// ---------------------------------------------------------------------------
// CLI adapter
// ---------------------------------------------------------------------------
/**
* Adapter that bridges the CLI framework (cac) to the observe command handler.
* This function signature matches what the CLI core expects.
*/
export async function handleObserve(
_args: string[],
ctx: CliContext,
): Promise<number> {
const options: ObserveOptions = {
profile: ctx.options.profile || undefined,
checkConfig: false,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
format: ctx.options.format as ObserveOptions['format'],
quiet: ctx.options.quiet,
verbose: ctx.options.verbose,
};
// Parse command-specific flags from process.argv
// cac passes these as parsed options, but we need to extract --check-config
// Since cac doesn't expose parsed command-specific flags in the options object,
// we scan process.argv directly for observe-specific flags
const argv = process.argv.slice(2);
if (argv.includes('--check-config')) {
options.checkConfig = true;
}
const result = await observeCommand(options, ctx);
// Output result based on format
if (!ctx.options.quiet && result.message) {
const format = options.format || ctx.options.format || 'human';
if (format === 'json') {
console.log(renderJson({
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}));
} else if (format === 'ndjson') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'observe',
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}) + '\n');
} else if (format === 'json-summary') {
console.log(renderJson({
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}));
} else if (format === 'ndjson-summary') {
process.stdout.write(JSON.stringify({
type: 'run.completed',
command: 'observe',
exitCode: result.exitCode,
checks: result.checks,
message: result.message,
}) + '\n');
} else {
console.log(result.message);
}
}
return result.exitCode;
}
src/cli/commands/observe/validator.ts
-539
View File
@@ -1,539 +0,0 @@
/**
* S5: Observe thread - Observe config validation logic
*
* Validates observe-specific configuration including:
* - Sink configuration checks (logs, metrics, traces)
* - Sampling rate validation
* - Feature restriction checks (no qualify-only features in observe)
* - Non-blocking semantics validation
*/
import type { Config, ProfileDefinition, PresetDefinition } from '../../core/config-loader.js';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface ObserveValidationResult {
valid: boolean;
errors: string[];
warnings: string[];
checks: ObserveCheck[];
}
export interface ObserveCheck {
name: string;
status: 'pass' | 'fail' | 'warn';
message: string;
detail?: string;
}
export interface SinkConfig {
logs?: boolean;
metrics?: boolean;
traces?: boolean;
endpoint?: string;
}
export interface ObserveProfileConfig {
sampling?: number;
blocking?: boolean;
sinks?: SinkConfig;
features?: string[];
}
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Features that are only valid in qualify mode */
const QUALIFY_ONLY_FEATURES = new Set([
'chaos',
'stateful',
'scenario',
'outbound-mocks',
'protocol-flow',
]);
/** Valid sampling rate bounds */
const SAMPLING_MIN = 0.0;
const SAMPLING_MAX = 1.0;
// ---------------------------------------------------------------------------
// Validation functions
// ---------------------------------------------------------------------------
/**
* Validate observe configuration for a given profile and environment.
*/
export function validateObserveConfig(
config: Config,
profileName: string | undefined,
env: string,
): ObserveValidationResult {
const errors: string[] = [];
const warnings: string[] = [];
const checks: ObserveCheck[] = [];
// Resolve the effective profile config (preset + profile overrides)
const profileConfig = resolveObserveProfileConfig(config, profileName);
// 1. Check profile exists and is observe mode
const profileCheck = validateProfileMode(config, profileName);
checks.push(profileCheck);
if (profileCheck.status === 'fail') {
errors.push(profileCheck.message);
}
// 2. Check for qualify-only features (uses resolved profile config)
const featureCheck = validateFeatures(profileConfig.features, profileName);
checks.push(featureCheck);
if (featureCheck.status === 'fail') {
errors.push(featureCheck.message);
}
// 3. Validate sampling rate (uses resolved profile config)
const samplingCheck = validateSamplingRate(profileConfig.sampling);
checks.push(samplingCheck);
if (samplingCheck.status === 'fail') {
errors.push(samplingCheck.message);
}
// 4. Validate sink configuration (uses resolved profile config)
const sinkCheck = validateSinkConfig(profileConfig.sinks, env, config);
checks.push(sinkCheck);
if (sinkCheck.status === 'fail') {
errors.push(sinkCheck.message);
} else if (sinkCheck.status === 'warn') {
warnings.push(sinkCheck.message);
}
// 5. Validate non-blocking semantics (uses resolved profile config)
const blockingCheck = validateBlockingSemantics(profileConfig.blocking, env, config);
checks.push(blockingCheck);
if (blockingCheck.status === 'fail') {
errors.push(blockingCheck.message);
}
// 6. Environment policy check: must explicitly allow observe
const envPolicyCheck = validateEnvironmentPolicy(config, env);
checks.push(envPolicyCheck);
if (envPolicyCheck.status === 'fail') {
errors.push(envPolicyCheck.message);
}
// 7. Environment safety check
const envCheck = validateEnvironmentSafety(env, profileConfig);
checks.push(envCheck);
if (envCheck.status === 'warn') {
warnings.push(envCheck.message);
}
// 8. Profile must be configured for observe mode
const profileObserveCheck = validateProfileObserveMode(config, profileName);
checks.push(profileObserveCheck);
if (profileObserveCheck.status === 'fail') {
errors.push(profileObserveCheck.message);
}
return {
valid: errors.length === 0,
errors,
warnings,
checks,
};
}
/**
* Resolve the observe-specific configuration from profile and preset.
* Preset values are applied first, then profile overrides.
*/
function resolveObserveProfileConfig(
config: Config,
profileName: string | undefined,
): ObserveProfileConfig {
const result: ObserveProfileConfig = {};
if (!profileName || !config.profiles) {
return result;
}
const profile = config.profiles[profileName];
if (!profile) {
return result;
}
// Apply preset first if referenced
if (profile.preset && config.presets) {
const preset = config.presets[profile.preset];
if (preset) {
Object.assign(result, presetToObserveConfig(preset));
}
}
// Apply profile overrides
Object.assign(result, profileToObserveConfig(profile));
return result;
}
/**
* Convert preset definition to observe config.
*/
function presetToObserveConfig(preset: PresetDefinition): ObserveProfileConfig {
return {
features: preset.features,
sampling: (preset as Record<string, unknown>).sampling as number | undefined,
blocking: (preset as Record<string, unknown>).blocking as boolean | undefined,
sinks: (preset as Record<string, unknown>).sinks as SinkConfig | undefined,
};
}
/**
* Convert profile definition to observe config.
*/
function profileToObserveConfig(profile: ProfileDefinition): ObserveProfileConfig {
return {
features: profile.features,
sampling: profile.sampling,
blocking: profile.blocking,
sinks: profile.sinks as SinkConfig | undefined,
};
}
/**
* Validate that the profile exists.
* Note: mode validation is handled by validateProfileObserveMode.
*/
function validateProfileMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-mode',
status: 'pass',
message: 'No profile specified, using default observe configuration',
};
}
if (!config.profiles || !config.profiles[profileName]) {
const available = config.profiles ? Object.keys(config.profiles).join(', ') : 'none';
return {
name: 'profile-mode',
status: 'fail',
message: `Profile "${profileName}" not found. Available profiles: ${available}`,
};
}
return {
name: 'profile-mode',
status: 'pass',
message: `Profile "${profileName}" exists`,
};
}
/**
* Validate that the profile is explicitly configured for observe mode.
*/
function validateProfileObserveMode(
config: Config,
profileName: string | undefined,
): ObserveCheck {
if (!profileName) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: 'No profile specified, mode will be determined by top-level config',
};
}
if (!config.profiles || !config.profiles[profileName]) {
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" not found — will be validated by profile-mode check`,
};
}
const profile = config.profiles[profileName];
const profileMode = profile.mode;
if (profileMode && profileMode !== 'observe') {
return {
name: 'profile-observe-mode',
status: 'fail',
message: `Profile "${profileName}" is configured for "${profileMode}" mode but observe command requires "observe" mode`,
detail: 'Change the profile mode to "observe" or use the appropriate command ' +
`for "${profileMode}" mode (e.g., apophis ${profileMode}).`,
};
}
return {
name: 'profile-observe-mode',
status: 'pass',
message: `Profile "${profileName}" is configured for observe mode`,
};
}
/**
* Validate that no qualify-only features are used in observe mode.
*/
function validateFeatures(
features: string[] | undefined,
profileName: string | undefined,
): ObserveCheck {
if (!features || features.length === 0) {
return {
name: 'feature-restrictions',
status: 'pass',
message: 'No features configured',
};
}
const invalidFeatures = features.filter(f => QUALIFY_ONLY_FEATURES.has(f));
if (invalidFeatures.length > 0) {
const profileRef = profileName ? `Profile "${profileName}"` : 'Configuration';
return {
name: 'feature-restrictions',
status: 'fail',
message: `${profileRef} references qualify-only features that cannot be used in observe mode: ${invalidFeatures.join(', ')}`,
detail: `Remove these features from the profile or preset. Qualify-only features: ${Array.from(QUALIFY_ONLY_FEATURES).join(', ')}`,
};
}
return {
name: 'feature-restrictions',
status: 'pass',
message: `All features are valid for observe mode: ${features.join(', ')}`,
};
}
/**
* Validate sampling rate is within valid bounds [0.0, 1.0].
*/
export function validateSamplingRate(sampling: number | undefined): ObserveCheck {
if (sampling === undefined || sampling === null) {
return {
name: 'sampling-rate',
status: 'pass',
message: 'No sampling rate configured, using default (1.0)',
};
}
if (typeof sampling !== 'number' || Number.isNaN(sampling)) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate must be a number, got ${typeof sampling}`,
detail: `Valid range: ${SAMPLING_MIN} to ${SAMPLING_MAX} (inclusive)`,
};
}
if (sampling < SAMPLING_MIN || sampling > SAMPLING_MAX) {
return {
name: 'sampling-rate',
status: 'fail',
message: `Sampling rate ${sampling} is out of bounds`,
detail: `Set sampling to a value between ${SAMPLING_MIN} and ${SAMPLING_MAX} (inclusive). ` +
`A rate of 0.0 disables observation, 1.0 observes all requests.`,
};
}
return {
name: 'sampling-rate',
status: 'pass',
message: `Sampling rate ${sampling} is valid`,
};
}
/**
* Validate sink configuration for the environment.
*/
function validateSinkConfig(
sinks: SinkConfig | undefined,
env: string,
config: Config,
): ObserveCheck {
// Check if environment requires sinks
const envPolicy = config.environments?.[env];
const requireSink = envPolicy?.requireSink ?? false;
if (!sinks || Object.keys(sinks).length === 0) {
if (requireSink) {
return {
name: 'sink-config',
status: 'fail',
message: `Environment "${env}" requires sink configuration but none is provided`,
detail: 'Add sinks to your profile (e.g., sinks: { logs: true }) ' +
'or set requireSink: false in the environment policy.',
};
}
return {
name: 'sink-config',
status: 'warn',
message: 'No sinks configured. Observation data will not be persisted.',
detail: 'Configure at least one sink (logs, metrics, or traces) ' +
'to capture observation data for analysis.',
};
}
const activeSinks = [];
if (sinks.logs) activeSinks.push('logs');
if (sinks.metrics) activeSinks.push('metrics');
if (sinks.traces) activeSinks.push('traces');
if (activeSinks.length === 0) {
return {
name: 'sink-config',
status: 'warn',
message: 'Sinks are configured but none are enabled. Observation data will not be persisted.',
detail: 'Set at least one of logs, metrics, or traces to true in your sink configuration.',
};
}
return {
name: 'sink-config',
status: 'pass',
message: `Active sinks: ${activeSinks.join(', ')}`,
};
}
/**
* Validate non-blocking semantics for the environment.
* Blocking is NEVER allowed in production unless explicitly enabled by a break-glass policy.
*/
function validateBlockingSemantics(
blocking: boolean | undefined,
env: string,
config: Config,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (blocking === true && isProd) {
// Check for break-glass policy override
const envPolicy = config.environments?.[env];
const allowBlocking = envPolicy?.allowBlocking ?? false;
if (!allowBlocking) {
return {
name: 'blocking-semantics',
status: 'fail',
message: `Blocking behavior is not allowed in production environment "${env}"`,
detail: 'Set blocking: false in your profile, use a non-production environment, ' +
'or set allowBlocking: true in the environment policy for break-glass scenarios.',
};
}
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in production "${env}" via break-glass policy`,
detail: 'WARNING: blocking observation can severely impact request latency. ' +
'This should only be used during active incident response.',
};
}
if (blocking === true) {
return {
name: 'blocking-semantics',
status: 'pass',
message: `Blocking behavior is enabled in non-production environment "${env}"`,
detail: 'Warning: blocking observation can increase request latency. ' +
'Only enable in environments where latency impact is acceptable.',
};
}
// blocking is false or undefined (default to non-blocking)
return {
name: 'blocking-semantics',
status: 'pass',
message: `Non-blocking semantics confirmed for environment "${env}"`,
detail: 'Observation will run asynchronously without blocking request handling.',
};
}
/**
* Validate environment policy explicitly allows observe mode.
*/
function validateEnvironmentPolicy(
config: Config,
env: string,
): ObserveCheck {
const envPolicy = config.environments?.[env];
if (!envPolicy) {
// No explicit policy for this environment — warn but don't fail
return {
name: 'environment-policy',
status: 'pass',
message: `No environment policy defined for "${env}"`,
detail: 'Observe mode is allowed by default when no policy is configured.',
};
}
const allowObserve = envPolicy.allowObserve;
if (allowObserve === false) {
return {
name: 'environment-policy',
status: 'fail',
message: `Environment policy for "${env}" explicitly blocks observe mode`,
detail: 'Set allowObserve: true in the environment policy to enable observe mode, ' +
'or run in an environment where observe is allowed.',
};
}
return {
name: 'environment-policy',
status: 'pass',
message: `Environment "${env}" explicitly allows observe mode`,
};
}
/**
* Validate environment-specific safety constraints.
*/
function validateEnvironmentSafety(
env: string,
profileConfig: ObserveProfileConfig,
): ObserveCheck {
const isProd = env === 'production' || env === 'prod';
if (isProd) {
const warnings = [];
if (profileConfig.sampling === undefined) {
warnings.push('sampling rate not configured (will use default 1.0)');
}
if (!profileConfig.sinks) {
warnings.push('no sinks configured');
}
if (warnings.length > 0) {
return {
name: 'environment-safety',
status: 'warn',
message: `Production environment "${env}" observe configuration has warnings: ${warnings.join(', ')}`,
detail: 'In production, configure explicit sampling rate and sinks ' +
'to control observation overhead and ensure data capture.',
};
}
}
return {
name: 'environment-safety',
status: 'pass',
message: `Environment "${env}" safety checks passed`,
};
}
// ---------------------------------------------------------------------------
// Exports for testing
// ---------------------------------------------------------------------------
export {
QUALIFY_ONLY_FEATURES,
SAMPLING_MIN,
SAMPLING_MAX,
};
src/cli/core/index.ts
+1 -24
View File
@@ -13,7 +13,6 @@ const HELP_HEADER = `
${pc.dim('Commands:')}
init Scaffold config, scripts, and example usage
verify Run deterministic contract verification
observe Validate runtime observe configuration and reporting setup
qualify Run scenario, stateful, protocol, or chaos-driven qualification
replay Replay a failure using seed and stored trace
doctor Validate config, environment safety, docs/example correctness
@@ -37,7 +36,6 @@ const HELP_HEADER = `
${pc.dim('Examples:')}
apophis init --preset safe-ci
apophis verify --profile quick --routes "POST /users"
apophis observe --profile staging-observe --check-config
apophis qualify --profile oauth-nightly --seed 42
apophis replay --artifact reports/apophis/failure-*.json
apophis doctor
@@ -80,20 +78,6 @@ function getCommandHelp(command: string): string {
apophis verify --routes "POST /users" --seed 42
apophis verify --changed
apophis verify --workspace
`,
observe: `
${pc.bold('apophis observe')} — Validate runtime observe configuration and reporting setup
${pc.dim('Usage:')}
apophis observe [options]
${pc.dim('Options:')}
--profile <name> Profile name from config
--check-config Only validate, do not activate
${pc.dim('Examples:')}
apophis observe --profile staging-observe
apophis observe --check-config
`,
qualify: `
${pc.bold('apophis qualify')} — Run scenario, stateful, protocol, or chaos-driven qualification
@@ -203,13 +187,12 @@ function writeMachineRecord(
process.stdout.write(JSON.stringify(payload) + '\n');
}
type CommandName = 'init' | 'verify' | 'observe' | 'qualify' | 'replay' | 'doctor' | 'migrate';
type CommandName = 'init' | 'verify' | 'qualify' | 'replay' | 'doctor' | 'migrate';
type CommandHandler = (args: string[], ctx: CliContext) => Promise<number>;
const commandLoaders: Record<CommandName, () => Promise<CommandHandler>> = {
init: async () => (await import('../commands/init/index.js')).handleInit,
verify: async () => (await import('../commands/verify/index.js')).handleVerify,
observe: async () => (await import('../commands/observe/index.js')).handleObserve,
qualify: async () => (await import('../commands/qualify/index.js')).handleQualify,
replay: async () => (await import('../commands/replay/index.js')).handleReplay,
doctor: async () => (await import('../commands/doctor/index.js')).handleDoctor,
@@ -254,7 +237,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
const commands = [
'init',
'verify',
'observe',
'qualify',
'replay',
'doctor',
@@ -277,10 +259,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
cmd.option('--seed <number>', 'Deterministic seed');
cmd.option('--changed', 'Filter to git-modified routes');
break;
case 'observe':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--check-config', 'Only validate, do not activate');
break;
case 'qualify':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--seed <number>', 'Deterministic seed');
@@ -382,7 +360,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
const commandSpecificFlags: Record<string, Set<string>> = {
init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']),
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config']),
qualify: new Set(['--profile', '--seed', '--changed']),
replay: new Set(['--artifact', '--route']),
doctor: new Set(['--mode', '--strict', '--workspace']),
src/test/cli/acceptance.test.ts
-6
View File
@@ -83,12 +83,6 @@ test('acceptance matrix routes through CLI main entrypoint', async () => {
exitClass: 'success',
requiredSignals: ['Summary', 'Seed: 42'],
},
{
name: 'observe profile succeeds on observe-config',
args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe'],
exitClass: 'success',
requiredSignals: ['Observe mode ready for environment'],
},
{
name: 'qualify runs protocol-lab scenario flow',
args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42'],
src/test/cli/dispatch.test.ts
-1
View File
@@ -33,7 +33,6 @@ function captureOutput<T>(fn: () => Promise<T>): Promise<{ result: T; stdout: st
const commands = [
'init',
'verify',
'observe',
'qualify',
'replay',
'doctor',
src/test/cli/docs-smoke.test.ts
+1 -1
View File
@@ -190,7 +190,7 @@ test('docs contain expected CLI commands', () => {
const docsDir = resolve(process.cwd(), 'docs');
const files = findMarkdownFiles(docsDir);
const commandNames = ['init', 'verify', 'observe', 'qualify', 'replay', 'doctor', 'migrate'];
const commandNames = ['init', 'verify', 'qualify', 'replay', 'doctor', 'migrate'];
const foundCommands = new Set<string>();
for (const file of files) {
src/test/cli/goldens.test.ts
-20
View File
@@ -49,7 +49,6 @@ test('global --help matches golden snapshot', async () => {
assert.ok(output.includes('apophis'), 'Should include apophis');
assert.ok(output.includes('init'), 'Should include init command');
assert.ok(output.includes('verify'), 'Should include verify command');
assert.ok(output.includes('observe'), 'Should include observe command');
assert.ok(output.includes('qualify'), 'Should include qualify command');
assert.ok(output.includes('replay'), 'Should include replay command');
assert.ok(output.includes('doctor'), 'Should include doctor command');
@@ -75,24 +74,6 @@ test('verify --help matches golden snapshot', async () => {
assert.ok(output.includes('--seed'), 'Should include --seed');
});
test('observe --help matches golden snapshot', async () => {
const golden = readGolden('observe-help.txt');
const originalLog = console.log;
let output = '';
console.log = (msg: string) => { output += msg + '\n'; };
try {
await main(['observe', '--help']);
} finally {
console.log = originalLog;
}
assert.ok(output.includes('apophis observe'), 'Should include observe header');
assert.ok(output.includes('--profile'), 'Should include --profile');
assert.ok(output.includes('--check-config'), 'Should include --check-config');
});
test('qualify --help matches golden snapshot', async () => {
const golden = readGolden('qualify-help.txt');
@@ -199,7 +180,6 @@ test('all golden files are accounted for', () => {
'help.txt',
'verify-help.txt',
'verify-failure.txt',
'observe-help.txt',
'qualify-help.txt',
'replay-help.txt',
'doctor-help.txt',
src/test/cli/observe-safety.test.ts
-980
View File
@@ -1,980 +0,0 @@
/**
* WS3: Observe mode safety hardening tests
*
* Comprehensive boundary and edge-case tests for observe mode safety.
* Every safety violation must return exit code 2 with clear boundary explanation.
*/
import { test } from 'node:test';
import assert from 'node:assert';
import { observeCommand } from '../../cli/commands/observe/index.js';
import { USAGE_ERROR } from '../../cli/core/exit-codes.js';
import { createTestContext, writeTempConfig, cleanupTempDir } from './helpers.js';
// ---------------------------------------------------------------------------
// Test 1: Blocking in production without break-glass
// ---------------------------------------------------------------------------
test('blocking in production without break-glass fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-prod`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
blocking: true,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
allowBlocking: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('Blocking'), 'Should mention blocking boundary');
assert.ok(result.message!.includes('production'), 'Should mention production');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 2: Blocking in production WITH break-glass (should pass)
// ---------------------------------------------------------------------------
test('blocking in production with break-glass policy passes', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-breakglass`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
blocking: true,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
allowBlocking: true,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('break-glass') || result.message!.includes('Blocking'), 'Should mention break-glass');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 3: Blocking in staging (should pass)
// ---------------------------------------------------------------------------
test('blocking in staging passes without break-glass', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'staging';
const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-staging`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'staging-observe': {
name: 'staging-observe',
mode: 'observe' as const,
blocking: true,
},
},
environments: {
staging: {
name: 'staging',
allowObserve: true,
allowBlocking: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'staging', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'staging-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('staging'), 'Should mention staging environment');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 4: Sampling rate = -0.1 (out of bounds)
// ---------------------------------------------------------------------------
test('sampling rate -0.1 fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-neg`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'bad-sampling': {
name: 'bad-sampling',
mode: 'observe' as const,
sampling: -0.1,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'bad-sampling', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sampling') || result.message!.includes('Sampling'), 'Should mention sampling boundary');
assert.ok(result.message!.includes('-0.1') || result.message!.includes('out of bounds'), 'Should mention the invalid value');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 5: Sampling rate = 1.5 (out of bounds)
// ---------------------------------------------------------------------------
test('sampling rate 1.5 fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-high`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'bad-sampling': {
name: 'bad-sampling',
mode: 'observe' as const,
sampling: 1.5,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'bad-sampling', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sampling') || result.message!.includes('Sampling'), 'Should mention sampling boundary');
assert.ok(result.message!.includes('1.5') || result.message!.includes('out of bounds'), 'Should mention the invalid value');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 6: Sampling rate = 1.0 (boundary, should pass)
// ---------------------------------------------------------------------------
test('sampling rate 1.0 at boundary passes', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-boundary`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'boundary-sampling': {
name: 'boundary-sampling',
mode: 'observe' as const,
sampling: 1.0,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'boundary-sampling', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
const samplingCheck = result.checks!.find(c => c.name === 'sampling-rate');
assert.ok(samplingCheck, 'Should have sampling-rate check');
assert.strictEqual(samplingCheck!.status, 'pass', 'Sampling rate 1.0 should pass');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 7: Missing sink in production (should fail)
// ---------------------------------------------------------------------------
test('missing sink in production fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-sink-prod`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
requireSink: true,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sink') || result.message!.includes('Sink'), 'Should mention sink boundary');
assert.ok(result.message!.includes('production') || result.message!.includes('requires'), 'Should mention production requirement');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 8: Missing sink in staging (should fail)
// ---------------------------------------------------------------------------
test('missing sink in staging fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'staging';
const tmpDir = `${process.cwd()}/tmp-observe-test-sink-staging`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'staging-observe': {
name: 'staging-observe',
mode: 'observe' as const,
},
},
environments: {
staging: {
name: 'staging',
allowObserve: true,
requireSink: true,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'staging', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'staging-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('sink') || result.message!.includes('Sink'), 'Should mention sink boundary');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 9: Missing sink in local (should pass or warn)
// ---------------------------------------------------------------------------
test('missing sink in local passes with warning', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'local';
const tmpDir = `${process.cwd()}/tmp-observe-test-sink-local`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'local-observe': {
name: 'local-observe',
mode: 'observe' as const,
},
},
environments: {
local: {
name: 'local',
allowObserve: true,
requireSink: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'local', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'local-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
const sinkCheck = result.checks!.find(c => c.name === 'sink-config');
assert.ok(sinkCheck, 'Should have sink-config check');
assert.ok(sinkCheck!.status === 'warn' || sinkCheck!.status === 'pass', 'Should warn or pass about missing sinks in local');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 10: Profile with chaos feature
// ---------------------------------------------------------------------------
test('profile with chaos feature fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-chaos`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'chaos-observe': {
name: 'chaos-observe',
mode: 'observe' as const,
features: ['chaos'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'chaos-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('chaos'), 'Should mention chaos feature');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 11: Profile with stateful feature
// ---------------------------------------------------------------------------
test('profile with stateful feature fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-stateful`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'stateful-observe': {
name: 'stateful-observe',
mode: 'observe' as const,
features: ['stateful'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'stateful-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('stateful'), 'Should mention stateful feature');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 12: Profile with scenario feature
// ---------------------------------------------------------------------------
test('profile with scenario feature fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-scenario`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'scenario-observe': {
name: 'scenario-observe',
mode: 'observe' as const,
features: ['scenario'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'scenario-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('scenario'), 'Should mention scenario feature');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 13: Profile configured for verify mode
// ---------------------------------------------------------------------------
test('profile configured for verify mode fails with exit code 2', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-verify-mode`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'verify-profile': {
name: 'verify-profile',
mode: 'verify' as const,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'verify-profile', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('verify'), 'Should mention verify mode');
assert.ok(result.message!.includes('observe'), 'Should mention observe mode requirement');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 14: Environment policy blocking observe
// ---------------------------------------------------------------------------
test('environment policy blocking observe fails with exit code 2', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-policy-block`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
},
},
environments: {
production: {
name: 'production',
allowObserve: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'prod-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('observe'), 'Should mention observe mode');
assert.ok(result.message!.includes('blocks') || result.message!.includes('not allowed') || result.message!.includes('Policy'), 'Should mention policy blocking');
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 15: No config found
// ---------------------------------------------------------------------------
test('no config found fails with exit code 2 and suggests init', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-noconfig`;
const fs = await import('node:fs');
fs.mkdirSync(tmpDir, { recursive: true });
try {
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('No config') || result.message!.includes('init'), 'Should mention missing config and suggest init');
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
// ---------------------------------------------------------------------------
// Test 16: Preset features are resolved and validated
// ---------------------------------------------------------------------------
test('preset with qualify-only features is caught via profile resolution', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-preset-features`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'preset-observe': {
name: 'preset-observe',
mode: 'observe' as const,
preset: 'bad-preset',
},
},
presets: {
'bad-preset': {
name: 'bad-preset',
features: ['chaos', 'stateful'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'preset-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('chaos') || result.message!.includes('stateful'), 'Should mention preset features');
assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 17: Observe output explains what would be observed and why safe
// ---------------------------------------------------------------------------
test('observe activation output explains what and why safe', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-safe-output`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'safe-observe': {
name: 'safe-observe',
mode: 'observe' as const,
sampling: 0.5,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'safe-observe', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`);
assert.ok(result.message!.includes('checked') || result.message!.includes('observe'), 'Should explain what would be observed');
assert.ok(result.message!.includes('safe') || result.message!.includes('safety') || result.message!.includes('non-blocking'), 'Should explain why it is safe');
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 18: Every safety violation returns exit code 2 with boundary name
// ---------------------------------------------------------------------------
test('every safety violation returns exit code 2 with exact boundary', async () => {
const violations = [
{
name: 'blocking-prod',
config: {
mode: 'observe',
profiles: { 'blocking-prod': { name: 'blocking-prod', mode: 'observe', blocking: true } },
environments: { production: { name: 'production', allowObserve: true, allowBlocking: false } },
},
env: 'production',
profile: 'blocking-prod',
expectedBoundary: 'blocking',
},
{
name: 'sampling-neg',
config: {
mode: 'observe',
profiles: { 'sampling-neg': { name: 'sampling-neg', mode: 'observe', sampling: -0.1 } },
},
env: 'test',
profile: 'sampling-neg',
expectedBoundary: 'sampling',
},
{
name: 'missing-sink-prod',
config: {
mode: 'observe',
profiles: { 'missing-sink-prod': { name: 'missing-sink-prod', mode: 'observe' } },
environments: { production: { name: 'production', allowObserve: true, requireSink: true } },
},
env: 'production',
profile: 'missing-sink-prod',
expectedBoundary: 'sink',
},
];
for (const violation of violations) {
const tmpDir = `${process.cwd()}/tmp-observe-test-${violation.name}`;
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = violation.env;
try {
const configPath = await writeTempConfig(tmpDir, violation.config);
const ctx = createTestContext({ env: { nodeEnv: violation.env, apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: violation.profile, config: configPath },
ctx,
);
assert.strictEqual(
result.exitCode,
USAGE_ERROR,
`${violation.name}: Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`,
);
assert.ok(
result.message!.toLowerCase().includes(violation.expectedBoundary),
`${violation.name}: Should mention "${violation.expectedBoundary}" boundary: ${result.message}`,
);
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
}
});
// ---------------------------------------------------------------------------
// Test 19: No silent passes — every failure has a message
// ---------------------------------------------------------------------------
test('no silent passes — every failure includes a message', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-nosilent`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'bad-profile': {
name: 'bad-profile',
mode: 'observe' as const,
features: ['chaos'],
sampling: -0.5,
blocking: true,
},
},
environments: {
production: {
name: 'production',
allowObserve: true,
requireSink: true,
allowBlocking: false,
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
try {
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{ cwd: tmpDir, profile: 'bad-profile', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail with usage error');
assert.ok(result.message && result.message.length > 0, 'Should have a non-empty error message');
// Note: result.checks may be undefined when caught by policy engine before validator runs
if (result.checks) {
const failedChecks = result.checks.filter(c => c.status === 'fail');
assert.ok(failedChecks.length >= 3, `Should have at least 3 failed checks, got ${failedChecks.length}`);
for (const check of failedChecks) {
assert.ok(check.message.length > 0, `Check "${check.name}" should have a message`);
}
}
} finally {
process.env.NODE_ENV = originalNodeEnv;
}
} finally {
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Fixture-based scenarios preserved from observe acceptance coverage
// ---------------------------------------------------------------------------
test('fixture profile validates successfully with expected checks', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'staging-observe',
},
ctx,
);
assert.strictEqual(result.exitCode, 0, `Expected success but got: ${result.message}`);
assert.ok(result.checks && result.checks.length > 0, 'Should return validation checks');
const checkNames = result.checks!.map(c => c.name);
assert.ok(checkNames.includes('profile-mode'), 'Should check profile mode');
assert.ok(checkNames.includes('feature-restrictions'), 'Should check feature restrictions');
assert.ok(checkNames.includes('sink-config'), 'Should check sink config');
assert.ok(checkNames.includes('blocking-semantics'), 'Should check blocking semantics');
});
test('fixture check-config validates only without activation semantics', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'staging-observe',
checkConfig: true,
},
ctx,
);
assert.strictEqual(result.exitCode, 0, `Should pass validation: ${result.message}`);
assert.ok(result.message, 'Should include validation output');
assert.ok(
result.message!.includes('Config validation') || result.message!.includes('check'),
`Should indicate check mode: ${result.message}`,
);
assert.ok(
!result.message!.includes('activated') || result.message!.includes('validate'),
'Should be validation-only output',
);
});
test('fixture unknown profile lists available profiles', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'nonexistent-profile',
},
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail with unknown profile');
assert.ok(
result.message!.includes('not found') || result.message!.includes('Available'),
`Should list available profiles: ${result.message}`,
);
});
test('fixture output explains safety boundaries clearly', async () => {
const ctx = createTestContext();
const result = await observeCommand(
{
cwd: 'src/cli/__fixtures__/observe-config',
profile: 'staging-observe',
},
ctx,
);
assert.strictEqual(result.exitCode, 0);
assert.ok(result.message, 'Should include output message');
assert.ok(
result.message!.includes('safe') || result.message!.includes('safety') || result.message!.includes('non-blocking'),
`Should mention safety boundaries: ${result.message}`,
);
assert.ok(
result.message!.includes('checked') || result.message!.includes('check'),
`Should mention checks: ${result.message}`,
);
});
test('environment allowedModes restriction blocks observe mode', async () => {
const originalNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = 'production';
const tmpDir = `${process.cwd()}/tmp-observe-test-allowed-modes`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'prod-observe': {
name: 'prod-observe',
mode: 'observe' as const,
},
},
environments: {
production: {
name: 'production',
allowedModes: ['verify'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } });
const result = await observeCommand(
{
cwd: tmpDir,
profile: 'prod-observe',
config: configPath,
},
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail when observe is not in allowedModes');
assert.ok(
result.message!.includes('not allowed') || result.message!.includes('Policy'),
`Should explain policy violation: ${result.message}`,
);
} finally {
process.env.NODE_ENV = originalNodeEnv;
await cleanupTempDir(tmpDir);
}
});
// ---------------------------------------------------------------------------
// Test 20: Suggest how to fix for each boundary violation
// ---------------------------------------------------------------------------
test('each boundary violation suggests how to fix it', async () => {
const tmpDir = `${process.cwd()}/tmp-observe-test-fix-suggestions`;
try {
const config = {
mode: 'observe' as const,
profiles: {
'fixme': {
name: 'fixme',
mode: 'observe' as const,
features: ['chaos'],
},
},
};
const configPath = await writeTempConfig(tmpDir, config);
const ctx = createTestContext();
const result = await observeCommand(
{ cwd: tmpDir, profile: 'fixme', config: configPath },
ctx,
);
assert.strictEqual(result.exitCode, USAGE_ERROR);
assert.ok(
result.message!.includes('Set') ||
result.message!.includes('Change') ||
result.message!.includes('remove') ||
result.message!.includes('Remove') ||
result.message!.includes('qualify-only'),
`Should suggest how to fix: ${result.message}`
);
} finally {
await cleanupTempDir(tmpDir);
}
});
src/test/cli/packaging.test.ts
+1 -1
View File
@@ -82,7 +82,7 @@ describe('packaging', () => {
});
// For each of the 7 commands, verify they do NOT print "Not yet implemented"
const commands = ['init', 'verify', 'observe', 'qualify', 'replay', 'doctor', 'migrate'];
const commands = ['init', 'verify', 'qualify', 'replay', 'doctor', 'migrate'];
for (const cmd of commands) {
it(`${cmd} does not print "Not yet implemented"`, () => {
// Some commands may fail for config reasons; we just assert they don't say "Not yet implemented"