From fe8e94fbaaa0068696908add7d6f9ef76157c405 Mon Sep 17 00:00:00 2001 From: John Dvorak Date: Fri, 22 May 2026 13:47:18 -0700 Subject: [PATCH] =?UTF-8?q?chore:=20remove=20observe=20CLI=20command=20?= =?UTF-8?q?=E2=80=94=20redundant=20with=20doctor=20+=20programmatic=20plug?= =?UTF-8?q?in?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli/commands/observe/index.ts | 351 --------- src/cli/commands/observe/validator.ts | 539 -------------- src/cli/core/index.ts | 25 +- src/test/cli/acceptance.test.ts | 6 - src/test/cli/dispatch.test.ts | 1 - src/test/cli/docs-smoke.test.ts | 2 +- src/test/cli/goldens.test.ts | 20 - src/test/cli/observe-safety.test.ts | 980 -------------------------- src/test/cli/packaging.test.ts | 2 +- 9 files changed, 3 insertions(+), 1923 deletions(-) delete mode 100644 src/cli/commands/observe/index.ts delete mode 100644 src/cli/commands/observe/validator.ts delete mode 100644 src/test/cli/observe-safety.test.ts diff --git a/src/cli/commands/observe/index.ts b/src/cli/commands/observe/index.ts deleted file mode 100644 index a0a9e45..0000000 --- a/src/cli/commands/observe/index.ts +++ /dev/null @@ -1,351 +0,0 @@ -/** - * S5: Observe thread - Observe command handler - * - * Responsibilities: - * - Load config and resolve profile - * - Validate observe configuration - * - Check reporting sink setup (logs, metrics, traces) - * - Validate non-blocking semantics - * - Environment safety checks (block blocking behavior in prod by default) - * - Support --check-config (validate only, don't activate) - * - Explain what would be checked and why it is safe - * - Clear output about safety boundaries - * - Exit 0 on valid config, 2 on safety violation - */ - -import type { CliContext } from '../../core/context.js'; -import { loadConfig } from '../../core/config-loader.js'; -import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'; -import { SUCCESS, USAGE_ERROR } from '../../core/exit-codes.js'; -import { validateObserveConfig } from './validator.js'; -import { renderDoctorChecks } from '../../renderers/human.js'; -import { renderJson } from '../../renderers/json.js'; -import type { OutputContext } from '../../renderers/shared.js'; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface ObserveOptions { - profile?: string; - checkConfig?: boolean; - config?: string; - cwd?: string; - format?: 'human' | 'json' | 'ndjson' | 'json-summary' | 'ndjson-summary'; - quiet?: boolean; - verbose?: boolean; -} - -export interface ObserveResult { - exitCode: number; - message?: string; - checks?: Array<{ - name: string; - status: 'pass' | 'fail' | 'warn'; - message: string; - detail?: string; - }>; -} - -// --------------------------------------------------------------------------- -// Command handler -// --------------------------------------------------------------------------- - -/** - * Main observe command handler. - * - * Flow: - * 1. Load and resolve config - * 2. Run policy engine checks - * 3. Validate observe-specific configuration - * 4. If --check-config, stop after validation - * 5. Otherwise, report what would be activated and why it is safe - * 6. Return appropriate exit code - */ -export async function observeCommand( - options: ObserveOptions, - ctx: CliContext, -): Promise { - const { profile, checkConfig, config: configPath, cwd } = options; - const workingDir = cwd || ctx.cwd; - - // Detect environment from context - const env = detectEnvironment(); - - try { - // 1. Load config - const loadResult = await loadConfig({ - cwd: workingDir, - configPath, - profileName: profile, - env, - }); - - if (!loadResult.configPath) { - return { - exitCode: USAGE_ERROR, - message: 'No config found. Run "apophis init" to create one.', - }; - } - - const config = loadResult.config; - - // 2. Run policy engine checks - const policyEngine = new PolicyEngine({ - config, - env, - mode: 'observe', - profileName: profile || undefined, - presetName: loadResult.presetName || undefined, - }); - - const policyResult = policyEngine.check(); - - if (!policyResult.allowed) { - const message = [ - 'Policy check failed:', - ...policyResult.errors.map(e => ` ✗ ${e}`), - ].join('\n'); - - return { - exitCode: USAGE_ERROR, - message, - }; - } - - // 3. Validate observe-specific configuration - const validationResult = validateObserveConfig(config, profile || undefined, env); - - if (!validationResult.valid) { - const message = formatValidationOutput(validationResult, { checkConfig, env, profile }); - return { - exitCode: USAGE_ERROR, - message, - checks: validationResult.checks, - }; - } - - // 4. If --check-config, stop after validation with success - if (checkConfig) { - const message = formatValidationOutput(validationResult, { - checkConfig: true, - env, - profile, - }); - return { - exitCode: SUCCESS, - message, - checks: validationResult.checks, - }; - } - - // 5. Report what would be activated and why it is safe - const activationMessage = formatActivationOutput(validationResult, { - env, - profile, - configPath: loadResult.configPath, - }); - - return { - exitCode: SUCCESS, - message: activationMessage, - checks: validationResult.checks, - }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { - exitCode: USAGE_ERROR, - message: `Failed to run observe command: ${message}`, - checks: [], - }; - } -} - -// --------------------------------------------------------------------------- -// Output formatting -// --------------------------------------------------------------------------- - -interface FormatOptions { - checkConfig?: boolean; - env: string; - profile?: string; - configPath?: string; -} - -/** - * Format validation results for human-readable output. - */ -function formatValidationOutput( - result: import('./validator.js').ObserveValidationResult, - options: FormatOptions, -): string { - const lines: string[] = []; - - const mode = options.checkConfig ? 'Config validation' : 'Observe validation'; - lines.push(`${mode} for environment "${options.env}"`); - if (options.profile) { - lines.push(`Profile: ${options.profile}`); - } - lines.push(''); - - // Print each check - for (const check of result.checks) { - const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗'; - lines.push(` ${icon} ${check.name}: ${check.message}`); - if (check.detail) { - lines.push(` ${check.detail}`); - } - } - - lines.push(''); - - // Summary - if (result.errors.length > 0) { - lines.push(`Failed with ${result.errors.length} error(s).`); - lines.push(''); - lines.push('Safety boundaries:'); - lines.push(' - Observe mode is non-blocking by default'); - lines.push(' - Blocking behavior is prohibited in production'); - lines.push(' - Qualify-only features (chaos, stateful, etc.) are not allowed'); - lines.push(' - Sampling rate must be between 0.0 and 1.0'); - lines.push(' - Sinks must be configured when required by environment policy'); - } else if (result.warnings.length > 0) { - lines.push(`Passed with ${result.warnings.length} warning(s).`); - } else { - lines.push('All checks passed.'); - } - - return lines.join('\n'); -} - -/** - * Format activation output explaining what would be checked and why it is safe. - */ -function formatActivationOutput( - result: import('./validator.js').ObserveValidationResult, - options: FormatOptions, -): string { - const lines: string[] = []; - - lines.push(`Observe mode ready for environment "${options.env}"`); - if (options.profile) { - lines.push(`Profile: ${options.profile}`); - } - if (options.configPath) { - lines.push(`Config: ${options.configPath}`); - } - lines.push(''); - - // Print checks - for (const check of result.checks) { - const icon = check.status === 'pass' ? '✓' : check.status === 'warn' ? '⚠' : '✗'; - lines.push(` ${icon} ${check.name}: ${check.message}`); - } - - lines.push(''); - lines.push('What would be checked:'); - lines.push(' - Request/response contracts are evaluated asynchronously'); - lines.push(' - Violations are logged to configured sinks without blocking'); - lines.push(' - Sampling controls the fraction of requests observed'); - lines.push(' - Metrics and traces provide runtime visibility into contract health'); - lines.push(''); - lines.push('Why this is safe:'); - lines.push(' - Non-blocking semantics guarantee observation does not affect latency'); - lines.push(' - No chaos injection or stateful sequences are activated in observe mode'); - lines.push(' - Production environments require explicit non-blocking configuration'); - lines.push(' - All qualify-only features are blocked by validation'); - - if (result.warnings.length > 0) { - lines.push(''); - lines.push('Warnings:'); - for (const warning of result.warnings) { - lines.push(` ⚠ ${warning}`); - } - } - - lines.push(''); - lines.push('The CLI validates configuration and reports readiness.'); - lines.push('To activate runtime observation, register the APOPHIS plugin in your'); - lines.push('application with observe options:'); - lines.push(''); - lines.push(' await fastify.register(apophisPlugin, {'); - lines.push(' runtime: \'warn\','); - lines.push(' observe: { enabled: true, sinks: [...] }'); - lines.push(' });'); - lines.push(''); - lines.push('See docs/observe.md for the full programmatic activation guide.'); - - return lines.join('\n'); -} - -// --------------------------------------------------------------------------- -// CLI adapter -// --------------------------------------------------------------------------- - -/** - * Adapter that bridges the CLI framework (cac) to the observe command handler. - * This function signature matches what the CLI core expects. - */ -export async function handleObserve( - _args: string[], - ctx: CliContext, -): Promise { - const options: ObserveOptions = { - profile: ctx.options.profile || undefined, - checkConfig: false, - config: ctx.options.config || undefined, - cwd: ctx.cwd, - format: ctx.options.format as ObserveOptions['format'], - quiet: ctx.options.quiet, - verbose: ctx.options.verbose, - }; - - // Parse command-specific flags from process.argv - // cac passes these as parsed options, but we need to extract --check-config - // Since cac doesn't expose parsed command-specific flags in the options object, - // we scan process.argv directly for observe-specific flags - const argv = process.argv.slice(2); - if (argv.includes('--check-config')) { - options.checkConfig = true; - } - - const result = await observeCommand(options, ctx); - - // Output result based on format - if (!ctx.options.quiet && result.message) { - const format = options.format || ctx.options.format || 'human'; - if (format === 'json') { - console.log(renderJson({ - exitCode: result.exitCode, - checks: result.checks, - message: result.message, - })); - } else if (format === 'ndjson') { - process.stdout.write(JSON.stringify({ - type: 'run.completed', - command: 'observe', - exitCode: result.exitCode, - checks: result.checks, - message: result.message, - }) + '\n'); - } else if (format === 'json-summary') { - console.log(renderJson({ - exitCode: result.exitCode, - checks: result.checks, - message: result.message, - })); - } else if (format === 'ndjson-summary') { - process.stdout.write(JSON.stringify({ - type: 'run.completed', - command: 'observe', - exitCode: result.exitCode, - checks: result.checks, - message: result.message, - }) + '\n'); - } else { - console.log(result.message); - } - } - - return result.exitCode; -} diff --git a/src/cli/commands/observe/validator.ts b/src/cli/commands/observe/validator.ts deleted file mode 100644 index 910889c..0000000 --- a/src/cli/commands/observe/validator.ts +++ /dev/null @@ -1,539 +0,0 @@ -/** - * S5: Observe thread - Observe config validation logic - * - * Validates observe-specific configuration including: - * - Sink configuration checks (logs, metrics, traces) - * - Sampling rate validation - * - Feature restriction checks (no qualify-only features in observe) - * - Non-blocking semantics validation - */ - -import type { Config, ProfileDefinition, PresetDefinition } from '../../core/config-loader.js'; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface ObserveValidationResult { - valid: boolean; - errors: string[]; - warnings: string[]; - checks: ObserveCheck[]; -} - -export interface ObserveCheck { - name: string; - status: 'pass' | 'fail' | 'warn'; - message: string; - detail?: string; -} - -export interface SinkConfig { - logs?: boolean; - metrics?: boolean; - traces?: boolean; - endpoint?: string; -} - -export interface ObserveProfileConfig { - sampling?: number; - blocking?: boolean; - sinks?: SinkConfig; - features?: string[]; -} - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -/** Features that are only valid in qualify mode */ -const QUALIFY_ONLY_FEATURES = new Set([ - 'chaos', - 'stateful', - 'scenario', - 'outbound-mocks', - 'protocol-flow', -]); - -/** Valid sampling rate bounds */ -const SAMPLING_MIN = 0.0; -const SAMPLING_MAX = 1.0; - -// --------------------------------------------------------------------------- -// Validation functions -// --------------------------------------------------------------------------- - -/** - * Validate observe configuration for a given profile and environment. - */ -export function validateObserveConfig( - config: Config, - profileName: string | undefined, - env: string, -): ObserveValidationResult { - const errors: string[] = []; - const warnings: string[] = []; - const checks: ObserveCheck[] = []; - - // Resolve the effective profile config (preset + profile overrides) - const profileConfig = resolveObserveProfileConfig(config, profileName); - - // 1. Check profile exists and is observe mode - const profileCheck = validateProfileMode(config, profileName); - checks.push(profileCheck); - if (profileCheck.status === 'fail') { - errors.push(profileCheck.message); - } - - // 2. Check for qualify-only features (uses resolved profile config) - const featureCheck = validateFeatures(profileConfig.features, profileName); - checks.push(featureCheck); - if (featureCheck.status === 'fail') { - errors.push(featureCheck.message); - } - - // 3. Validate sampling rate (uses resolved profile config) - const samplingCheck = validateSamplingRate(profileConfig.sampling); - checks.push(samplingCheck); - if (samplingCheck.status === 'fail') { - errors.push(samplingCheck.message); - } - - // 4. Validate sink configuration (uses resolved profile config) - const sinkCheck = validateSinkConfig(profileConfig.sinks, env, config); - checks.push(sinkCheck); - if (sinkCheck.status === 'fail') { - errors.push(sinkCheck.message); - } else if (sinkCheck.status === 'warn') { - warnings.push(sinkCheck.message); - } - - // 5. Validate non-blocking semantics (uses resolved profile config) - const blockingCheck = validateBlockingSemantics(profileConfig.blocking, env, config); - checks.push(blockingCheck); - if (blockingCheck.status === 'fail') { - errors.push(blockingCheck.message); - } - - // 6. Environment policy check: must explicitly allow observe - const envPolicyCheck = validateEnvironmentPolicy(config, env); - checks.push(envPolicyCheck); - if (envPolicyCheck.status === 'fail') { - errors.push(envPolicyCheck.message); - } - - // 7. Environment safety check - const envCheck = validateEnvironmentSafety(env, profileConfig); - checks.push(envCheck); - if (envCheck.status === 'warn') { - warnings.push(envCheck.message); - } - - // 8. Profile must be configured for observe mode - const profileObserveCheck = validateProfileObserveMode(config, profileName); - checks.push(profileObserveCheck); - if (profileObserveCheck.status === 'fail') { - errors.push(profileObserveCheck.message); - } - - return { - valid: errors.length === 0, - errors, - warnings, - checks, - }; -} - -/** - * Resolve the observe-specific configuration from profile and preset. - * Preset values are applied first, then profile overrides. - */ -function resolveObserveProfileConfig( - config: Config, - profileName: string | undefined, -): ObserveProfileConfig { - const result: ObserveProfileConfig = {}; - - if (!profileName || !config.profiles) { - return result; - } - - const profile = config.profiles[profileName]; - if (!profile) { - return result; - } - - // Apply preset first if referenced - if (profile.preset && config.presets) { - const preset = config.presets[profile.preset]; - if (preset) { - Object.assign(result, presetToObserveConfig(preset)); - } - } - - // Apply profile overrides - Object.assign(result, profileToObserveConfig(profile)); - - return result; -} - -/** - * Convert preset definition to observe config. - */ -function presetToObserveConfig(preset: PresetDefinition): ObserveProfileConfig { - return { - features: preset.features, - sampling: (preset as Record).sampling as number | undefined, - blocking: (preset as Record).blocking as boolean | undefined, - sinks: (preset as Record).sinks as SinkConfig | undefined, - }; -} - -/** - * Convert profile definition to observe config. - */ -function profileToObserveConfig(profile: ProfileDefinition): ObserveProfileConfig { - return { - features: profile.features, - sampling: profile.sampling, - blocking: profile.blocking, - sinks: profile.sinks as SinkConfig | undefined, - }; -} - -/** - * Validate that the profile exists. - * Note: mode validation is handled by validateProfileObserveMode. - */ -function validateProfileMode( - config: Config, - profileName: string | undefined, -): ObserveCheck { - if (!profileName) { - return { - name: 'profile-mode', - status: 'pass', - message: 'No profile specified, using default observe configuration', - }; - } - - if (!config.profiles || !config.profiles[profileName]) { - const available = config.profiles ? Object.keys(config.profiles).join(', ') : 'none'; - return { - name: 'profile-mode', - status: 'fail', - message: `Profile "${profileName}" not found. Available profiles: ${available}`, - }; - } - - return { - name: 'profile-mode', - status: 'pass', - message: `Profile "${profileName}" exists`, - }; -} - -/** - * Validate that the profile is explicitly configured for observe mode. - */ -function validateProfileObserveMode( - config: Config, - profileName: string | undefined, -): ObserveCheck { - if (!profileName) { - return { - name: 'profile-observe-mode', - status: 'pass', - message: 'No profile specified, mode will be determined by top-level config', - }; - } - - if (!config.profiles || !config.profiles[profileName]) { - return { - name: 'profile-observe-mode', - status: 'pass', - message: `Profile "${profileName}" not found — will be validated by profile-mode check`, - }; - } - - const profile = config.profiles[profileName]; - const profileMode = profile.mode; - - if (profileMode && profileMode !== 'observe') { - return { - name: 'profile-observe-mode', - status: 'fail', - message: `Profile "${profileName}" is configured for "${profileMode}" mode but observe command requires "observe" mode`, - detail: 'Change the profile mode to "observe" or use the appropriate command ' + - `for "${profileMode}" mode (e.g., apophis ${profileMode}).`, - }; - } - - return { - name: 'profile-observe-mode', - status: 'pass', - message: `Profile "${profileName}" is configured for observe mode`, - }; -} - -/** - * Validate that no qualify-only features are used in observe mode. - */ -function validateFeatures( - features: string[] | undefined, - profileName: string | undefined, -): ObserveCheck { - if (!features || features.length === 0) { - return { - name: 'feature-restrictions', - status: 'pass', - message: 'No features configured', - }; - } - - const invalidFeatures = features.filter(f => QUALIFY_ONLY_FEATURES.has(f)); - if (invalidFeatures.length > 0) { - const profileRef = profileName ? `Profile "${profileName}"` : 'Configuration'; - return { - name: 'feature-restrictions', - status: 'fail', - message: `${profileRef} references qualify-only features that cannot be used in observe mode: ${invalidFeatures.join(', ')}`, - detail: `Remove these features from the profile or preset. Qualify-only features: ${Array.from(QUALIFY_ONLY_FEATURES).join(', ')}`, - }; - } - - return { - name: 'feature-restrictions', - status: 'pass', - message: `All features are valid for observe mode: ${features.join(', ')}`, - }; -} - -/** - * Validate sampling rate is within valid bounds [0.0, 1.0]. - */ -export function validateSamplingRate(sampling: number | undefined): ObserveCheck { - if (sampling === undefined || sampling === null) { - return { - name: 'sampling-rate', - status: 'pass', - message: 'No sampling rate configured, using default (1.0)', - }; - } - - if (typeof sampling !== 'number' || Number.isNaN(sampling)) { - return { - name: 'sampling-rate', - status: 'fail', - message: `Sampling rate must be a number, got ${typeof sampling}`, - detail: `Valid range: ${SAMPLING_MIN} to ${SAMPLING_MAX} (inclusive)`, - }; - } - - if (sampling < SAMPLING_MIN || sampling > SAMPLING_MAX) { - return { - name: 'sampling-rate', - status: 'fail', - message: `Sampling rate ${sampling} is out of bounds`, - detail: `Set sampling to a value between ${SAMPLING_MIN} and ${SAMPLING_MAX} (inclusive). ` + - `A rate of 0.0 disables observation, 1.0 observes all requests.`, - }; - } - - return { - name: 'sampling-rate', - status: 'pass', - message: `Sampling rate ${sampling} is valid`, - }; -} - -/** - * Validate sink configuration for the environment. - */ -function validateSinkConfig( - sinks: SinkConfig | undefined, - env: string, - config: Config, -): ObserveCheck { - // Check if environment requires sinks - const envPolicy = config.environments?.[env]; - const requireSink = envPolicy?.requireSink ?? false; - - if (!sinks || Object.keys(sinks).length === 0) { - if (requireSink) { - return { - name: 'sink-config', - status: 'fail', - message: `Environment "${env}" requires sink configuration but none is provided`, - detail: 'Add sinks to your profile (e.g., sinks: { logs: true }) ' + - 'or set requireSink: false in the environment policy.', - }; - } - - return { - name: 'sink-config', - status: 'warn', - message: 'No sinks configured. Observation data will not be persisted.', - detail: 'Configure at least one sink (logs, metrics, or traces) ' + - 'to capture observation data for analysis.', - }; - } - - const activeSinks = []; - if (sinks.logs) activeSinks.push('logs'); - if (sinks.metrics) activeSinks.push('metrics'); - if (sinks.traces) activeSinks.push('traces'); - - if (activeSinks.length === 0) { - return { - name: 'sink-config', - status: 'warn', - message: 'Sinks are configured but none are enabled. Observation data will not be persisted.', - detail: 'Set at least one of logs, metrics, or traces to true in your sink configuration.', - }; - } - - return { - name: 'sink-config', - status: 'pass', - message: `Active sinks: ${activeSinks.join(', ')}`, - }; -} - -/** - * Validate non-blocking semantics for the environment. - * Blocking is NEVER allowed in production unless explicitly enabled by a break-glass policy. - */ -function validateBlockingSemantics( - blocking: boolean | undefined, - env: string, - config: Config, -): ObserveCheck { - const isProd = env === 'production' || env === 'prod'; - - if (blocking === true && isProd) { - // Check for break-glass policy override - const envPolicy = config.environments?.[env]; - const allowBlocking = envPolicy?.allowBlocking ?? false; - - if (!allowBlocking) { - return { - name: 'blocking-semantics', - status: 'fail', - message: `Blocking behavior is not allowed in production environment "${env}"`, - detail: 'Set blocking: false in your profile, use a non-production environment, ' + - 'or set allowBlocking: true in the environment policy for break-glass scenarios.', - }; - } - - return { - name: 'blocking-semantics', - status: 'pass', - message: `Blocking behavior is enabled in production "${env}" via break-glass policy`, - detail: 'WARNING: blocking observation can severely impact request latency. ' + - 'This should only be used during active incident response.', - }; - } - - if (blocking === true) { - return { - name: 'blocking-semantics', - status: 'pass', - message: `Blocking behavior is enabled in non-production environment "${env}"`, - detail: 'Warning: blocking observation can increase request latency. ' + - 'Only enable in environments where latency impact is acceptable.', - }; - } - - // blocking is false or undefined (default to non-blocking) - return { - name: 'blocking-semantics', - status: 'pass', - message: `Non-blocking semantics confirmed for environment "${env}"`, - detail: 'Observation will run asynchronously without blocking request handling.', - }; -} - -/** - * Validate environment policy explicitly allows observe mode. - */ -function validateEnvironmentPolicy( - config: Config, - env: string, -): ObserveCheck { - const envPolicy = config.environments?.[env]; - - if (!envPolicy) { - // No explicit policy for this environment — warn but don't fail - return { - name: 'environment-policy', - status: 'pass', - message: `No environment policy defined for "${env}"`, - detail: 'Observe mode is allowed by default when no policy is configured.', - }; - } - - const allowObserve = envPolicy.allowObserve; - - if (allowObserve === false) { - return { - name: 'environment-policy', - status: 'fail', - message: `Environment policy for "${env}" explicitly blocks observe mode`, - detail: 'Set allowObserve: true in the environment policy to enable observe mode, ' + - 'or run in an environment where observe is allowed.', - }; - } - - return { - name: 'environment-policy', - status: 'pass', - message: `Environment "${env}" explicitly allows observe mode`, - }; -} - -/** - * Validate environment-specific safety constraints. - */ -function validateEnvironmentSafety( - env: string, - profileConfig: ObserveProfileConfig, -): ObserveCheck { - const isProd = env === 'production' || env === 'prod'; - - if (isProd) { - const warnings = []; - if (profileConfig.sampling === undefined) { - warnings.push('sampling rate not configured (will use default 1.0)'); - } - if (!profileConfig.sinks) { - warnings.push('no sinks configured'); - } - - if (warnings.length > 0) { - return { - name: 'environment-safety', - status: 'warn', - message: `Production environment "${env}" observe configuration has warnings: ${warnings.join(', ')}`, - detail: 'In production, configure explicit sampling rate and sinks ' + - 'to control observation overhead and ensure data capture.', - }; - } - } - - return { - name: 'environment-safety', - status: 'pass', - message: `Environment "${env}" safety checks passed`, - }; -} - -// --------------------------------------------------------------------------- -// Exports for testing -// --------------------------------------------------------------------------- - -export { - QUALIFY_ONLY_FEATURES, - SAMPLING_MIN, - SAMPLING_MAX, -}; diff --git a/src/cli/core/index.ts b/src/cli/core/index.ts index 9f7a4c5..28f0023 100644 --- a/src/cli/core/index.ts +++ b/src/cli/core/index.ts @@ -13,7 +13,6 @@ const HELP_HEADER = ` ${pc.dim('Commands:')} init Scaffold config, scripts, and example usage verify Run deterministic contract verification - observe Validate runtime observe configuration and reporting setup qualify Run scenario, stateful, protocol, or chaos-driven qualification replay Replay a failure using seed and stored trace doctor Validate config, environment safety, docs/example correctness @@ -37,7 +36,6 @@ const HELP_HEADER = ` ${pc.dim('Examples:')} apophis init --preset safe-ci apophis verify --profile quick --routes "POST /users" - apophis observe --profile staging-observe --check-config apophis qualify --profile oauth-nightly --seed 42 apophis replay --artifact reports/apophis/failure-*.json apophis doctor @@ -80,20 +78,6 @@ function getCommandHelp(command: string): string { apophis verify --routes "POST /users" --seed 42 apophis verify --changed apophis verify --workspace -`, - observe: ` - ${pc.bold('apophis observe')} — Validate runtime observe configuration and reporting setup - - ${pc.dim('Usage:')} - apophis observe [options] - - ${pc.dim('Options:')} - --profile Profile name from config - --check-config Only validate, do not activate - - ${pc.dim('Examples:')} - apophis observe --profile staging-observe - apophis observe --check-config `, qualify: ` ${pc.bold('apophis qualify')} — Run scenario, stateful, protocol, or chaos-driven qualification @@ -203,13 +187,12 @@ function writeMachineRecord( process.stdout.write(JSON.stringify(payload) + '\n'); } -type CommandName = 'init' | 'verify' | 'observe' | 'qualify' | 'replay' | 'doctor' | 'migrate'; +type CommandName = 'init' | 'verify' | 'qualify' | 'replay' | 'doctor' | 'migrate'; type CommandHandler = (args: string[], ctx: CliContext) => Promise; const commandLoaders: Record Promise> = { init: async () => (await import('../commands/init/index.js')).handleInit, verify: async () => (await import('../commands/verify/index.js')).handleVerify, - observe: async () => (await import('../commands/observe/index.js')).handleObserve, qualify: async () => (await import('../commands/qualify/index.js')).handleQualify, replay: async () => (await import('../commands/replay/index.js')).handleReplay, doctor: async () => (await import('../commands/doctor/index.js')).handleDoctor, @@ -254,7 +237,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise', 'Deterministic seed'); cmd.option('--changed', 'Filter to git-modified routes'); break; - case 'observe': - cmd.option('--profile ', 'Profile name from config'); - cmd.option('--check-config', 'Only validate, do not activate'); - break; case 'qualify': cmd.option('--profile ', 'Profile name from config'); cmd.option('--seed ', 'Deterministic seed'); @@ -382,7 +360,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise> = { init: new Set(['--preset', '-p', '--force', '-f', '--noninteractive']), verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']), - observe: new Set(['--profile', '--check-config']), qualify: new Set(['--profile', '--seed', '--changed']), replay: new Set(['--artifact', '--route']), doctor: new Set(['--mode', '--strict', '--workspace']), diff --git a/src/test/cli/acceptance.test.ts b/src/test/cli/acceptance.test.ts index 282c207..1294797 100644 --- a/src/test/cli/acceptance.test.ts +++ b/src/test/cli/acceptance.test.ts @@ -83,12 +83,6 @@ test('acceptance matrix routes through CLI main entrypoint', async () => { exitClass: 'success', requiredSignals: ['Summary', 'Seed: 42'], }, - { - name: 'observe profile succeeds on observe-config', - args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe'], - exitClass: 'success', - requiredSignals: ['Observe mode ready for environment'], - }, { name: 'qualify runs protocol-lab scenario flow', args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42'], diff --git a/src/test/cli/dispatch.test.ts b/src/test/cli/dispatch.test.ts index 3ff67b0..66c0f3a 100644 --- a/src/test/cli/dispatch.test.ts +++ b/src/test/cli/dispatch.test.ts @@ -33,7 +33,6 @@ function captureOutput(fn: () => Promise): Promise<{ result: T; stdout: st const commands = [ 'init', 'verify', - 'observe', 'qualify', 'replay', 'doctor', diff --git a/src/test/cli/docs-smoke.test.ts b/src/test/cli/docs-smoke.test.ts index ba1fef7..ce5fcc9 100644 --- a/src/test/cli/docs-smoke.test.ts +++ b/src/test/cli/docs-smoke.test.ts @@ -190,7 +190,7 @@ test('docs contain expected CLI commands', () => { const docsDir = resolve(process.cwd(), 'docs'); const files = findMarkdownFiles(docsDir); - const commandNames = ['init', 'verify', 'observe', 'qualify', 'replay', 'doctor', 'migrate']; + const commandNames = ['init', 'verify', 'qualify', 'replay', 'doctor', 'migrate']; const foundCommands = new Set(); for (const file of files) { diff --git a/src/test/cli/goldens.test.ts b/src/test/cli/goldens.test.ts index d2be111..535d894 100644 --- a/src/test/cli/goldens.test.ts +++ b/src/test/cli/goldens.test.ts @@ -49,7 +49,6 @@ test('global --help matches golden snapshot', async () => { assert.ok(output.includes('apophis'), 'Should include apophis'); assert.ok(output.includes('init'), 'Should include init command'); assert.ok(output.includes('verify'), 'Should include verify command'); - assert.ok(output.includes('observe'), 'Should include observe command'); assert.ok(output.includes('qualify'), 'Should include qualify command'); assert.ok(output.includes('replay'), 'Should include replay command'); assert.ok(output.includes('doctor'), 'Should include doctor command'); @@ -75,24 +74,6 @@ test('verify --help matches golden snapshot', async () => { assert.ok(output.includes('--seed'), 'Should include --seed'); }); -test('observe --help matches golden snapshot', async () => { - const golden = readGolden('observe-help.txt'); - - const originalLog = console.log; - let output = ''; - console.log = (msg: string) => { output += msg + '\n'; }; - - try { - await main(['observe', '--help']); - } finally { - console.log = originalLog; - } - - assert.ok(output.includes('apophis observe'), 'Should include observe header'); - assert.ok(output.includes('--profile'), 'Should include --profile'); - assert.ok(output.includes('--check-config'), 'Should include --check-config'); -}); - test('qualify --help matches golden snapshot', async () => { const golden = readGolden('qualify-help.txt'); @@ -199,7 +180,6 @@ test('all golden files are accounted for', () => { 'help.txt', 'verify-help.txt', 'verify-failure.txt', - 'observe-help.txt', 'qualify-help.txt', 'replay-help.txt', 'doctor-help.txt', diff --git a/src/test/cli/observe-safety.test.ts b/src/test/cli/observe-safety.test.ts deleted file mode 100644 index 4e0763b..0000000 --- a/src/test/cli/observe-safety.test.ts +++ /dev/null @@ -1,980 +0,0 @@ -/** - * WS3: Observe mode safety hardening tests - * - * Comprehensive boundary and edge-case tests for observe mode safety. - * Every safety violation must return exit code 2 with clear boundary explanation. - */ - -import { test } from 'node:test'; -import assert from 'node:assert'; -import { observeCommand } from '../../cli/commands/observe/index.js'; -import { USAGE_ERROR } from '../../cli/core/exit-codes.js'; -import { createTestContext, writeTempConfig, cleanupTempDir } from './helpers.js'; - -// --------------------------------------------------------------------------- -// Test 1: Blocking in production without break-glass -// --------------------------------------------------------------------------- - -test('blocking in production without break-glass fails with exit code 2', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'production'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-prod`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'prod-observe': { - name: 'prod-observe', - mode: 'observe' as const, - blocking: true, - }, - }, - environments: { - production: { - name: 'production', - allowObserve: true, - allowBlocking: false, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'prod-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('Blocking'), 'Should mention blocking boundary'); - assert.ok(result.message!.includes('production'), 'Should mention production'); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 2: Blocking in production WITH break-glass (should pass) -// --------------------------------------------------------------------------- - -test('blocking in production with break-glass policy passes', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'production'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-breakglass`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'prod-observe': { - name: 'prod-observe', - mode: 'observe' as const, - blocking: true, - }, - }, - environments: { - production: { - name: 'production', - allowObserve: true, - allowBlocking: true, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'prod-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('break-glass') || result.message!.includes('Blocking'), 'Should mention break-glass'); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 3: Blocking in staging (should pass) -// --------------------------------------------------------------------------- - -test('blocking in staging passes without break-glass', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'staging'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-blocking-staging`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'staging-observe': { - name: 'staging-observe', - mode: 'observe' as const, - blocking: true, - }, - }, - environments: { - staging: { - name: 'staging', - allowObserve: true, - allowBlocking: false, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'staging', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'staging-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('staging'), 'Should mention staging environment'); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 4: Sampling rate = -0.1 (out of bounds) -// --------------------------------------------------------------------------- - -test('sampling rate -0.1 fails with exit code 2', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-neg`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'bad-sampling': { - name: 'bad-sampling', - mode: 'observe' as const, - sampling: -0.1, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'bad-sampling', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('sampling') || result.message!.includes('Sampling'), 'Should mention sampling boundary'); - assert.ok(result.message!.includes('-0.1') || result.message!.includes('out of bounds'), 'Should mention the invalid value'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 5: Sampling rate = 1.5 (out of bounds) -// --------------------------------------------------------------------------- - -test('sampling rate 1.5 fails with exit code 2', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-high`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'bad-sampling': { - name: 'bad-sampling', - mode: 'observe' as const, - sampling: 1.5, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'bad-sampling', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('sampling') || result.message!.includes('Sampling'), 'Should mention sampling boundary'); - assert.ok(result.message!.includes('1.5') || result.message!.includes('out of bounds'), 'Should mention the invalid value'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 6: Sampling rate = 1.0 (boundary, should pass) -// --------------------------------------------------------------------------- - -test('sampling rate 1.0 at boundary passes', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-sampling-boundary`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'boundary-sampling': { - name: 'boundary-sampling', - mode: 'observe' as const, - sampling: 1.0, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'boundary-sampling', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`); - const samplingCheck = result.checks!.find(c => c.name === 'sampling-rate'); - assert.ok(samplingCheck, 'Should have sampling-rate check'); - assert.strictEqual(samplingCheck!.status, 'pass', 'Sampling rate 1.0 should pass'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 7: Missing sink in production (should fail) -// --------------------------------------------------------------------------- - -test('missing sink in production fails with exit code 2', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'production'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-sink-prod`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'prod-observe': { - name: 'prod-observe', - mode: 'observe' as const, - }, - }, - environments: { - production: { - name: 'production', - allowObserve: true, - requireSink: true, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'prod-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('sink') || result.message!.includes('Sink'), 'Should mention sink boundary'); - assert.ok(result.message!.includes('production') || result.message!.includes('requires'), 'Should mention production requirement'); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 8: Missing sink in staging (should fail) -// --------------------------------------------------------------------------- - -test('missing sink in staging fails with exit code 2', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'staging'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-sink-staging`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'staging-observe': { - name: 'staging-observe', - mode: 'observe' as const, - }, - }, - environments: { - staging: { - name: 'staging', - allowObserve: true, - requireSink: true, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'staging', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'staging-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('sink') || result.message!.includes('Sink'), 'Should mention sink boundary'); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 9: Missing sink in local (should pass or warn) -// --------------------------------------------------------------------------- - -test('missing sink in local passes with warning', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'local'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-sink-local`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'local-observe': { - name: 'local-observe', - mode: 'observe' as const, - }, - }, - environments: { - local: { - name: 'local', - allowObserve: true, - requireSink: false, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'local', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'local-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`); - const sinkCheck = result.checks!.find(c => c.name === 'sink-config'); - assert.ok(sinkCheck, 'Should have sink-config check'); - assert.ok(sinkCheck!.status === 'warn' || sinkCheck!.status === 'pass', 'Should warn or pass about missing sinks in local'); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 10: Profile with chaos feature -// --------------------------------------------------------------------------- - -test('profile with chaos feature fails with exit code 2', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-chaos`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'chaos-observe': { - name: 'chaos-observe', - mode: 'observe' as const, - features: ['chaos'], - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'chaos-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('chaos'), 'Should mention chaos feature'); - assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 11: Profile with stateful feature -// --------------------------------------------------------------------------- - -test('profile with stateful feature fails with exit code 2', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-stateful`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'stateful-observe': { - name: 'stateful-observe', - mode: 'observe' as const, - features: ['stateful'], - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'stateful-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('stateful'), 'Should mention stateful feature'); - assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 12: Profile with scenario feature -// --------------------------------------------------------------------------- - -test('profile with scenario feature fails with exit code 2', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-scenario`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'scenario-observe': { - name: 'scenario-observe', - mode: 'observe' as const, - features: ['scenario'], - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'scenario-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('scenario'), 'Should mention scenario feature'); - assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 13: Profile configured for verify mode -// --------------------------------------------------------------------------- - -test('profile configured for verify mode fails with exit code 2', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-verify-mode`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'verify-profile': { - name: 'verify-profile', - mode: 'verify' as const, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'verify-profile', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('verify'), 'Should mention verify mode'); - assert.ok(result.message!.includes('observe'), 'Should mention observe mode requirement'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 14: Environment policy blocking observe -// --------------------------------------------------------------------------- - -test('environment policy blocking observe fails with exit code 2', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'production'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-policy-block`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'prod-observe': { - name: 'prod-observe', - mode: 'observe' as const, - }, - }, - environments: { - production: { - name: 'production', - allowObserve: false, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'prod-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('observe'), 'Should mention observe mode'); - assert.ok(result.message!.includes('blocks') || result.message!.includes('not allowed') || result.message!.includes('Policy'), 'Should mention policy blocking'); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 15: No config found -// --------------------------------------------------------------------------- - -test('no config found fails with exit code 2 and suggests init', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-noconfig`; - const fs = await import('node:fs'); - - fs.mkdirSync(tmpDir, { recursive: true }); - - try { - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('No config') || result.message!.includes('init'), 'Should mention missing config and suggest init'); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } -}); - -// --------------------------------------------------------------------------- -// Test 16: Preset features are resolved and validated -// --------------------------------------------------------------------------- - -test('preset with qualify-only features is caught via profile resolution', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-preset-features`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'preset-observe': { - name: 'preset-observe', - mode: 'observe' as const, - preset: 'bad-preset', - }, - }, - presets: { - 'bad-preset': { - name: 'bad-preset', - features: ['chaos', 'stateful'], - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'preset-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, `Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('chaos') || result.message!.includes('stateful'), 'Should mention preset features'); - assert.ok(result.message!.includes('qualify-only'), 'Should mention qualify-only boundary'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 17: Observe output explains what would be observed and why safe -// --------------------------------------------------------------------------- - -test('observe activation output explains what and why safe', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-safe-output`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'safe-observe': { - name: 'safe-observe', - mode: 'observe' as const, - sampling: 0.5, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'safe-observe', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0, `Expected success, got ${result.exitCode}: ${result.message}`); - assert.ok(result.message!.includes('checked') || result.message!.includes('observe'), 'Should explain what would be observed'); - assert.ok(result.message!.includes('safe') || result.message!.includes('safety') || result.message!.includes('non-blocking'), 'Should explain why it is safe'); - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 18: Every safety violation returns exit code 2 with boundary name -// --------------------------------------------------------------------------- - -test('every safety violation returns exit code 2 with exact boundary', async () => { - const violations = [ - { - name: 'blocking-prod', - config: { - mode: 'observe', - profiles: { 'blocking-prod': { name: 'blocking-prod', mode: 'observe', blocking: true } }, - environments: { production: { name: 'production', allowObserve: true, allowBlocking: false } }, - }, - env: 'production', - profile: 'blocking-prod', - expectedBoundary: 'blocking', - }, - { - name: 'sampling-neg', - config: { - mode: 'observe', - profiles: { 'sampling-neg': { name: 'sampling-neg', mode: 'observe', sampling: -0.1 } }, - }, - env: 'test', - profile: 'sampling-neg', - expectedBoundary: 'sampling', - }, - { - name: 'missing-sink-prod', - config: { - mode: 'observe', - profiles: { 'missing-sink-prod': { name: 'missing-sink-prod', mode: 'observe' } }, - environments: { production: { name: 'production', allowObserve: true, requireSink: true } }, - }, - env: 'production', - profile: 'missing-sink-prod', - expectedBoundary: 'sink', - }, - ]; - - for (const violation of violations) { - const tmpDir = `${process.cwd()}/tmp-observe-test-${violation.name}`; - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = violation.env; - - try { - const configPath = await writeTempConfig(tmpDir, violation.config); - const ctx = createTestContext({ env: { nodeEnv: violation.env, apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: violation.profile, config: configPath }, - ctx, - ); - - assert.strictEqual( - result.exitCode, - USAGE_ERROR, - `${violation.name}: Expected exit code ${USAGE_ERROR}, got ${result.exitCode}: ${result.message}`, - ); - assert.ok( - result.message!.toLowerCase().includes(violation.expectedBoundary), - `${violation.name}: Should mention "${violation.expectedBoundary}" boundary: ${result.message}`, - ); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } - } -}); - -// --------------------------------------------------------------------------- -// Test 19: No silent passes — every failure has a message -// --------------------------------------------------------------------------- - -test('no silent passes — every failure includes a message', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-nosilent`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'bad-profile': { - name: 'bad-profile', - mode: 'observe' as const, - features: ['chaos'], - sampling: -0.5, - blocking: true, - }, - }, - environments: { - production: { - name: 'production', - allowObserve: true, - requireSink: true, - allowBlocking: false, - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'production'; - - try { - const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } }); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'bad-profile', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail with usage error'); - assert.ok(result.message && result.message.length > 0, 'Should have a non-empty error message'); - // Note: result.checks may be undefined when caught by policy engine before validator runs - if (result.checks) { - const failedChecks = result.checks.filter(c => c.status === 'fail'); - assert.ok(failedChecks.length >= 3, `Should have at least 3 failed checks, got ${failedChecks.length}`); - for (const check of failedChecks) { - assert.ok(check.message.length > 0, `Check "${check.name}" should have a message`); - } - } - - } finally { - process.env.NODE_ENV = originalNodeEnv; - } - } finally { - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Fixture-based scenarios preserved from observe acceptance coverage -// --------------------------------------------------------------------------- - -test('fixture profile validates successfully with expected checks', async () => { - const ctx = createTestContext(); - - const result = await observeCommand( - { - cwd: 'src/cli/__fixtures__/observe-config', - profile: 'staging-observe', - }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0, `Expected success but got: ${result.message}`); - assert.ok(result.checks && result.checks.length > 0, 'Should return validation checks'); - - const checkNames = result.checks!.map(c => c.name); - assert.ok(checkNames.includes('profile-mode'), 'Should check profile mode'); - assert.ok(checkNames.includes('feature-restrictions'), 'Should check feature restrictions'); - assert.ok(checkNames.includes('sink-config'), 'Should check sink config'); - assert.ok(checkNames.includes('blocking-semantics'), 'Should check blocking semantics'); -}); - -test('fixture check-config validates only without activation semantics', async () => { - const ctx = createTestContext(); - - const result = await observeCommand( - { - cwd: 'src/cli/__fixtures__/observe-config', - profile: 'staging-observe', - checkConfig: true, - }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0, `Should pass validation: ${result.message}`); - assert.ok(result.message, 'Should include validation output'); - assert.ok( - result.message!.includes('Config validation') || result.message!.includes('check'), - `Should indicate check mode: ${result.message}`, - ); - assert.ok( - !result.message!.includes('activated') || result.message!.includes('validate'), - 'Should be validation-only output', - ); -}); - -test('fixture unknown profile lists available profiles', async () => { - const ctx = createTestContext(); - - const result = await observeCommand( - { - cwd: 'src/cli/__fixtures__/observe-config', - profile: 'nonexistent-profile', - }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail with unknown profile'); - assert.ok( - result.message!.includes('not found') || result.message!.includes('Available'), - `Should list available profiles: ${result.message}`, - ); -}); - -test('fixture output explains safety boundaries clearly', async () => { - const ctx = createTestContext(); - - const result = await observeCommand( - { - cwd: 'src/cli/__fixtures__/observe-config', - profile: 'staging-observe', - }, - ctx, - ); - - assert.strictEqual(result.exitCode, 0); - assert.ok(result.message, 'Should include output message'); - assert.ok( - result.message!.includes('safe') || result.message!.includes('safety') || result.message!.includes('non-blocking'), - `Should mention safety boundaries: ${result.message}`, - ); - assert.ok( - result.message!.includes('checked') || result.message!.includes('check'), - `Should mention checks: ${result.message}`, - ); -}); - -test('environment allowedModes restriction blocks observe mode', async () => { - const originalNodeEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'production'; - - const tmpDir = `${process.cwd()}/tmp-observe-test-allowed-modes`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'prod-observe': { - name: 'prod-observe', - mode: 'observe' as const, - }, - }, - environments: { - production: { - name: 'production', - allowedModes: ['verify'], - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext({ env: { nodeEnv: 'production', apophisEnv: undefined } }); - - const result = await observeCommand( - { - cwd: tmpDir, - profile: 'prod-observe', - config: configPath, - }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR, 'Should fail when observe is not in allowedModes'); - assert.ok( - result.message!.includes('not allowed') || result.message!.includes('Policy'), - `Should explain policy violation: ${result.message}`, - ); - } finally { - process.env.NODE_ENV = originalNodeEnv; - await cleanupTempDir(tmpDir); - } -}); - -// --------------------------------------------------------------------------- -// Test 20: Suggest how to fix for each boundary violation -// --------------------------------------------------------------------------- - -test('each boundary violation suggests how to fix it', async () => { - const tmpDir = `${process.cwd()}/tmp-observe-test-fix-suggestions`; - - try { - const config = { - mode: 'observe' as const, - profiles: { - 'fixme': { - name: 'fixme', - mode: 'observe' as const, - features: ['chaos'], - }, - }, - }; - - const configPath = await writeTempConfig(tmpDir, config); - const ctx = createTestContext(); - - const result = await observeCommand( - { cwd: tmpDir, profile: 'fixme', config: configPath }, - ctx, - ); - - assert.strictEqual(result.exitCode, USAGE_ERROR); - assert.ok( - result.message!.includes('Set') || - result.message!.includes('Change') || - result.message!.includes('remove') || - result.message!.includes('Remove') || - result.message!.includes('qualify-only'), - `Should suggest how to fix: ${result.message}` - ); - } finally { - await cleanupTempDir(tmpDir); - } -}); diff --git a/src/test/cli/packaging.test.ts b/src/test/cli/packaging.test.ts index e3e5c15..08621d2 100644 --- a/src/test/cli/packaging.test.ts +++ b/src/test/cli/packaging.test.ts @@ -82,7 +82,7 @@ describe('packaging', () => { }); // For each of the 7 commands, verify they do NOT print "Not yet implemented" - const commands = ['init', 'verify', 'observe', 'qualify', 'replay', 'doctor', 'migrate']; + const commands = ['init', 'verify', 'qualify', 'replay', 'doctor', 'migrate']; for (const cmd of commands) { it(`${cmd} does not print "Not yet implemented"`, () => { // Some commands may fail for config reasons; we just assert they don't say "Not yet implemented"