/** * Reporter -> Diagnostics integration tests. * * Verifies that solver results produce correct diagnostics, that proof traces * map to human-readable output, and that witness shrinking works end-to-end. */ import { describe, it } from 'node:test' import assert from 'node:assert' import { makeDiagnostic, diagnosticFromProof, renderHumanReport, renderJsonReport, buildJsonReport, shrinkWitness, createTraceBuilder, findClauseTraces, formatDiagnosticCompact, } from 'imhotep-reporter' import type { ProofLike, Witness, ShrinkResult, } from 'imhotep-reporter' import { buildMockSolverResult } from './integration-mocks.js' describe('Reporter -> Diagnostics', () => { it('solver results produce correct diagnostics from failing proofs', () => { // Arrange: create a failing proof with predicate and witness const proof: ProofLike = { proofId: 'p1', clauseId: 'c1', outcome: 'fail', truth: 'determinate', failedPredicate: { op: '>=', left: 10, right: 24 }, witness: { subjectId: 1, referenceId: 2, envCaseId: 'env_1', snapshotId: 'default', }, } // Act: convert proof to diagnostic const diagnostic = diagnosticFromProof(proof, { idGen: () => 'd1', codeForClause: () => 'IMH_RELATION_ABOVE_FAILED', messageForClause: () => 'Expected .tooltip to be above .trigger', fixHintsForClause: () => ['Increase vertical gap to at least 24px'], }) // Assert assert.ok(diagnostic) assert.strictEqual(diagnostic!.code, 'IMH_RELATION_ABOVE_FAILED') assert.strictEqual(diagnostic!.category, 'contract-failure') assert.strictEqual(diagnostic!.severity, 'error') assert.ok( diagnostic!.related.some((rel) => rel.message.includes('Predicate'), ), ) assert.ok( diagnostic!.fixHints.some((hint) => hint.includes('gap'), ), ) }) it('indeterminate proofs produce indeterminate-result diagnostics', () => { // Arrange: create an indeterminate failing proof const proof: ProofLike = { proofId: 'p1', clauseId: 'c1', outcome: 'fail', truth: 'indeterminate', witness: { envCaseId: 'env_1' }, } // Act const diagnostic = diagnosticFromProof(proof, { idGen: () => 'd1', codeForClause: () => 'IMH_RELATION_ABOVE_FAILED', messageForClause: () => 'Could not determine relation', fixHintsForClause: () => [], }) // Assert assert.ok(diagnostic) assert.strictEqual(diagnostic!.category, 'indeterminate-result') }) it('passing proofs produce no diagnostics', () => { // Arrange: create a passing proof const proof: ProofLike = { proofId: 'p1', clauseId: 'c1', outcome: 'pass', truth: 'determinate', } // Act const diagnostic = diagnosticFromProof(proof, { idGen: () => 'd1', codeForClause: () => 'IMH_RELATION_ABOVE_FAILED', messageForClause: () => 'Should not be called', fixHintsForClause: () => [], }) // Assert assert.strictEqual(diagnostic, null) }) it('proof traces map to human-readable output', () => { // Arrange: create diagnostic linked to a trace const diagnostic = makeDiagnostic( { code: 'IMH_RELATION_LEFT_OF_FAILED', category: 'contract-failure', message: 'Button is not left of field', traceRef: 't1', position: { start: { line: 3, column: 5, offset: 20 }, end: { line: 3, column: 40, offset: 55 }, }, }, { idGen: () => 'd1' }, ) const traces = [ { traceEventId: 't1', phase: 'clause-evaluated' as const, at: 1000, refs: { clauseId: 'c1', diagnosticId: 'd1' }, payload: { status: 'fail' }, }, { traceEventId: 't2', phase: 'proof-created' as const, at: 1001, refs: { clauseId: 'c1', proofId: 'p1' }, payload: {}, }, ] // Act: render human report with traces const report = renderHumanReport( [diagnostic], traces, new Map(), { showTraces: true }, ) // Assert assert.ok(report.includes('Button is not left of field')) assert.ok(report.includes('clause-evaluated')) assert.ok(report.includes('IMH_RELATION_LEFT_OF_FAILED')) assert.ok(report.includes('line 3')) }) it('trace builder emits events findable by clause id', () => { // Arrange: create trace builder const builder = createTraceBuilder({ idGen: () => 't1', now: () => Date.now(), }) // Act: emit events builder.emit({ phase: 'clause-evaluated', refs: { clauseId: 'c1' }, payload: { status: 'pass' }, }) builder.emit({ phase: 'clause-evaluated', refs: { clauseId: 'c2' }, payload: { status: 'fail' }, }) builder.emit({ phase: 'proof-created', refs: { clauseId: 'c1', proofId: 'p1' }, payload: {}, }) // Assert: findClauseTraces returns correct subset const found = findClauseTraces(builder.events(), 'c1') assert.strictEqual(found.length, 2) assert.ok( found.every((ev) => ev.refs.clauseId === 'c1'), ) }) it('witness shrinking works end-to-end', () => { // Arrange: create a witness with redundant env cases const witness: Witness = { proof: { proofId: 'p1', clauseId: 'c1', outcome: 'fail', truth: 'determinate', }, envCases: ['env_1', 'env_2', 'env_3'], snapshots: ['default', 'hover'], subjects: [1, 2, 3], facts: [100, 101, 102], } // Act: shrink witness — only env_2 and default snapshot are needed const stillFails = (w: Witness) => w.envCases.includes('env_2') && w.snapshots.includes('default') const result = shrinkWitness(witness, stillFails) // Assert: witness was reduced assert.strictEqual(result.reduced, true) assert.deepStrictEqual(result.witness.envCases, ['env_2']) assert.deepStrictEqual(result.witness.snapshots, ['default']) assert.ok(result.axes.includes('env-case')) assert.ok(result.axes.includes('snapshot')) assert.ok(result.steps > 0) }) it('shrink results render in JSON report', () => { // Arrange: create a shrunk witness const witness: Witness = { proof: { proofId: 'p1', clauseId: 'c1', outcome: 'fail', truth: 'determinate', }, envCases: ['env_1'], snapshots: ['default'], subjects: [1], facts: [100], } const shrinkResult: ShrinkResult = { reduced: true, witness, axes: ['env-case', 'subject'], steps: 3, } const shrinkMap = new Map() shrinkMap.set('c1', shrinkResult) const diagnostic = makeDiagnostic( { code: 'IMH_TEST', category: 'contract-failure', message: 'Fail', clauseId: 'c1', }, { idGen: () => 'd1' }, ) // Act: build JSON report with shrink const report = buildJsonReport( [diagnostic], [], shrinkMap, { includeShrink: true }, ) // Assert assert.strictEqual(report.summary.errorCount, 1) assert.ok(report.shrinkResults) assert.ok(report.shrinkResults!.c1) assert.strictEqual(report.shrinkResults!.c1.reduced, true) assert.ok(report.shrinkResults!.c1.axes.includes('env-case')) // Act: stringify const json = renderJsonReport( [diagnostic], [], shrinkMap, { includeShrink: true, indent: 2 }, ) assert.ok(json.includes('shrinkResults')) assert.ok(json.includes('env-case')) }) it('solver diagnostics flow through to reporter formatting', () => { // Arrange: create solver result with diagnostics const solverResult = buildMockSolverResult({ clauseResults: [ { clauseId: 'c1', status: 'fail', truth: 'determinate', metrics: { observedGap: 4, minGap: 16 }, witness: { subjectId: 1, referenceId: 2 }, }, ], diagnostics: [ { code: 'IMH_INTERNAL_UNKNOWN_CLAUSE_KIND', severity: 'error', category: 'internal-error', message: 'Evaluation error', }, ], }) // Act: convert solver diagnostics to reporter diagnostics const diagnostics = solverResult.diagnostics.map((d) => makeDiagnostic( { code: d.code, category: d.category as any, message: d.message, severity: d.severity as any, }, { idGen: () => 'd1' }, ), ) // Assert assert.strictEqual(diagnostics.length, 1) assert.strictEqual(diagnostics[0].code, 'IMH_INTERNAL_UNKNOWN_CLAUSE_KIND') // Act: render compact const compact = formatDiagnosticCompact(diagnostics[0]) assert.ok(compact.includes('IMH_INTERNAL_UNKNOWN_CLAUSE_KIND')) }) })