From 4ceb411028c022136ab12baf4ad6f4bedf7f618f Mon Sep 17 00:00:00 2001 From: John Dvorak Date: Thu, 21 May 2026 11:26:26 -0700 Subject: [PATCH] refactor: extract check-all.ts from public.ts (checkAll closure) Extracted the 629-line checkAll closure from imhotep() into a standalone makeCheckAll() factory in check-all.ts. Handles temporal dead zone by passing getUi() getter instead of ui value. public.ts reduced from 3568 lines to 916 lines (-74.3%). Now contains only: imports/re-exports, normalizeRuntime, the imhotep() factory function, property-run internals, and entry points. --- packages/imhotep-playwright/src/check-all.ts | 682 +++++++++++++++++++ packages/imhotep-playwright/src/public.ts | 639 +---------------- 2 files changed, 693 insertions(+), 628 deletions(-) create mode 100644 packages/imhotep-playwright/src/check-all.ts diff --git a/packages/imhotep-playwright/src/check-all.ts b/packages/imhotep-playwright/src/check-all.ts new file mode 100644 index 0000000..45b562a --- /dev/null +++ b/packages/imhotep-playwright/src/check-all.ts @@ -0,0 +1,682 @@ +/** + * checkAll() implementation — the assertion evaluation pipeline. + * + * Extracted from public.ts to reduce the imhotep() factory function size. + * Takes all state captured from the imhotep() closure as explicit parameters. + */ + +import type { Page } from 'playwright' +import { + FluentRelation, + parseSpec, + lowerToCanonical, + extractFOLFormulas, + compileDenseFOLToFormula, + validateRelation, +} from 'imhotep-dsl' +import type { ImhotepPage } from './page.js' +import type { ImhotepUi } from './public-types.js' +import type { ImhotepResult, ClauseResult, ImhotepDiagnostic, DiagnosticCode } from 'imhotep-core' +import type { SourceReference } from 'imhotep-core' +import type { FormulaNode } from 'imhotep-core' +import { + evaluateLogic, + registerDefaultPredicates, + type LogicEngineInput, +} from 'imhotep-solver' +import { compileAssertionsToFOL } from './fol-compiler.js' +import { attachFailureAnalysis } from 'imhotep-reporter' +import { + assertionStore, + specStore, + extractWorld, + SelectorDomainResolver, + compileCanonicalClauseToFormula, + getSelectorsFromAssertion, + getSelectorsFromFormula, + buildFluentLabel, + buildDenseLabel, + buildFOLLabel, + buildNormalizedContracts, + evaluateCardinalityAssertion, + isDuckTypedFluentQuantifier, + hasSubjectQuantifier, + buildCompatibilityReport, + maybeEmitCompatibilityWarning, + buildCompatibilityDiagnostic, + adaptFOLResultToImhotepResult, + computeRequiredFacts, + usesLayoutSpace, +} from './extraction.js' +import { formatAsLLM } from './llm-output.js' + +// --------------------------------------------------------------------------- +// Internal type for the check-all compile phase +// --------------------------------------------------------------------------- + +interface CheckItem { + clauseId: string + formula: FormulaNode + selectors: string[] + sourceRef: SourceReference + clauseLabel: string + state?: { selector: string; kind: string } + allowEmptyDomains?: boolean +} + +export interface CheckAllDeps { + /** Getter for the UI handle (lazily resolved to avoid temporal dead zone). */ + getUi: () => ImhotepUi + /** The raw Playwright page for extraction. */ + page: Page + /** The wrapped ImhotepPage for state materialization. */ + wrapped: ImhotepPage + /** Resolved cache directory (null to disable). */ + resolvedCacheDir: string | null +} + +/** + * Create the checkAll function bound to the given dependencies. + */ +export function makeCheckAll(deps: CheckAllDeps): ImhotepUi['checkAll'] { + const { getUi, page, wrapped, resolvedCacheDir } = deps + + return (async (options?: { format?: 'default' | 'llm'; includeNormalized?: boolean }) => { + const ui = getUi() + const compatibilityReport = buildCompatibilityReport(ui) + maybeEmitCompatibilityWarning(ui) + const compatibilityDiagnostic = process.env.IMHOTEP_INCLUDE_COMPAT_DIAGNOSTICS === '1' + ? buildCompatibilityDiagnostic(compatibilityReport) + : null + const format = options?.format ?? 'default' + const assertions = assertionStore.get(ui) || [] + const specs = specStore.get(ui) || [] + assertionStore.set(ui, []) + specStore.set(ui, []) + + // Build normalized contracts before we mutate the collections. + const normalizedContracts = options?.includeNormalized + ? buildNormalizedContracts(assertions, specs) + : undefined + + if (assertions.length === 0 && specs.length === 0) { + const emptyResult: ImhotepResult = { + schemaVersion: '1.0', + passed: true, + summary: '0 assertions', + clauseResults: [], + diagnostics: compatibilityDiagnostic ? [compatibilityDiagnostic] : [], + ...(normalizedContracts !== undefined ? { normalizedContracts } : {}), + } + return format === 'llm' ? formatAsLLM(emptyResult) : emptyResult + } + + // Ensure predicates are registered before evaluation. + registerDefaultPredicates() + + // Compile all check items into formulas first. + const checks: CheckItem[] = [] + const cardinalityChecks: Array<{ clauseId: string; assertion: any }> = [] + const preflightDiagnostics: ImhotepDiagnostic[] = [] + const validationClauseResults: ClauseResult[] = [] + + for (let i = 0; i < assertions.length; i++) { + const assertion = assertions[i] as any + // Cardinality assertions are evaluated outside the FOL engine. + if (assertion && ['exactlyOne', 'atLeastN', 'atMostN'].includes(assertion.kind)) { + cardinalityChecks.push({ clauseId: `clause_${i}`, assertion }) + continue + } + // Validate fluent relation options before compilation. + if (assertion instanceof FluentRelation) { + const validation = validateRelation(assertion.toAst()) + if (!validation.valid) { + for (const d of validation.diagnostics) { + preflightDiagnostics.push({ + code: d.code as DiagnosticCode, + severity: d.severity as 'error' | 'warning' | 'info', + category: (d.category as ImhotepDiagnostic['category']) ?? 'validation-error', + message: d.message, + source: 'imhotep-dsl', + related: [], + fixHints: d.fixHints ?? [], + metrics: d.metrics ?? {}, + sourceRef: { fluentIndex: i }, + clauseLabel: buildFluentLabel(assertions[i]), + }) + } + validationClauseResults.push({ + clauseId: `clause_${i}`, + status: 'error' as const, + truth: 'indeterminate' as const, + metrics: {}, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: validation.diagnostics.map((d) => d.code as string), + sourceRef: { fluentIndex: i }, + clauseLabel: buildFluentLabel(assertions[i]), + }) + continue + } + } + const formula = compileAssertionsToFOL([assertions[i]]) + if (!formula) { + preflightDiagnostics.push({ + code: 'IMH_INTERNAL_UNKNOWN_CLAUSE_KIND', + severity: 'error', + category: 'internal-error', + message: 'Assertion compilation produced no FOL formula.', + source: 'imhotep-playwright', + related: [], + fixHints: [ + 'Check that the assertion uses a supported relation or predicate.', + 'Supported relations: leftOf, rightOf, above, below, alignedWith, centeredWithin, inside, contains, overlaps.', + ], + metrics: {}, + sourceRef: { file: 'public.ts' }, + clauseLabel: buildFluentLabel(assertions[i]), + }) + continue + } + checks.push({ + clauseId: `clause_${i}`, + formula, + selectors: getSelectorsFromAssertion(assertions[i]), + sourceRef: { fluentIndex: i }, + clauseLabel: buildFluentLabel(assertions[i]), + allowEmptyDomains: + isDuckTypedFluentQuantifier(assertion) || hasSubjectQuantifier(assertion), + state: (() => { + const state = assertion?.assertion?.getState?.() + const subject = assertion?.assertion?.getSubject?.() + if (state?.kind && subject) { + return { selector: subject, kind: state.kind } + } + return undefined + })(), + }) + } + + let specBase = assertions.length + for (const source of specs) { + const parsed = parseSpec(source) + if (parsed.diagnostics.some((d) => d.severity === 'error')) { + for (const d of parsed.diagnostics) { + const hints: string[] = [ + 'Selectors must be single-quoted strings, e.g. \'[data-testid="x"]\'.', + 'Use relation keywords directly without "is": e.g. \'a\' leftOf \'b\'.', + 'Gap values require units: e.g. gap 8px.', + ] + const sourceRef: SourceReference = d.position?.start + ? { specLine: d.position.start.line, specColumn: d.position.start.column } + : {} + preflightDiagnostics.push({ + code: (d.code as DiagnosticCode) || 'IMH_PARSE_INVALID_SYNTAX', + severity: 'error', + category: 'parse-error', + message: d.message, + source: 'imhotep-dsl', + related: [], + fixHints: hints, + metrics: {}, + sourceRef, + clauseLabel: 'parse error', + }) + validationClauseResults.push({ + clauseId: `clause_${specBase++}`, + status: 'error', + truth: 'indeterminate', + metrics: {}, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: [(d.code as string) || 'IMH_PARSE_INVALID_SYNTAX'], + sourceRef, + clauseLabel: 'parse error', + }) + } + continue + } + + // FOL formula path: dense DSL forall/exists bypass canonical lowering + const folFormulas = extractFOLFormulas(parsed.ast) + for (const folAst of folFormulas) { + const solverFormula = compileDenseFOLToFormula(folAst) + const selectors = getSelectorsFromFormula(solverFormula) + const sourceRef: SourceReference = (folAst as any).position?.start + ? { specLine: (folAst as any).position.start.line, specColumn: (folAst as any).position.start.column } + : {} + checks.push({ + clauseId: `clause_${specBase++}`, + formula: solverFormula, + selectors, + sourceRef, + clauseLabel: buildFOLLabel(folAst), + allowEmptyDomains: true, + }) + } + + const canonicalClauses = lowerToCanonical(parsed.ast) + + // Group compound assertions so 'and'/'or' clauses are evaluated as one FOL formula + const clauseGroups: Array<{ + clauses: typeof canonicalClauses + operator?: 'and' | 'or' + }> = [] + let currentGroup: (typeof clauseGroups)[0] | null = null + + for (const clause of canonicalClauses) { + if (clause.compoundGroupId !== undefined && clause.compoundOperator) { + if (!currentGroup || currentGroup.operator !== clause.compoundOperator) { + currentGroup = { clauses: [], operator: clause.compoundOperator } + clauseGroups.push(currentGroup) + } + currentGroup.clauses.push(clause) + } else { + clauseGroups.push({ clauses: [clause] }) + currentGroup = null + } + } + + for (const group of clauseGroups) { + if (group.operator && group.clauses.length > 1) { + const formulas = group.clauses.map((c) => compileCanonicalClauseToFormula(c)) + let combinedFormula: FormulaNode = formulas[0] + for (let i = 1; i < formulas.length; i++) { + combinedFormula = { + type: 'FormulaNode', + kind: group.operator, + left: combinedFormula, + right: formulas[i], + } as FormulaNode + } + + const selectors = Array.from( + new Set( + group.clauses.flatMap((c) => + [c.subject, c.reference].filter((s): s is string => typeof s === 'string' && s.length > 0), + ), + ), + ) + const firstClause = group.clauses[0] + const sourceRef: SourceReference = firstClause.position?.start + ? { specLine: firstClause.position.start.line, specColumn: firstClause.position.start.column } + : {} + + checks.push({ + clauseId: `clause_${specBase++}`, + formula: combinedFormula, + selectors, + sourceRef, + clauseLabel: group.clauses.map(buildDenseLabel).join(` ${group.operator} `), + allowEmptyDomains: false, + state: (() => { + const firstState = group.clauses[0]?.state + const firstSubject = group.clauses[0]?.subject + if (!firstState?.kind || !firstSubject) return undefined + const uniform = group.clauses.every((c) => c.state?.kind === firstState.kind && c.subject === firstSubject) + return uniform ? { selector: firstSubject, kind: firstState.kind } : undefined + })(), + }) + } else { + for (const clause of group.clauses) { + const formula = compileCanonicalClauseToFormula(clause) + const sourceRef: SourceReference = clause.position?.start + ? { specLine: clause.position.start.line, specColumn: clause.position.start.column } + : {} + checks.push({ + clauseId: `clause_${specBase++}`, + formula, + selectors: [clause.subject, clause.reference].filter((s): s is string => typeof s === 'string' && s.length > 0), + sourceRef, + clauseLabel: buildDenseLabel(clause), + allowEmptyDomains: false, + state: clause.state?.kind ? { selector: clause.subject, kind: clause.state.kind } : undefined, + }) + } + } + } + + // Guard: non-empty spec with no diagnostics and no clauses = parser/version mismatch + const parsedAnyClauses = folFormulas.length > 0 || canonicalClauses.length > 0 + if (!parsedAnyClauses && source.trim().length > 0) { + const sourceRef: SourceReference = { specLine: 1, specColumn: 1 } + preflightDiagnostics.push({ + code: 'IMH_PARSE_ERROR', + severity: 'error', + category: 'parse-error', + message: 'Spec produced no clauses. This may indicate parser/version mismatch in imhotep-dsl.', + source: 'imhotep-playwright', + related: [], + fixHints: [ + 'Ensure imhotep-playwright and imhotep-dsl are installed from the same source build.', + 'If using local packages, reinstall both packages together.', + ], + metrics: {}, + sourceRef, + clauseLabel: 'parse error', + }) + validationClauseResults.push({ + clauseId: `clause_${specBase++}`, + status: 'error', + truth: 'indeterminate', + metrics: {}, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: ['IMH_PARSE_ERROR'], + sourceRef, + clauseLabel: 'parse error', + }) + } + } + + // Collect selectors from both spatial and cardinality assertions. + const cardinalitySelectors = cardinalityChecks.map((c) => c.assertion.selector as string) + const statelessChecks = checks.filter((c) => !c.state) + const statefulChecks = checks.filter((c) => !!c.state) + const allSelectors = Array.from(new Set([...statelessChecks.flatMap((c) => c.selectors), ...cardinalitySelectors])) + + // 1. Extract world and collect selector domains. + const requiredFacts = computeRequiredFacts(statelessChecks.map((c) => c.formula)) + const allowFastGeometry = !statelessChecks.some((c) => usesLayoutSpace(c.formula)) + let world: import('imhotep-solver').GeometryWorld | null = null + let selectorToIds = new Map() + let errors: ImhotepDiagnostic[] = [] + try { + if (allSelectors.length > 0 || cardinalityChecks.length > 0) { + const extracted = await extractWorld(page, allSelectors, resolvedCacheDir, requiredFacts, allowFastGeometry) + world = extracted.world + selectorToIds = extracted.selectorToIds + errors = extracted.errors + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + const firstCheck = checks[0] + const extractDiagnostic: ImhotepDiagnostic = { + code: 'IMH_EXTRACT_PROTOCOL_ERROR', + severity: 'error', + category: 'extraction-error', + message, + source: 'imhotep-playwright', + related: [], + fixHints: [ + 'Verify the page is fully loaded before running assertions.', + 'Check that selectors are valid CSS selectors or semantic references.', + 'Use ui.extract(selector) to debug selector resolution.', + ], + metrics: {}, + sourceRef: firstCheck?.sourceRef, + clauseLabel: firstCheck?.clauseLabel ?? 'extraction', + } + const emptySelectorToIds = new Map() + const cardClauseResults: ClauseResult[] = [] + const cardDiagnostics: ImhotepDiagnostic[] = [] + for (const cardCheck of cardinalityChecks) { + const cardResult = evaluateCardinalityAssertion( + cardCheck.assertion, + cardCheck.clauseId, + emptySelectorToIds, + ) + cardClauseResults.push({ + clauseId: cardResult.clauseId, + status: cardResult.status, + truth: cardResult.truth, + metrics: { + observedCount: 0, + expectedCount: cardResult.expectedCount, + selector: cardResult.selector, + } as any, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: cardResult.diagnostics.map((d) => d.code), + sourceRef: { selector: cardResult.selector } as any, + clauseLabel: cardResult.label, + }) + cardDiagnostics.push(...cardResult.diagnostics) + } + const extractDiagnostics = [ + ...(compatibilityDiagnostic ? [compatibilityDiagnostic] : []), + extractDiagnostic, + ...preflightDiagnostics, + ...cardDiagnostics, + ].map((d) => + attachFailureAnalysis(d), + ) + const extractResult: ImhotepResult = { + schemaVersion: '1.0', + passed: false, + summary: `${checks.length + cardinalityChecks.length} assertion(s), ${extractDiagnostics.length} diagnostic(s)`, + clauseResults: [ + ...validationClauseResults, + ...cardClauseResults, + ...checks.map((c) => ({ + clauseId: c.clauseId, + status: 'error' as const, + truth: 'indeterminate' as const, + metrics: {}, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: ['IMH_EXTRACT_PROTOCOL_ERROR'], + sourceRef: c.sourceRef, + clauseLabel: c.clauseLabel, + })), + ], + diagnostics: extractDiagnostics, + ...(normalizedContracts !== undefined ? { normalizedContracts } : {}), + } + return format === 'llm' ? formatAsLLM(extractResult) : extractResult + } + + // 2. Evaluate cardinality assertions using extracted selector counts. + const allClauseResults: ClauseResult[] = [...validationClauseResults] + const allDiagnostics: ImhotepDiagnostic[] = [ + ...(compatibilityDiagnostic ? [compatibilityDiagnostic] : []), + ...preflightDiagnostics, + ] + + for (const cardCheck of cardinalityChecks) { + const cardResult = evaluateCardinalityAssertion( + cardCheck.assertion, + cardCheck.clauseId, + selectorToIds, + ) + allClauseResults.push({ + clauseId: cardResult.clauseId, + status: cardResult.status, + truth: cardResult.truth, + metrics: { + observedCount: selectorToIds.get(cardResult.selector)?.length ?? 0, + expectedCount: cardResult.expectedCount, + selector: cardResult.selector, + } as any, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: cardResult.diagnostics.map((d) => d.code), + sourceRef: { selector: cardResult.selector } as any, + clauseLabel: cardResult.label, + }) + allDiagnostics.push(...cardResult.diagnostics) + } + + // Build a set of selectors that failed to resolve (zero matches). + const failedSelectors = new Set() + const selectorErrorsBySelector = new Map() + const emittedSelectorErrorKeys = new Set() + for (const err of errors) { + const sel = (err.sourceRef as { selector?: string })?.selector + if (typeof sel === 'string') { + failedSelectors.add(sel) + const list = selectorErrorsBySelector.get(sel) ?? [] + list.push(err) + selectorErrorsBySelector.set(sel, list) + } + } + + // Partition stateless checks: those referencing a failed selector vs. clean. + const failedChecks: CheckItem[] = [] + const validChecks: CheckItem[] = [] + for (const check of statelessChecks) { + const hasMissingSelector = check.selectors.some((s) => failedSelectors.has(s)) + if (hasMissingSelector && !check.allowEmptyDomains) { + failedChecks.push(check) + } else { + validChecks.push(check) + } + } + + for (const check of failedChecks) { + allClauseResults.push({ + clauseId: check.clauseId, + status: 'error' as const, + truth: 'indeterminate' as const, + metrics: {}, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: ['IMH_SELECTOR_ZERO_MATCHES'], + sourceRef: check.sourceRef, + clauseLabel: check.clauseLabel, + }) + for (const selector of check.selectors) { + const selectorErrors = selectorErrorsBySelector.get(selector) + if (selectorErrors) { + for (const d of selectorErrors) { + const key = `${selector}:${d.code}:${d.message}` + if (!emittedSelectorErrorKeys.has(key)) { + emittedSelectorErrorKeys.add(key) + allDiagnostics.push(d) + } + } + } + } + } + + // 3. Build domain resolver and evaluate clean stateless assertions through FOL. + if (validChecks.length > 0 && world) { + const resolver = new SelectorDomainResolver() + for (const [selector, ids] of selectorToIds) { + resolver.register(selector, ids) + } + for (const selector of failedSelectors) { + if (!selectorToIds.has(selector)) { + resolver.register(selector, []) + } + } + + for (let i = 0; i < validChecks.length; i++) { + const check = validChecks[i] + const input: LogicEngineInput = { + formula: check.formula, + world, + resolver, + options: { trace: false }, + } + const folResult = evaluateLogic(input) + const adapted = adaptFOLResultToImhotepResult(folResult, 1, check.clauseId, { + sourceRef: check.sourceRef, + clauseLabel: check.clauseLabel, + }) + allClauseResults.push(...adapted.clauseResults) + const checkFailed = adapted.clauseResults.some((r) => r.status !== 'pass') + if (checkFailed) { + allDiagnostics.push(...adapted.diagnostics) + } + } + } + + // 4. Evaluate stateful assertions one-by-one with explicit state materialization. + for (const check of statefulChecks) { + const stateInfo = check.state + if (!stateInfo) continue + + try { + await wrapped.materializeState(stateInfo.selector, stateInfo.kind as import('./page.js').StateSnapshot) + } catch (err) { + allClauseResults.push({ + clauseId: check.clauseId, + status: 'error' as const, + truth: 'indeterminate' as const, + metrics: {}, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: ['IMH_EXTRACT_PROTOCOL_ERROR'], + sourceRef: check.sourceRef, + clauseLabel: check.clauseLabel, + }) + allDiagnostics.push({ + code: 'IMH_EXTRACT_PROTOCOL_ERROR', + severity: 'error', + category: 'extraction-error', + message: err instanceof Error ? err.message : String(err), + source: 'imhotep-playwright', + related: [], + fixHints: [ + `Verify selector "${stateInfo.selector}" exists before applying state "${stateInfo.kind}".`, + 'Use ui.materializeState(selector, state) directly to debug state setup.', + ], + metrics: {}, + sourceRef: check.sourceRef, + clauseLabel: check.clauseLabel, + }) + continue + } + + const perCheckFacts = computeRequiredFacts([check.formula]) + const perCheckFast = !usesLayoutSpace(check.formula) + const extracted = await extractWorld(page, check.selectors, resolvedCacheDir, perCheckFacts, perCheckFast) + + const perCheckFailedSelectors = new Set() + for (const err of extracted.errors) { + const sel = (err.sourceRef as { selector?: string })?.selector + if (typeof sel === 'string') { + perCheckFailedSelectors.add(sel) + } + } + + const hasMissingSelector = check.selectors.some((s) => perCheckFailedSelectors.has(s)) + if (hasMissingSelector) { + allClauseResults.push({ + clauseId: check.clauseId, + status: 'error' as const, + truth: 'indeterminate' as const, + metrics: {}, + witness: { subjectId: 0, frameId: 0 }, + diagnostics: ['IMH_SELECTOR_ZERO_MATCHES'], + sourceRef: check.sourceRef, + clauseLabel: check.clauseLabel, + }) + allDiagnostics.push(...extracted.errors) + continue + } + + const resolver = new SelectorDomainResolver() + for (const [selector, ids] of extracted.selectorToIds) { + resolver.register(selector, ids) + } + + const folResult = evaluateLogic({ + formula: check.formula, + world: extracted.world, + resolver, + options: { trace: false }, + }) + const adapted = adaptFOLResultToImhotepResult(folResult, 1, check.clauseId, { + sourceRef: check.sourceRef, + clauseLabel: check.clauseLabel, + }) + allClauseResults.push(...adapted.clauseResults) + const checkFailed = adapted.clauseResults.some((r) => r.status !== 'pass') + if (checkFailed) { + allDiagnostics.push(...adapted.diagnostics) + } + } + + const hasPreflightErrors = preflightDiagnostics.some((d) => d.severity === 'error') + const passed = allClauseResults.every((r) => r.status === 'pass') && !allDiagnostics.some((d) => d.severity === 'error') && !hasPreflightErrors + const totalAssertions = checks.length + cardinalityChecks.length + + const analyzedDiagnostics = allDiagnostics.map((d) => + attachFailureAnalysis(d), + ) + + const result: ImhotepResult = { + schemaVersion: '1.0', + passed, + summary: `${totalAssertions} assertion(s), ${analyzedDiagnostics.length} diagnostic(s)`, + clauseResults: allClauseResults, + diagnostics: analyzedDiagnostics, + ...(normalizedContracts !== undefined ? { normalizedContracts } : {}), + } + return format === 'llm' ? formatAsLLM(result) : result + }) as ImhotepUi['checkAll'] +} diff --git a/packages/imhotep-playwright/src/public.ts b/packages/imhotep-playwright/src/public.ts index 55019a2..e1c1f2c 100644 --- a/packages/imhotep-playwright/src/public.ts +++ b/packages/imhotep-playwright/src/public.ts @@ -208,6 +208,16 @@ import { formatAsLLM } from './llm-output.js' export { formatAsLLM } +// --------------------------------------------------------------------------- +// checkAll pipeline (imported from check-all.ts) +// --------------------------------------------------------------------------- + +import { makeCheckAll, type CheckAllDeps } from './check-all.js' + +export { makeCheckAll, type CheckAllDeps } + +// --------------------------------------------------------------------------- +// Runtime normalization helper (accepts Page for backward compat) // --------------------------------------------------------------------------- // Main Public API // --------------------------------------------------------------------------- @@ -422,635 +432,8 @@ export async function imhotep( return issues }, - checkAll: (async (options?: { format?: 'default' | 'llm'; includeNormalized?: boolean }) => { - const compatibilityReport = buildCompatibilityReport(ui) - maybeEmitCompatibilityWarning(ui) - const compatibilityDiagnostic = process.env.IMHOTEP_INCLUDE_COMPAT_DIAGNOSTICS === '1' - ? buildCompatibilityDiagnostic(compatibilityReport) - : null - const format = options?.format ?? 'default' - const assertions = assertionStore.get(ui) || [] - const specs = specStore.get(ui) || [] - assertionStore.set(ui, []) - specStore.set(ui, []) + checkAll: makeCheckAll({ getUi: () => ui, page, wrapped, resolvedCacheDir }), - // Build normalized contracts before we mutate the collections. - const normalizedContracts = options?.includeNormalized - ? buildNormalizedContracts(assertions, specs) - : undefined - - if (assertions.length === 0 && specs.length === 0) { - const emptyResult: ImhotepResult = { - schemaVersion: '1.0', - passed: true, - summary: '0 assertions', - clauseResults: [], - diagnostics: compatibilityDiagnostic ? [compatibilityDiagnostic] : [], - ...(normalizedContracts !== undefined ? { normalizedContracts } : {}), - } - return format === 'llm' ? formatAsLLM(emptyResult) : emptyResult - } - - // Ensure predicates are registered before evaluation. - registerDefaultPredicates() - - // Compile all check items into formulas first. - interface CheckItem { - clauseId: string - formula: FormulaNode - selectors: string[] - sourceRef: SourceReference - clauseLabel: string - state?: { selector: string; kind: string } - allowEmptyDomains?: boolean - } - - const checks: CheckItem[] = [] - const cardinalityChecks: Array<{ clauseId: string; assertion: any }> = [] - const preflightDiagnostics: ImhotepDiagnostic[] = [] - const validationClauseResults: ClauseResult[] = [] - - for (let i = 0; i < assertions.length; i++) { - const assertion = assertions[i] as any - // Cardinality assertions are evaluated outside the FOL engine. - if (assertion && ['exactlyOne', 'atLeastN', 'atMostN'].includes(assertion.kind)) { - cardinalityChecks.push({ clauseId: `clause_${i}`, assertion }) - continue - } - // Validate fluent relation options before compilation. - if (assertion instanceof FluentRelation) { - const validation = validateRelation(assertion.toAst()) - if (!validation.valid) { - for (const d of validation.diagnostics) { - preflightDiagnostics.push({ - code: d.code as DiagnosticCode, - severity: d.severity as 'error' | 'warning' | 'info', - category: (d.category as ImhotepDiagnostic['category']) ?? 'validation-error', - message: d.message, - source: 'imhotep-dsl', - related: [], - fixHints: d.fixHints ?? [], - metrics: d.metrics ?? {}, - sourceRef: { fluentIndex: i }, - clauseLabel: buildFluentLabel(assertions[i]), - }) - } - // Emit a synthetic clause result so the caller sees the validation failure. - validationClauseResults.push({ - clauseId: `clause_${i}`, - status: 'error' as const, - truth: 'indeterminate' as const, - metrics: {}, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: validation.diagnostics.map((d) => d.code as string), - sourceRef: { fluentIndex: i }, - clauseLabel: buildFluentLabel(assertions[i]), - }) - continue - } - } - const formula = compileAssertionsToFOL([assertions[i]]) - if (!formula) { - preflightDiagnostics.push({ - code: 'IMH_INTERNAL_UNKNOWN_CLAUSE_KIND', - severity: 'error', - category: 'internal-error', - message: 'Assertion compilation produced no FOL formula.', - source: 'imhotep-playwright', - related: [], - fixHints: [ - 'Check that the assertion uses a supported relation or predicate.', - 'Supported relations: leftOf, rightOf, above, below, alignedWith, centeredWithin, inside, contains, overlaps.', - ], - metrics: {}, - sourceRef: { file: 'public.ts' }, - clauseLabel: buildFluentLabel(assertions[i]), - }) - continue - } - checks.push({ - clauseId: `clause_${i}`, - formula, - selectors: getSelectorsFromAssertion(assertions[i]), - sourceRef: { fluentIndex: i }, - clauseLabel: buildFluentLabel(assertions[i]), - allowEmptyDomains: - isDuckTypedFluentQuantifier(assertion) || hasSubjectQuantifier(assertion), - state: (() => { - const state = assertion?.assertion?.getState?.() - const subject = assertion?.assertion?.getSubject?.() - if (state?.kind && subject) { - return { selector: subject, kind: state.kind } - } - return undefined - })(), - }) - } - - let specBase = assertions.length - for (const source of specs) { - const parsed = parseSpec(source) - if (parsed.diagnostics.some((d) => d.severity === 'error')) { - for (const d of parsed.diagnostics) { - const hints: string[] = [ - 'Selectors must be single-quoted strings, e.g. \'[data-testid="x"]\'.', - 'Use relation keywords directly without "is": e.g. \'a\' leftOf \'b\'.', - 'Gap values require units: e.g. gap 8px.', - ] - const sourceRef: SourceReference = d.position?.start - ? { specLine: d.position.start.line, specColumn: d.position.start.column } - : {} - preflightDiagnostics.push({ - code: (d.code as DiagnosticCode) || 'IMH_PARSE_INVALID_SYNTAX', - severity: 'error', - category: 'parse-error', - message: d.message, - source: 'imhotep-dsl', - related: [], - fixHints: hints, - metrics: {}, - sourceRef, - clauseLabel: 'parse error', - }) - validationClauseResults.push({ - clauseId: `clause_${specBase++}`, - status: 'error', - truth: 'indeterminate', - metrics: {}, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: [(d.code as string) || 'IMH_PARSE_INVALID_SYNTAX'], - sourceRef, - clauseLabel: 'parse error', - }) - } - continue - } - - // ------------------------------------------------------------------- - // FOL formula path: dense DSL forall/exists bypass canonical lowering - // and are compiled directly to solver FormulaNode. - // ------------------------------------------------------------------- - const folFormulas = extractFOLFormulas(parsed.ast) - for (const folAst of folFormulas) { - const solverFormula = compileDenseFOLToFormula(folAst) - const selectors = getSelectorsFromFormula(solverFormula) - const sourceRef: SourceReference = (folAst as any).position?.start - ? { specLine: (folAst as any).position.start.line, specColumn: (folAst as any).position.start.column } - : {} - checks.push({ - clauseId: `clause_${specBase++}`, - formula: solverFormula, - selectors, - sourceRef, - clauseLabel: buildFOLLabel(folAst), - allowEmptyDomains: true, - }) - } - - const canonicalClauses = lowerToCanonical(parsed.ast) - - // Group compound assertions so that 'and'/'or' clauses are evaluated - // as a single FOL formula rather than independent checks. - const clauseGroups: Array<{ - clauses: typeof canonicalClauses - operator?: 'and' | 'or' - }> = [] - let currentGroup: (typeof clauseGroups)[0] | null = null - - for (const clause of canonicalClauses) { - if (clause.compoundGroupId !== undefined && clause.compoundOperator) { - if (!currentGroup || currentGroup.operator !== clause.compoundOperator) { - currentGroup = { clauses: [], operator: clause.compoundOperator } - clauseGroups.push(currentGroup) - } - currentGroup.clauses.push(clause) - } else { - clauseGroups.push({ clauses: [clause] }) - currentGroup = null - } - } - - for (const group of clauseGroups) { - if (group.operator && group.clauses.length > 1) { - const formulas = group.clauses.map((c) => compileCanonicalClauseToFormula(c)) - let combinedFormula: FormulaNode = formulas[0] - for (let i = 1; i < formulas.length; i++) { - combinedFormula = { - type: 'FormulaNode', - kind: group.operator, - left: combinedFormula, - right: formulas[i], - } as FormulaNode - } - - const selectors = Array.from( - new Set( - group.clauses.flatMap((c) => - [c.subject, c.reference].filter((s): s is string => typeof s === 'string' && s.length > 0), - ), - ), - ) - const firstClause = group.clauses[0] - const sourceRef: SourceReference = firstClause.position?.start - ? { specLine: firstClause.position.start.line, specColumn: firstClause.position.start.column } - : {} - - checks.push({ - clauseId: `clause_${specBase++}`, - formula: combinedFormula, - selectors, - sourceRef, - clauseLabel: group.clauses.map(buildDenseLabel).join(` ${group.operator} `), - allowEmptyDomains: false, - state: (() => { - const firstState = group.clauses[0]?.state - const firstSubject = group.clauses[0]?.subject - if (!firstState?.kind || !firstSubject) return undefined - const uniform = group.clauses.every((c) => c.state?.kind === firstState.kind && c.subject === firstSubject) - return uniform ? { selector: firstSubject, kind: firstState.kind } : undefined - })(), - }) - } else { - for (const clause of group.clauses) { - const formula = compileCanonicalClauseToFormula(clause) - const sourceRef: SourceReference = clause.position?.start - ? { specLine: clause.position.start.line, specColumn: clause.position.start.column } - : {} - checks.push({ - clauseId: `clause_${specBase++}`, - formula, - selectors: [clause.subject, clause.reference].filter((s): s is string => typeof s === 'string' && s.length > 0), - sourceRef, - clauseLabel: buildDenseLabel(clause), - allowEmptyDomains: false, - state: clause.state?.kind ? { selector: clause.subject, kind: clause.state.kind } : undefined, - }) - } - } - } - - // Guard against parser/version mismatch where a non-empty spec source - // yields no diagnostics and no clauses. - const parsedAnyClauses = folFormulas.length > 0 || canonicalClauses.length > 0 - if (!parsedAnyClauses && source.trim().length > 0) { - const sourceRef: SourceReference = { specLine: 1, specColumn: 1 } - preflightDiagnostics.push({ - code: 'IMH_PARSE_ERROR', - severity: 'error', - category: 'parse-error', - message: 'Spec produced no clauses. This may indicate parser/version mismatch in imhotep-dsl.', - source: 'imhotep-playwright', - related: [], - fixHints: [ - 'Ensure imhotep-playwright and imhotep-dsl are installed from the same source build.', - 'If using local packages, reinstall both packages together.', - ], - metrics: {}, - sourceRef, - clauseLabel: 'parse error', - }) - validationClauseResults.push({ - clauseId: `clause_${specBase++}`, - status: 'error', - truth: 'indeterminate', - metrics: {}, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: ['IMH_PARSE_ERROR'], - sourceRef, - clauseLabel: 'parse error', - }) - } - } - - // Collect selectors from both spatial and cardinality assertions. - const cardinalitySelectors = cardinalityChecks.map((c) => c.assertion.selector as string) - const statelessChecks = checks.filter((c) => !c.state) - const statefulChecks = checks.filter((c) => !!c.state) - const allSelectors = Array.from(new Set([...statelessChecks.flatMap((c) => c.selectors), ...cardinalitySelectors])) - - // 1. Extract world and collect selector domains. - // Compute minimal facts from compiled formulas to avoid extracting - // unused geometry, topology, styles, or fragments. - const requiredFacts = computeRequiredFacts(statelessChecks.map((c) => c.formula)) - const allowFastGeometry = !statelessChecks.some((c) => usesLayoutSpace(c.formula)) - let world: GeometryWorld | null = null - let selectorToIds = new Map() - let errors: ImhotepDiagnostic[] = [] - try { - if (allSelectors.length > 0 || cardinalityChecks.length > 0) { - const extracted = await extractWorld(page, allSelectors, resolvedCacheDir, requiredFacts, allowFastGeometry) - world = extracted.world - selectorToIds = extracted.selectorToIds - errors = extracted.errors - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err) - const firstCheck = checks[0] - const extractDiagnostic: ImhotepDiagnostic = { - code: 'IMH_EXTRACT_PROTOCOL_ERROR', - severity: 'error', - category: 'extraction-error', - message, - source: 'imhotep-playwright', - related: [], - fixHints: [ - 'Verify the page is fully loaded before running assertions.', - 'Check that selectors are valid CSS selectors or semantic references.', - 'Use ui.extract(selector) to debug selector resolution.', - ], - metrics: {}, - sourceRef: firstCheck?.sourceRef, - clauseLabel: firstCheck?.clauseLabel ?? 'extraction', - } - // Evaluate cardinality assertions even when extraction throws - // so that clauseResults always include cardinality entries. - const emptySelectorToIds = new Map() - const cardClauseResults: ClauseResult[] = [] - const cardDiagnostics: ImhotepDiagnostic[] = [] - for (const cardCheck of cardinalityChecks) { - const cardResult = evaluateCardinalityAssertion( - cardCheck.assertion, - cardCheck.clauseId, - emptySelectorToIds, - ) - cardClauseResults.push({ - clauseId: cardResult.clauseId, - status: cardResult.status, - truth: cardResult.truth, - metrics: { - observedCount: 0, - expectedCount: cardResult.expectedCount, - selector: cardResult.selector, - } as any, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: cardResult.diagnostics.map((d) => d.code), - sourceRef: { selector: cardResult.selector } as any, - clauseLabel: cardResult.label, - }) - cardDiagnostics.push(...cardResult.diagnostics) - } - const extractDiagnostics = [ - ...(compatibilityDiagnostic ? [compatibilityDiagnostic] : []), - extractDiagnostic, - ...preflightDiagnostics, - ...cardDiagnostics, - ].map((d) => - attachFailureAnalysis(d), - ) - const extractResult: ImhotepResult = { - schemaVersion: '1.0', - passed: false, - summary: `${checks.length + cardinalityChecks.length} assertion(s), ${extractDiagnostics.length} diagnostic(s)`, - clauseResults: [ - ...validationClauseResults, - ...cardClauseResults, - ...checks.map((c) => ({ - clauseId: c.clauseId, - status: 'error' as const, - truth: 'indeterminate' as const, - metrics: {}, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: ['IMH_EXTRACT_PROTOCOL_ERROR'], - sourceRef: c.sourceRef, - clauseLabel: c.clauseLabel, - })), - ], - diagnostics: extractDiagnostics, - ...(normalizedContracts !== undefined ? { normalizedContracts } : {}), - } - return format === 'llm' ? formatAsLLM(extractResult) : extractResult - } - - // 2. Evaluate cardinality assertions using extracted selector counts. - // Cardinality is evaluated regardless of extraction errors so that - // zero-match selectors still produce observedCount metrics. - const allClauseResults: ClauseResult[] = [...validationClauseResults] - const allDiagnostics: ImhotepDiagnostic[] = [ - ...(compatibilityDiagnostic ? [compatibilityDiagnostic] : []), - ...preflightDiagnostics, - ] - - for (const cardCheck of cardinalityChecks) { - const cardResult = evaluateCardinalityAssertion( - cardCheck.assertion, - cardCheck.clauseId, - selectorToIds, - ) - allClauseResults.push({ - clauseId: cardResult.clauseId, - status: cardResult.status, - truth: cardResult.truth, - metrics: { - observedCount: selectorToIds.get(cardResult.selector)?.length ?? 0, - expectedCount: cardResult.expectedCount, - selector: cardResult.selector, - } as any, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: cardResult.diagnostics.map((d) => d.code), - sourceRef: { selector: cardResult.selector } as any, - clauseLabel: cardResult.label, - }) - allDiagnostics.push(...cardResult.diagnostics) - } - - // Build a set of selectors that failed to resolve (zero matches). - // Per-assertion selector failure isolation: a missing selector must - // poison only the assertions that reference it, not the entire batch. - const failedSelectors = new Set() - const selectorErrorsBySelector = new Map() - const emittedSelectorErrorKeys = new Set() - for (const err of errors) { - const sel = (err.sourceRef as { selector?: string })?.selector - if (typeof sel === 'string') { - failedSelectors.add(sel) - const list = selectorErrorsBySelector.get(sel) ?? [] - list.push(err) - selectorErrorsBySelector.set(sel, list) - } - } - - // Partition stateless checks: those referencing a failed selector vs. clean. - const failedChecks: CheckItem[] = [] - const validChecks: CheckItem[] = [] - for (const check of statelessChecks) { - const hasMissingSelector = check.selectors.some((s) => failedSelectors.has(s)) - if (hasMissingSelector && !check.allowEmptyDomains) { - failedChecks.push(check) - } else { - validChecks.push(check) - } - } - - // Emit error results for checks referencing missing selectors. - for (const check of failedChecks) { - allClauseResults.push({ - clauseId: check.clauseId, - status: 'error' as const, - truth: 'indeterminate' as const, - metrics: {}, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: ['IMH_SELECTOR_ZERO_MATCHES'], - sourceRef: check.sourceRef, - clauseLabel: check.clauseLabel, - }) - for (const selector of check.selectors) { - const selectorErrors = selectorErrorsBySelector.get(selector) - if (selectorErrors) { - for (const d of selectorErrors) { - const key = `${selector}:${d.code}:${d.message}` - if (!emittedSelectorErrorKeys.has(key)) { - emittedSelectorErrorKeys.add(key) - allDiagnostics.push(d) - } - } - } - } - } - - // If preflight diagnostics exist (parse/compilation errors), those - // already prevented corresponding checks from entering the checks array. - // We include preflight diagnostics in the result but do not let them - // poison unrelated assertions. - - // 3. Build domain resolver and evaluate clean stateless assertions through FOL. - if (validChecks.length > 0 && world) { - const resolver = new SelectorDomainResolver() - for (const [selector, ids] of selectorToIds) { - resolver.register(selector, ids) - } - for (const selector of failedSelectors) { - if (!selectorToIds.has(selector)) { - resolver.register(selector, []) - } - } - - for (let i = 0; i < validChecks.length; i++) { - const check = validChecks[i] - const input: LogicEngineInput = { - formula: check.formula, - world, - resolver, - options: { trace: false }, - } - const folResult = evaluateLogic(input) - const adapted = adaptFOLResultToImhotepResult(folResult, 1, check.clauseId, { - sourceRef: check.sourceRef, - clauseLabel: check.clauseLabel, - }) - allClauseResults.push(...adapted.clauseResults) - // Only include diagnostics from failed checks so that OR short-circuit - // does not poison the overall result with diagnostics from absorbed branches. - const checkFailed = adapted.clauseResults.some((r) => r.status !== 'pass') - if (checkFailed) { - allDiagnostics.push(...adapted.diagnostics) - } - } - } - - // 4. Evaluate stateful assertions one-by-one with explicit state materialization. - for (const check of statefulChecks) { - const stateInfo = check.state - if (!stateInfo) continue - - try { - await wrapped.materializeState(stateInfo.selector, stateInfo.kind as import('./page.js').StateSnapshot) - } catch (err) { - allClauseResults.push({ - clauseId: check.clauseId, - status: 'error' as const, - truth: 'indeterminate' as const, - metrics: {}, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: ['IMH_EXTRACT_PROTOCOL_ERROR'], - sourceRef: check.sourceRef, - clauseLabel: check.clauseLabel, - }) - allDiagnostics.push({ - code: 'IMH_EXTRACT_PROTOCOL_ERROR', - severity: 'error', - category: 'extraction-error', - message: err instanceof Error ? err.message : String(err), - source: 'imhotep-playwright', - related: [], - fixHints: [ - `Verify selector "${stateInfo.selector}" exists before applying state "${stateInfo.kind}".`, - 'Use ui.materializeState(selector, state) directly to debug state setup.', - ], - metrics: {}, - sourceRef: check.sourceRef, - clauseLabel: check.clauseLabel, - }) - continue - } - - const perCheckFacts = computeRequiredFacts([check.formula]) - const perCheckFast = !usesLayoutSpace(check.formula) - const extracted = await extractWorld(page, check.selectors, resolvedCacheDir, perCheckFacts, perCheckFast) - - const perCheckFailedSelectors = new Set() - for (const err of extracted.errors) { - const sel = (err.sourceRef as { selector?: string })?.selector - if (typeof sel === 'string') { - perCheckFailedSelectors.add(sel) - } - } - - const hasMissingSelector = check.selectors.some((s) => perCheckFailedSelectors.has(s)) - if (hasMissingSelector) { - allClauseResults.push({ - clauseId: check.clauseId, - status: 'error' as const, - truth: 'indeterminate' as const, - metrics: {}, - witness: { subjectId: 0, frameId: 0 }, - diagnostics: ['IMH_SELECTOR_ZERO_MATCHES'], - sourceRef: check.sourceRef, - clauseLabel: check.clauseLabel, - }) - allDiagnostics.push(...extracted.errors) - continue - } - - const resolver = new SelectorDomainResolver() - for (const [selector, ids] of extracted.selectorToIds) { - resolver.register(selector, ids) - } - - const folResult = evaluateLogic({ - formula: check.formula, - world: extracted.world, - resolver, - options: { trace: false }, - }) - const adapted = adaptFOLResultToImhotepResult(folResult, 1, check.clauseId, { - sourceRef: check.sourceRef, - clauseLabel: check.clauseLabel, - }) - allClauseResults.push(...adapted.clauseResults) - const checkFailed = adapted.clauseResults.some((r) => r.status !== 'pass') - if (checkFailed) { - allDiagnostics.push(...adapted.diagnostics) - } - } - - const hasPreflightErrors = preflightDiagnostics.some((d) => d.severity === 'error') - const passed = allClauseResults.every((r) => r.status === 'pass') && !allDiagnostics.some((d) => d.severity === 'error') && !hasPreflightErrors - const totalAssertions = checks.length + cardinalityChecks.length - - const analyzedDiagnostics = allDiagnostics.map((d) => - attachFailureAnalysis(d), - ) - - const result: ImhotepResult = { - schemaVersion: '1.0', - passed, - summary: `${totalAssertions} assertion(s), ${analyzedDiagnostics.length} diagnostic(s)`, - clauseResults: allClauseResults, - diagnostics: analyzedDiagnostics, - ...(normalizedContracts !== undefined ? { normalizedContracts } : {}), - } - return format === 'llm' ? formatAsLLM(result) : result - }) as ImhotepUi['checkAll'], extract: (subject: string | SubjectReference): Promise => wrapped.extract(subject),