v1.1.0: pooled runtime, 959 tests, production hardening (0 squash)
This commit is contained in:
John Dvorak
@@ -0,0 +1,120 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { execSync } from 'node:child_process'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
const repoRoot = path.resolve(__dirname, '..')
|
||||
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'imhotep-external-smoke-'))
|
||||
const projectDir = path.join(tmp, 'project')
|
||||
fs.mkdirSync(projectDir, { recursive: true })
|
||||
|
||||
const localDeps = {
|
||||
imhotep: path.join(repoRoot, 'packages/imhotep'),
|
||||
'imhotep-playwright': path.join(repoRoot, 'packages/imhotep-playwright'),
|
||||
'imhotep-core': path.join(repoRoot, 'packages/imhotep-core'),
|
||||
'imhotep-dsl': path.join(repoRoot, 'packages/imhotep-dsl'),
|
||||
'imhotep-extractor': path.join(repoRoot, 'packages/imhotep-extractor'),
|
||||
'imhotep-geometry': path.join(repoRoot, 'packages/imhotep-geometry'),
|
||||
'imhotep-cdp': path.join(repoRoot, 'packages/imhotep-cdp'),
|
||||
'imhotep-reporter': path.join(repoRoot, 'packages/imhotep-reporter'),
|
||||
'imhotep-solver': path.join(repoRoot, 'packages/imhotep-solver'),
|
||||
'imhotep-state': path.join(repoRoot, 'packages/imhotep-state'),
|
||||
'imhotep-topology': path.join(repoRoot, 'packages/imhotep-topology'),
|
||||
}
|
||||
|
||||
const pkg = {
|
||||
name: 'imhotep-external-smoke',
|
||||
version: '1.0.0',
|
||||
type: 'module',
|
||||
private: true,
|
||||
dependencies: Object.fromEntries(
|
||||
Object.entries(localDeps).map(([name, depPath]) => [name, `file:${depPath}`]),
|
||||
),
|
||||
devDependencies: {
|
||||
'@playwright/test': '^1.59.1',
|
||||
playwright: '^1.59.1',
|
||||
},
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(projectDir, 'package.json'), JSON.stringify(pkg, null, 2))
|
||||
|
||||
const fixtureDir = path.join(projectDir, 'fixtures')
|
||||
fs.mkdirSync(fixtureDir, { recursive: true })
|
||||
fs.writeFileSync(
|
||||
path.join(fixtureDir, 'smoke.html'),
|
||||
`<!doctype html>
|
||||
<html lang="en"><head><meta charset="UTF-8"><title>smoke</title>
|
||||
<style>
|
||||
.wrap { display:flex; gap: 16px; align-items:center; }
|
||||
.left,.right { width: 100px; height: 44px; }
|
||||
.toolbar { display:flex; gap: 8px; }
|
||||
.toolbar button { min-width: 44px; min-height: 44px; }
|
||||
</style></head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="left" data-testid="left"></div>
|
||||
<div class="right" data-testid="right"></div>
|
||||
</div>
|
||||
<div class="toolbar" data-testid="toolbar">
|
||||
<button data-testid="a">A</button>
|
||||
<button data-testid="b">B</button>
|
||||
</div>
|
||||
</body></html>`,
|
||||
)
|
||||
|
||||
execSync('npm install --no-audit --no-fund', { cwd: projectDir, stdio: 'inherit' })
|
||||
|
||||
const smokeScript = path.join(projectDir, 'smoke.mjs')
|
||||
fs.writeFileSync(
|
||||
smokeScript,
|
||||
`import assert from 'node:assert/strict'
|
||||
import { chromium } from 'playwright'
|
||||
import { imhotep, imhotepFixture, enumeratedDomain, touchTarget, toolbarAlignment } from 'imhotep'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import path from 'node:path'
|
||||
|
||||
const browser = await chromium.launch({ headless: true })
|
||||
const page = await browser.newPage({ viewport: { width: 1200, height: 800 } })
|
||||
const fixturePath = path.resolve('fixtures/smoke.html')
|
||||
await page.goto(pathToFileURL(fixturePath).href)
|
||||
|
||||
const ui = await imhotep(page, { deterministic: true, seed: 42 })
|
||||
ui.expect('[data-testid="left"]').to.be.leftOf('[data-testid="right"]', { minGap: 8 })
|
||||
touchTarget(ui, '[data-testid="a"]')
|
||||
toolbarAlignment(ui, ['[data-testid="a"]', '[data-testid="b"]'])
|
||||
|
||||
const result = await ui.checkAll({ includeNormalized: true })
|
||||
assert.equal(typeof result.schemaVersion, 'string')
|
||||
assert.ok(Array.isArray(result.normalizedContracts))
|
||||
|
||||
ui.spec("a is leftOf b")
|
||||
const parseLlm = await ui.checkAll({ format: 'llm' })
|
||||
const parsePayload = JSON.parse(parseLlm)
|
||||
assert.equal(parsePayload.passed, false)
|
||||
assert.ok(parsePayload.failingClauses.length > 0)
|
||||
|
||||
const fixture = imhotepFixture('smoke.html', { fixturesDir: './fixtures', env: { deterministic: true, seed: 7 } })
|
||||
const run = await fixture.exhaustivelyForAllInputs(
|
||||
page,
|
||||
enumeratedDomain([{ variant: 'ok' }]),
|
||||
async (scene) => {
|
||||
scene.expect('[data-testid="left"]').to.be.leftOf('[data-testid="right"]', { minGap: 8 })
|
||||
},
|
||||
{ reportMode: 'compact' },
|
||||
)
|
||||
assert.equal(run.mode, 'enumerated-determinate')
|
||||
assert.ok('minimalFailingCase' in run || run.passed)
|
||||
|
||||
await browser.close()
|
||||
console.log('external smoke passed')
|
||||
`,
|
||||
)
|
||||
|
||||
execSync('node smoke.mjs', { cwd: projectDir, stdio: 'inherit' })
|
||||
|
||||
console.log(`External smoke succeeded: ${projectDir}`)
|
||||
@@ -0,0 +1,170 @@
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs'
|
||||
import { execSync } from 'child_process'
|
||||
import { globSync } from 'glob'
|
||||
import path from 'path'
|
||||
|
||||
const PACKAGES = [
|
||||
'imhotep-solver',
|
||||
'imhotep-geometry',
|
||||
'imhotep-topology',
|
||||
'imhotep-dsl'
|
||||
]
|
||||
|
||||
const MUTATIONS = [
|
||||
{ name: 'reverse-equality', pattern: /===/g, replacement: '!==' },
|
||||
{ name: 'reverse-inequality', pattern: /!==/g, replacement: '===' },
|
||||
{ name: 'gt-to-gte', pattern: /(?<!<)>(?!=)/g, replacement: '>=' },
|
||||
{ name: 'lt-to-lte', pattern: /(?<!>)<(?!-)/g, replacement: '<=' },
|
||||
{ name: 'gte-to-gt', pattern: />=/g, replacement: '>' },
|
||||
{ name: 'lte-to-lt', pattern: /<=/g, replacement: '<' },
|
||||
{ name: 'and-to-or', pattern: /&&/g, replacement: '||' },
|
||||
{ name: 'or-to-and', pattern: /\|\|/g, replacement: '&&' },
|
||||
{ name: 'plus-to-minus', pattern: /(?<![+])\+(?![+=])/g, replacement: '-' },
|
||||
{ name: 'minus-to-plus', pattern: /(?<!-)\-(?!\-|=)/g, replacement: '+' },
|
||||
{ name: 'true-to-false', pattern: /: true\b/g, replacement: ': false' },
|
||||
{ name: 'false-to-true', pattern: /: false\b/g, replacement: ': true' }
|
||||
]
|
||||
|
||||
function getSourceFiles(pkg) {
|
||||
const srcDir = path.join('packages', pkg, 'src')
|
||||
return globSync('**/*.ts', { cwd: srcDir, absolute: true })
|
||||
.filter(f => !f.endsWith('.test.ts'))
|
||||
}
|
||||
|
||||
function runTests(pkg) {
|
||||
try {
|
||||
execSync('npm test', {
|
||||
cwd: path.join('packages', pkg),
|
||||
stdio: 'pipe',
|
||||
timeout: 30000
|
||||
})
|
||||
return { passed: true }
|
||||
} catch (e) {
|
||||
return { passed: false, output: e.stdout?.toString() || e.message }
|
||||
}
|
||||
}
|
||||
|
||||
function build(pkg) {
|
||||
try {
|
||||
execSync('npm run build', {
|
||||
cwd: path.join('packages', pkg),
|
||||
stdio: 'pipe',
|
||||
timeout: 60000
|
||||
})
|
||||
return true
|
||||
} catch (e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function applyMutation(filePath, mutation, index) {
|
||||
const content = readFileSync(filePath, 'utf8')
|
||||
const matches = [...content.matchAll(mutation.pattern)]
|
||||
|
||||
if (matches.length === 0) return null
|
||||
if (index >= matches.length) return null
|
||||
|
||||
const match = matches[index]
|
||||
const before = content.slice(0, match.index)
|
||||
const after = content.slice(match.index + match[0].length)
|
||||
const mutated = before + mutation.replacement + after
|
||||
|
||||
writeFileSync(filePath, mutated)
|
||||
return {
|
||||
line: content.slice(0, match.index).split('\n').length,
|
||||
original: match[0],
|
||||
mutated: mutation.replacement
|
||||
}
|
||||
}
|
||||
|
||||
function restoreFile(filePath, original) {
|
||||
writeFileSync(filePath, original)
|
||||
}
|
||||
|
||||
const results = {}
|
||||
|
||||
for (const pkg of PACKAGES) {
|
||||
console.log(`\n=== Package: ${pkg} ===`)
|
||||
const files = getSourceFiles(pkg)
|
||||
const pkgResults = {
|
||||
total: 0,
|
||||
killed: 0,
|
||||
survived: 0,
|
||||
timedOut: 0,
|
||||
mutants: []
|
||||
}
|
||||
|
||||
for (const filePath of files) {
|
||||
const content = readFileSync(filePath, 'utf8')
|
||||
|
||||
for (const mutation of MUTATIONS) {
|
||||
const matches = [...content.matchAll(mutation.pattern)]
|
||||
|
||||
for (let i = 0; i < matches.length; i++) {
|
||||
const info = applyMutation(filePath, mutation, i)
|
||||
if (!info) continue
|
||||
|
||||
pkgResults.total++
|
||||
const shortFile = path.relative(process.cwd(), filePath)
|
||||
|
||||
process.stdout.write(` Mutant ${pkgResults.total}: ${shortFile}:${info.line} ${info.original} -> ${info.mutated} `)
|
||||
|
||||
const built = build(pkg)
|
||||
if (!built) {
|
||||
pkgResults.timedOut++
|
||||
console.log('BUILD ERROR')
|
||||
restoreFile(filePath, content)
|
||||
continue
|
||||
}
|
||||
|
||||
const testResult = runTests(pkg)
|
||||
|
||||
if (!testResult.passed) {
|
||||
pkgResults.killed++
|
||||
console.log('KILLED')
|
||||
} else {
|
||||
pkgResults.survived++
|
||||
console.log('SURVIVED')
|
||||
pkgResults.mutants.push({
|
||||
file: shortFile,
|
||||
line: info.line,
|
||||
mutation: `${info.original} -> ${info.mutated}`,
|
||||
type: mutation.name
|
||||
})
|
||||
}
|
||||
|
||||
restoreFile(filePath, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const score = pkgResults.total > 0
|
||||
? Math.round((pkgResults.killed / pkgResults.total) * 100)
|
||||
: 0
|
||||
pkgResults.score = score
|
||||
results[pkg] = pkgResults
|
||||
|
||||
console.log(`\n Score: ${score}% (${pkgResults.killed}/${pkgResults.total})`)
|
||||
console.log(` Survived: ${pkgResults.survived}, TimedOut: ${pkgResults.timedOut}`)
|
||||
}
|
||||
|
||||
console.log('\n\n========== MUTATION TESTING SUMMARY ==========')
|
||||
console.log('| Package | Score | Survived | Killed | Timeout |')
|
||||
console.log('|---------|-------|----------|--------|---------|')
|
||||
for (const pkg of PACKAGES) {
|
||||
const r = results[pkg]
|
||||
console.log(`| ${pkg.padEnd(18)} | ${String(r.score).padStart(3)}% | ${String(r.survived).padStart(8)} | ${String(r.killed).padStart(6)} | ${String(r.timedOut).padStart(7)} |`)
|
||||
}
|
||||
|
||||
console.log('\n\n========== SURVIVING MUTANTS ==========')
|
||||
for (const pkg of PACKAGES) {
|
||||
const r = results[pkg]
|
||||
if (r.mutants.length === 0) continue
|
||||
|
||||
console.log(`\n--- ${pkg} ---`)
|
||||
for (const m of r.mutants) {
|
||||
console.log(` ${m.file}:${m.line} ${m.mutation} (${m.type})`)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n\nDone.')
|
||||
Reference in New Issue
Block a user