diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1b2b6ba..4eb4551a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,9 @@ jobs: os: [ubuntu-latest, macos-latest] node: ['22', '26'] runs-on: ${{ matrix.os }} - timeout-minutes: 30 + # macOS Node 26 runs take 22-25 minutes (2026-11-12); 30 left one queue hiccup between a green + # suite and a cancelled one. A hung test is caught by its own timeout well inside this. + timeout-minutes: 40 steps: # Actions pinned to commit SHAs, not tags: a tag can be silently repointed by the action # owner (the trivy-action / tj-actions compromise class). The trailing comment names the diff --git a/admin/test/unit-routes-offbox-record-page.test.mjs b/admin/test/unit-routes-offbox-record-page.test.mjs index 8140d005..2cd7e9a4 100644 --- a/admin/test/unit-routes-offbox-record-page.test.mjs +++ b/admin/test/unit-routes-offbox-record-page.test.mjs @@ -1,8 +1,26 @@ // Records a normalised offbox page entry and appends it to the page store (admin/routes/offbox.mjs recordPage). -import { test } from 'node:test'; +import { test, before, after } from 'node:test'; import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { recordPage } from '../routes/offbox.mjs'; +// recordPage appends to the page store. Unscoped, every call below lands in the checkout's own +// .claude/store/offbox-pages.jsonl, which bin/test/sidecar-paths.test.mjs then reads as a store +// written with nothing versioning it. +const TMP = mkdtempSync(join(tmpdir(), 'cw-offbox-record-page-')); +let saved; + +before(() => { + saved = process.env.CW_OFFBOX_PAGE_STORE; + process.env.CW_OFFBOX_PAGE_STORE = join(TMP, 'pages.jsonl'); +}); +after(() => { + if (saved === undefined) delete process.env.CW_OFFBOX_PAGE_STORE; else process.env.CW_OFFBOX_PAGE_STORE = saved; + rmSync(TMP, { recursive: true, force: true }); +}); + test('normalises a valid ok state with all fields present', () => { const at = '2024-01-01T00:00:00.000Z'; const body = { diff --git a/bin/commit-phase.mjs b/bin/commit-phase.mjs index c4c31360..607a95aa 100644 --- a/bin/commit-phase.mjs +++ b/bin/commit-phase.mjs @@ -670,7 +670,7 @@ function replay({ onto, ref, gitDir, key, check }) { } const tree = git(['write-tree'], { env, allowFail: true }); if (typeof tree !== 'string' || !/^[0-9a-f]{40}$/.test(tree)) return giveUp('write-tree failed during replay — nothing landed.'); - const rawMsg = execFileSync('git', ['log', '-1', '--format=%B', c], { cwd: repo(), encoding: 'utf8' }); + const rawMsg = git(['log', '-1', '--format=%B', c], { raw: true }); const { text: msg, removed } = stripForbiddenTrailers(rawMsg); if (removed) console.error(`commit-phase: stripped ${removed} Co-Authored-By trailer(s) from ${c.slice(0, 7)} during replay.`); const rebuilt = buildCommit([tree, '-p', parent], { input: msg, env }); diff --git a/bin/disclosure-review.mjs b/bin/disclosure-review.mjs index be60ab17..485ef60f 100644 --- a/bin/disclosure-review.mjs +++ b/bin/disclosure-review.mjs @@ -22,20 +22,16 @@ // --out output dir (default $CW_DISCLOSURE_OUT or ./reports/disclosure-) // Exit 0 always on a completed run; 1 on bad input. -import { readFileSync, writeFileSync, mkdirSync, renameSync } from 'node:fs'; +import { readFileSync, mkdirSync } from 'node:fs'; import { join, basename } from 'node:path'; import { nowISO } from '../lib/clock.mjs'; import { judge } from '../lib/judge.mjs'; import { classify, priorityKey } from '../lib/judge-taxonomy.mjs'; +import { isMainModule } from '../lib/is-main.mjs'; +import { writeAtomic } from '../monitor/lockfile.mjs'; export const BUCKETS = Object.freeze(['confirmed', 'dismissed', 'undetermined']); -function writeAtomic(path, text) { - const tmp = `${path}.tmp-${process.pid}`; - writeFileSync(tmp, text); - renameSync(tmp, path); -} - async function mapPool(items, n, fn) { const out = new Array(items.length); let next = 0; @@ -184,4 +180,4 @@ async function main() { process.exit(0); } -if (import.meta.url === `file://${process.argv[1]}`) main(); +if (isMainModule(import.meta.url)) main(); diff --git a/bin/init.mjs b/bin/init.mjs index 2db5cb74..0160efde 100644 --- a/bin/init.mjs +++ b/bin/init.mjs @@ -24,7 +24,7 @@ import { privateDir, registryPathFor, authStorePath, annotationsPathFor, gateExemptionsPathFor, stateRootFor, STATE_PATHS, stubAllowlistPathFor, imageAcceptancePathFor, ownerMapPathFor, programWorklistPathFor, bolaManifestDirFor, craProductsPathFor, credentialScopePathFor, configCorrectnessLedgerPathFor, - disregardedWarningsPathFor, + disregardedWarningsPathFor, releaseSourcesPathFor, } from '../monitor/store-paths.mjs'; import { loadRegistry, validateRegistry, validateAgainstSchema } from '../monitor/registry.mjs'; @@ -45,6 +45,7 @@ export const OPTIONAL_RECORDS = [ { env: 'CW_DISREGARDED_WARNINGS', at: disregardedWarningsPathFor, absent: 'no warning has been set aside', example: 'monitor/disregarded-warnings.example.json' }, { env: 'CW_PRODUCTS', at: craProductsPathFor, absent: 'CRA tools refuse with "not configured"', example: 'cra/products.example.json' }, { env: 'CW_BOLA_MANIFEST_DIR', at: bolaManifestDirFor, absent: 'the BOLA lane has no manifests to run', example: 'manifests/bola/example.json' }, + { env: 'CW_RELEASE_SOURCES', at: releaseSourcesPathFor, absent: 'release-report --fetch has nothing to fetch', example: 'monitor/releases.example.json' }, ]; export function parseInitArgs(argv) { diff --git a/bin/lib/sandbox-net.mjs b/bin/lib/sandbox-net.mjs index 054ff060..80f929c8 100644 --- a/bin/lib/sandbox-net.mjs +++ b/bin/lib/sandbox-net.mjs @@ -126,6 +126,12 @@ function main() { b.kill('SIGKILL'); }; b.on('error', (e) => { refused = refused || `${opts.bwrap[0]}: ${e.code || e.message}`; say(`sandbox-net: ${refused}`); process.exitCode = NOT_RUN; }); + // A refusal SIGKILLs bwrap, and a read or write already in flight on these pipes then fails with + // ECONNRESET or EPIPE. Unhandled, that crashes this helper with exit 1, and a caller reads 1 as the + // lane's own failure instead of NOT_RUN. The close handler below gives the verdict, so a pipe error + // after a kill is expected; any other pipe error is a refusal of its own. + const pipeError = (fd) => (e) => { if (e.code !== 'ECONNRESET' && e.code !== 'EPIPE') refuse(`bwrap fd ${fd}: ${e.code || e.message}`); }; + for (const i of [3, 4, 5]) if (b.stdio[i]) b.stdio[i].on('error', pipeError(i)); if (data) b.stdio[5].end(resolvConf(readOr('/etc/resolv.conf'))); let info = ''; b.stdio[3].on('data', (d) => { diff --git a/bin/release-report.mjs b/bin/release-report.mjs index 5f919b91..f1826ea4 100644 --- a/bin/release-report.mjs +++ b/bin/release-report.mjs @@ -40,7 +40,7 @@ import { houseCss } from '../lib/house-css.mjs'; import { MARK_ICON } from '../lib/brand-tokens.mjs'; import { validateAgainstSchema } from '../lib/json-schema.mjs'; import { writeAtomic } from '../monitor/lockfile.mjs'; -import { privateDir, reportsRootFor } from '../monitor/store-paths.mjs'; +import { releaseSourcesPathFor, reportsRootFor } from '../monitor/store-paths.mjs'; import { kevFreshness, kevSet, epssFor, loadJSON, craRoot } from '../cra/lib.mjs'; import { loadIndex } from './advisory-index.mjs'; import { baseScore, band, versionOf } from './cvss.mjs'; @@ -52,7 +52,7 @@ const HOUR = 3600_000; const SSH_TIMEOUT_MS = 60_000; const releasesDir = () => resolve(process.env.CW_RELEASES_DIR || join(homedir(), '.commitwork', 'releases')); -const sourcesPath = () => resolve(process.env.CW_RELEASE_SOURCES || join(privateDir(CW), 'releases.json')); +const sourcesPath = () => releaseSourcesPathFor(CW); const kevPath = () => resolve(process.env.CW_KEV || join(craRoot(), 'monitor', 'data', 'kev.json')); const epssPath = () => resolve(process.env.CW_EPSS || join(craRoot(), 'monitor', 'data', 'epss.json')); diff --git a/bin/setup.mjs b/bin/setup.mjs index 6f3ba560..fbe5601a 100755 --- a/bin/setup.mjs +++ b/bin/setup.mjs @@ -28,6 +28,7 @@ import { safeSpawnSync } from '../lib/win-spawn.mjs'; // npm/scoop/gem are .cmd import { resolvePinnedTool } from '../lib/cobolwork-resolve.mjs'; import { verifyReleaseSignature } from '../lib/release-signing.mjs'; import { recordProvenance } from '../lib/tool-provenance.mjs'; +import { unzip } from './cwe-graph.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); // The same override the panel's scanner routes read; both read at call time. @@ -106,9 +107,22 @@ const releaseBase = () => process.env.CW_SETUP_RELEASE_BASE || 'https://github.c export const releaseUrl = (spec, a) => `${releaseBase()}/${spec.repo}/releases/download/${spec.tag}/${a.asset}`; const releaseBinDir = () => process.env.CW_SETUP_BIN_DIR || join(homedir(), '.local', 'bin'); export const LICENSE_FILE = /(^|\/)(LICEN[CS]E|COPYING|NOTICE)([._-][^/]*)?$/i; +// fact: GNU tar does not read a zip, bsdtar does / GNU tar is the tar on Linux and the one Git for Windows puts first on PATH under Git Bash, so a .zip pin installed only where bsdtar answered to `tar` (expiry: never, prev: broken) +/** The entries of a zip as [{ name, data }], or null when the bytes are not a zip this reader handles. */ +function zipEntries(buf) { + try { return unzip(buf); } catch { return null; } +} /** Copy an archive's licence files to dest; false when it holds none or they will not extract. */ function copyLicenses(archive, dest, tmp) { const list = spawnSync('tar', ['-tf', archive], { encoding: 'utf8' }); + if (list.status !== 0) { + // A backslash is a separator on Windows, so a name carrying one is not taken as a file name. + const held = (zipEntries(readFileSync(archive)) || []).filter((e) => LICENSE_FILE.test(e.name) && !e.name.includes('\\')); + if (!held.length) return false; + mkdirSync(dest, { recursive: true }); + for (const e of held) writeFileSync(join(dest, e.name.split('/').pop()), e.data); + return true; + } const names = list.status === 0 ? list.stdout.split('\n').filter((n) => LICENSE_FILE.test(n)) : []; if (!names.length || spawnSync('tar', ['-xf', archive, '-C', tmp, ...names], { stdio: 'ignore' }).status !== 0) return false; mkdirSync(dest, { recursive: true }); @@ -485,7 +499,13 @@ async function installRelease(row) { // verb, and the format is detected from the bytes rather than assumed from a flag. writeFileSync(join(tmp, a.asset), buf); const x = spawnSync('tar', ['-xf', join(tmp, a.asset), '-C', tmp, bin], { stdio: 'ignore' }); - if (x.status !== 0 || !existsSync(join(tmp, bin))) { console.log(red(` ✗ ${row.name}: ${bin} not found in ${a.asset}`)); return false; } + let extracted = x.status === 0 && existsSync(join(tmp, bin)); + if (!extracted) { + // Where `tar` is GNU tar a zip is unreadable to it: the member is read from the verified bytes instead. + const member = (zipEntries(buf) || []).find((e) => e.name === bin); + if (member) { writeFileSync(join(tmp, bin), member.data); extracted = true; } + } + if (!extracted) { console.log(red(` ✗ ${row.name}: ${bin} not found in ${a.asset}`)); return false; } const lic = process.env.CW_SETUP_LICENSE_DIR; if (lic && !copyLicenses(join(tmp, a.asset), join(lic, row.name), tmp)) { console.log(red(` ✗ ${row.name}: no licence file in ${a.asset}`)); return false; } const dir = releaseBinDir(); diff --git a/bin/test/daily-run.test.mjs b/bin/test/daily-run.test.mjs index 4b7c1e2f..b31c6382 100644 --- a/bin/test/daily-run.test.mjs +++ b/bin/test/daily-run.test.mjs @@ -36,6 +36,20 @@ if (n === 1) out.notActioned.push({ findingId: '0123456789abcdef', reason: 'low- console.log(JSON.stringify({ is_error: false, subtype: 'success', structured_output: out, total_cost_usd: 0.25, usage: { input_tokens: 100, cache_read_input_tokens: 50, output_tokens: 10 }, modelUsage: { 'claude-test': { outputTokens: 10 } } })); `; +/** + * Snapshot process.env and put it back IN PLACE after the test. Reassigning process.env (`process.env = + * saved`) swaps the live environment for a plain object; from Node 26 child_process no longer reads that + * object when no `env` option is passed, so later tests that put stubs on PATH spawn against the real + * environment and the stub never runs. + */ +function restoreEnvAfter(t) { + const saved = { ...process.env }; + t.after(() => { + for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k]; + Object.assign(process.env, saved); + }); +} + function fakeVeld() { const todos = []; const server = createServer((req, res) => { @@ -75,8 +89,7 @@ test('a scheduled run writes a validated report after one refusal, files the tod writeFileSync(skill, 'skill text'); const veld = await fakeVeld(); t.after(() => veld.server.close()); - const saved = { ...process.env }; - t.after(() => { process.env = saved; }); + restoreEnvAfter(t); Object.assign(process.env, { CW_REGISTRY: fx.registry, CW_DAILY_CONFIG: fx.configPath, CW_CLAUDE_BIN: join(bin, 'claude'), CW_DAILY_SKILL: skill, CW_NOW: '2026-10-02T01:00:00Z', CW_VELD_URL: veld.url, CW_VELD_USER: 'tester', VELD_API_KEY: 'test-key', @@ -123,25 +136,27 @@ test('a scheduled run writes a validated report after one refusal, files the tod }); test('a model that never answers within the schema leaves a failed run record and no report', async (t) => { - // The stubs drain stdin before answering. A stub that exits first races the prompt write, and the - // run then fails on EPIPE instead of on the missing schema. + // Arrange. The stubs drain stdin before answering. A stub that exits first races the prompt write, + // and the run then fails on EPIPE instead of on the missing schema. const fx = dailyFixture({ previousRows: {}, currentRows: { sastSemgrep: [rows.exec] } }); const bin = join(fx.root, 'bin'); mkdirSync(bin); writeFileSync(join(bin, 'claude'), `#!/bin/sh\ncat >/dev/null\necho x >> ${join(fx.root, 'calls')}\necho '{"is_error":true,"subtype":"error_max_turns","result":"no"}'\n`); chmodSync(join(bin, 'claude'), 0o755); writeFileSync(join(bin, 'osascript'), `#!/bin/sh\necho "$2" >> ${join(fx.root, 'notices')}\n`); chmodSync(join(bin, 'osascript'), 0o755); writeFileSync(join(fx.root, 'daily.md'), 'skill'); - const saved = { ...process.env }; - t.after(() => { process.env = saved; }); + restoreEnvAfter(t); Object.assign(process.env, { CW_REGISTRY: fx.registry, CW_DAILY_CONFIG: fx.configPath, CW_CLAUDE_BIN: join(bin, 'claude'), CW_DAILY_SKILL: join(fx.root, 'daily.md'), PATH: `${bin}:${process.env.PATH}` }); + const calls = () => readFileSync(join(fx.root, 'calls'), 'utf8').trim().split('\n').length; + const notices = () => readFileSync(join(fx.root, 'notices'), 'utf8').trim().split('\n').length; + + // Act and assert, per pass: the first run fails, the next skips, --force retries. assert.equal(await main(['--area', AREA, '--no-todos']), 1); const daily = join(fx.areaOut, 'daily'); assert.ok(!existsSync(join(daily, 'sweep-20261002000000.json'))); const run = JSON.parse(readFileSync(join(daily, 'sweep-20261002000000.run.json'), 'utf8')); assert.equal(run.ok, false); assert.match(run.attempts[0].error, /no structured output/); - const calls = () => readFileSync(join(fx.root, 'calls'), 'utf8').trim().split('\n').length; - const notices = () => readFileSync(join(fx.root, 'notices'), 'utf8').trim().split('\n').length; + assert.equal(notices(), 1, 'the failure notifies'); assert.equal(calls(), 1, 'the failed attempt is not retried within the run when claude itself errored'); assert.equal(await main(['--area', AREA, '--no-todos']), 0, 'the next pass skips a batch whose model run failed'); assert.equal(calls(), 1); @@ -158,8 +173,7 @@ test('a run the area lock is held against does nothing, and leaves the holder it writeFileSync(join(bin, 'claude'), `#!/bin/sh\ncat >/dev/null\necho x >> ${join(fx.root, 'calls')}\necho '{"is_error":true,"subtype":"error_max_turns","result":"no"}'\n`); chmodSync(join(bin, 'claude'), 0o755); writeFileSync(join(bin, 'osascript'), '#!/bin/sh\n'); chmodSync(join(bin, 'osascript'), 0o755); writeFileSync(join(fx.root, 'daily.md'), 'skill'); - const saved = { ...process.env }; - t.after(() => { process.env = saved; }); + restoreEnvAfter(t); Object.assign(process.env, { CW_REGISTRY: fx.registry, CW_DAILY_CONFIG: fx.configPath, CW_CLAUDE_BIN: join(bin, 'claude'), CW_DAILY_SKILL: join(fx.root, 'daily.md'), PATH: `${bin}:${process.env.PATH}` }); const held = acquireLock(join(fx.areaOut, 'daily', '.lock'), { label: 'another daily run' }); assert.ok(held.ok); @@ -174,8 +188,7 @@ test('a run the area lock is held against does nothing, and leaves the holder it test('a refused suggestions file leaves no run record, and one for a reported batch is refused', async (t) => { const fx = dailyFixture({ previousRows: {}, currentRows: { sastSemgrep: [rows.exec] } }); - const saved = { ...process.env }; - t.after(() => { process.env = saved; }); + restoreEnvAfter(t); Object.assign(process.env, { CW_REGISTRY: fx.registry, CW_DAILY_CONFIG: fx.configPath }); const bad = join(fx.root, 'bad.json'); writeFileSync(bad, JSON.stringify({ headline: 'h', suggestions: [], notActioned: [] })); @@ -192,6 +205,7 @@ test('a refused suggestions file leaves no run record, and one for a reported ba }); test('a failure notifies when it appears, when it changes, and once a day while it lasts', (t) => { + // Arrange const root = mkdtempSync(join(tmpdir(), 'cw-daily-notice-')); const bin = join(root, 'bin'); mkdirSync(bin); @@ -201,12 +215,22 @@ test('a failure notifies when it appears, when it changes, and once a day while process.env.PATH = `${bin}:${saved}`; const state = join(root, 'state.json'); const t0 = new Date('2026-10-03T00:00:00Z'); - assert.equal(noticeFailure(state, 'ar', 'skill not committed', t0), true); - assert.equal(noticeFailure(state, 'ar', 'skill not committed', new Date(t0.getTime() + 30 * 60_000)), false); - assert.equal(noticeFailure(state, 'ar', 'anchor moved', new Date(t0.getTime() + 60 * 60_000)), true); - assert.equal(noticeFailure(state, 'ar', 'anchor moved', new Date(t0.getTime() + 25 * 3_600_000)), true); + const at = (ms) => new Date(t0.getTime() + ms); + + // Act + const appears = noticeFailure(state, 'ar', 'skill not committed', t0); + const sameWithinADay = noticeFailure(state, 'ar', 'skill not committed', at(30 * 60_000)); + const changes = noticeFailure(state, 'ar', 'anchor moved', at(60 * 60_000)); + const sameNextDay = noticeFailure(state, 'ar', 'anchor moved', at(25 * 3_600_000)); clearFailure(state, 'ar'); - assert.equal(noticeFailure(state, 'ar', 'anchor moved', new Date(t0.getTime() + 26 * 3_600_000)), true); + const afterClear = noticeFailure(state, 'ar', 'anchor moved', at(26 * 3_600_000)); + + // Assert + assert.equal(appears, true); + assert.equal(sameWithinADay, false); + assert.equal(changes, true); + assert.equal(sameNextDay, true); + assert.equal(afterClear, true); assert.equal(readFileSync(join(root, 'notices'), 'utf8').trim().split('\n').length, 4); }); diff --git a/bin/test/docsite-og-image-entry.test.mjs b/bin/test/docsite-og-image-entry.test.mjs index e3927288..7890db64 100644 --- a/bin/test/docsite-og-image-entry.test.mjs +++ b/bin/test/docsite-og-image-entry.test.mjs @@ -134,19 +134,37 @@ const stressExits = (args, env = process.env) => Promise.all(Array.from({ length c.on('exit', (code, signal) => { clearTimeout(timer); done(code ?? signal); }); }))); +// The control does as much work as one renderer run and exits the same way, naturally. Under this +// stress Node itself can hang ANY process at shutdown once it has run for a while (nodejs/node#54918 +// "Deadlock at process shutdown", #64274 "Hang at shutdown", both open): on 2026-11-12, 4-core Linux, +// Node 22.23, a bare setTimeout(2 s) hung 5/24 and a 2 s busy loop 4/24, while a millisecond +// allocation loop hung none. A short control therefore says nothing about a renderer that runs longer. +// This one spins hot integer work and deflate until its own CPU time reaches what one unstressed +// renderer run took, so 24 at once stretch it the same way they stretch the renderer. If Node hangs +// it, the renderer cannot exit more cleanly than it, and the run skips with the counts. A renderer +// that reintroduces process.exit() is still caught by the static exitHazards test above. +const workControl = (cpuMs) => "const z=require('node:zlib');const b=new Uint8Array(1200*630*3);" + + `for(let r=0;process.cpuUsage().user<${cpuMs}*1000;r++){for(let i=0;i { - const control = await stressExits(['-e', 'const a = []; for (let i = 0; i < 2e5; i++) a.push({ i });']); - const controlHung = control.filter((c) => c === 'hung').length; - if (controlHung) { - t.skip(`Node ${process.version} hung ${controlHung}/${STRESS_RUNS} allocation-only controls under the same stress`); - return; - } + // Arrange const { root, png } = site(t); mkdirSync(join(root, 'public'), { recursive: true }); writeFileSync(png, Buffer.from('not the card')); const env = { ...process.env, CW_DOCSITE_ROOT: root }; delete env.CW_DOCSITE_PRIVATE; + const t0 = Date.now(); + spawnSync(process.execPath, [OG, '--check'], { env, stdio: 'ignore' }); + const work = Math.max(50, Date.now() - t0); + const control = await stressExits(['-e', workControl(work)]); + const controlHung = control.filter((c) => c === 'hung').length; + if (controlHung) { + t.skip(`Node ${process.version} hung ${controlHung}/${STRESS_RUNS} controls doing ${work} ms of CPU work and exiting naturally, under the same stress (nodejs/node#54918); the renderer's exit was NOT measured`); + return; + } + // Act const exits = await stressExits([OG, '--check'], env); + // Assert assert.ok(!exits.includes('hung'), `"hung" is the deadlock: ${exits.join(', ')}`); const killed = exits.filter((e) => typeof e === 'string'); const killedBy = [...new Set(killed)].join(', '); diff --git a/bin/test/github-action.test.mjs b/bin/test/github-action.test.mjs index 1e39d97b..f73be891 100644 --- a/bin/test/github-action.test.mjs +++ b/bin/test/github-action.test.mjs @@ -10,6 +10,7 @@ import { join, dirname, delimiter } from 'node:path'; import { fileURLToPath } from 'node:url'; import { parseWorkflow, repoOf } from '../pin-actions.mjs'; import { decide, readSarifSummary, annotation, report, FAIL_ON } from '../lib/github-action.mjs'; +import { probeHostSandbox } from '../lib/sandbox.mjs'; const CW = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); const ACTION = readFileSync(join(CW, 'action.yml'), 'utf8'); @@ -253,12 +254,14 @@ test('a repo-supplied script runs only when trust-repo-manifest is true, whateve writeFileSync(join(sb.ws, 'app', 'security', 'authz-isolation-test.sh'), '#!/bin/sh\necho \'{"ran":"repo script"}\'\n'); chmodSync(join(sb.ws, 'app', 'security', 'authz-isolation-test.sh'), 0o755); const step = stepNamed('Run the baseline'); - const held = runStep(step, sb, { inputs: { path: 'app', checks: 'authz-test' }, env: { COMMITWORK_TRUST_REPO_MANIFEST: '1' } }); + // With no host sandbox a repository-code lane is refused whatever the trust (sandbox-repo-code.test.mjs); trust is the subject here. + const unconfined = probeHostSandbox().available ? {} : { CW_SANDBOX: 'off' }; + const held = runStep(step, sb, { inputs: { path: 'app', checks: 'authz-test' }, env: { COMMITWORK_TRUST_REPO_MANIFEST: '1', ...unconfined } }); assert.equal(held.status, 0, held.stderr); assert.ok(!existsSync(join(held.out['report-dir'], 'authz.json')), 'the script ran without the input'); const note = JSON.parse(readFileSync(held.out.sarif, 'utf8')).runs[0].invocations[0].toolExecutionNotifications.find((n) => n.descriptor.id === 'authz-test'); assert.match(note.message.text, /--trust-repo-manifest/); - const trusted = runStep(step, sb, { inputs: { path: 'app', checks: 'authz-test', 'trust-repo-manifest': 'true' } }); + const trusted = runStep(step, sb, { inputs: { path: 'app', checks: 'authz-test', 'trust-repo-manifest': 'true' }, env: unconfined }); assert.equal(trusted.status, 0, trusted.stderr); assert.match(readFileSync(join(trusted.out['report-dir'], 'authz.json'), 'utf8'), /repo script/); }); diff --git a/bin/test/lane-windows-env.test.mjs b/bin/test/lane-windows-env.test.mjs index d687d58b..ca87db7b 100644 --- a/bin/test/lane-windows-env.test.mjs +++ b/bin/test/lane-windows-env.test.mjs @@ -15,10 +15,11 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { readFileSync, writeFileSync, mkdirSync, mkdtempSync, rmSync, readdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join, resolve, dirname } from 'node:path'; +import { join, resolve, dirname, delimiter } from 'node:path'; import { fileURLToPath } from 'node:url'; import { laneWindowsEnv } from '../lib/scanner-env.mjs'; import { msysPathconvEnv, MSYS_PATHCONV_ENV } from '../../lib/posix-shell.mjs'; +import { fakeDocker, fakeSimple } from '../../monitor/test/lib/fake-bin.mjs'; const CW = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); const semgrep = { id: 'sast', requires: { tools: ['semgrep'] } }; @@ -90,6 +91,10 @@ test('every bin/*.sh that hands a container or the sandbox CLI a path exports th // child's real environment. It can only be run where the suite runs, so what it proves here is the // off-Windows direction: a semgrep-shaped lane and a docker-shaped lane both execute, and neither // carries a variable this platform never needed. +// The requirement gate would refuse both lanes on a host with no semgrep or no docker daemon, and a +// refused lane writes no env to read. Neither tool is ever run by `env > file`, so a fake of each +// goes first on PATH and the docker probe cache is the fixture's own: the gate is satisfied by the +// fixture, not by what this machine has installed. test('a lane runs with its env untouched on this platform', async () => { const T = mkdtempSync(join(tmpdir(), 'cw-win-env-')); try { @@ -97,11 +102,16 @@ test('a lane runs with its env untouched on this platform', async () => { writeFileSync(join(repo, 'README.md'), 'fixture\n'); const reports = join(T, 'reports'); mkdirSync(reports); const manifest = join(T, 'm.json'); + const fakes = join(T, 'bin'); mkdirSync(fakes); + fakeSimple(fakes, 'semgrep'); + fakeDocker(fakes, { info: 'ok' }); const dump = (id, requires) => ({ id, egress: 'none', groups: ['all'], requires, local: [`env > "$CW_REPORT_DIR/${id}.env"`] }); writeFileSync(manifest, JSON.stringify({ repo: 'fixture', checks: [dump('py', { tools: ['semgrep'] }), dump('box', { tools: ['docker'], docker: true })] })); const env = { ...process.env, CW_REPORT_DIR: reports, CW_SKIP_SETUP: '1', CW_ASSERT_TREE: '0', CW_SELF_SWEEP: '0', - CW_SANDBOX: 'off', CW_PERF_FEEDBACK: join(T, 'perf.jsonl'), FORCE_COLOR: '0' }; + CW_SANDBOX: 'off', CW_PERF_FEEDBACK: join(T, 'perf.jsonl'), FORCE_COLOR: '0', + PATH: `${fakes}${delimiter}${process.env.PATH || ''}`, FAKE_LOG: join(T, 'fakes.log'), FAKE_DIR: T, + CW_DOCKER_PROBE_CACHE: join(T, 'docker-probe.json') }; delete env.PYTHONUTF8; delete env.MSYS_NO_PATHCONV; delete env.MSYS2_ARG_CONV_EXCL; const out = await new Promise((res) => { const c = spawn(process.execPath, [join(CW, 'bin', 'commitwork.mjs'), 'run', 'all', '--manifest', manifest, '--repo', repo, '--no-fail-fast'], diff --git a/bin/test/no-argv-secrets.test.mjs b/bin/test/no-argv-secrets.test.mjs index 7a5cf712..b02773a1 100644 --- a/bin/test/no-argv-secrets.test.mjs +++ b/bin/test/no-argv-secrets.test.mjs @@ -74,6 +74,11 @@ test('no .mjs pushes a secret-bearing FLAG into a spawn args array', () => { + offenders.map((o) => ' ' + o).join('\n')); }); +// A spawn option `input: `…`` is the child's STDIN, which is the remedy the shell-script rule above +// names, not the defect: no shell parses it and `ps` never shows it. It is removed before matching so +// a secret written to a child's stdin on the same line as the call is not read as interpolation. +const withoutStdin = (line) => line.replace(/\binput\s*:\s*`(?:[^`\\]|\\.)*`/g, 'input: ``'); + test('no .mjs interpolates a secret into a SHELL string', () => { // execSync(`… ${TOKEN} …`) is the same exposure with an extra layer: the shell sees it too. const INTERP = new RegExp( @@ -83,7 +88,7 @@ test('no .mjs interpolates a secret into a SHELL string', () => { if (f.endsWith('.test.mjs')) continue; const lines = stripComments(readFileSync(f, 'utf8'), 'js'); lines.forEach((line, i) => { - if (INTERP.test(line)) offenders.push(`${relative(REPO, f)}:${i + 1} ${line.trim().slice(0, 110)}`); + if (INTERP.test(withoutStdin(line))) offenders.push(`${relative(REPO, f)}:${i + 1} ${line.trim().slice(0, 110)}`); }); } assert.deepEqual(offenders, [], 'a secret is interpolated into a shell command string:\n' @@ -93,6 +98,19 @@ test('no .mjs interpolates a secret into a SHELL string', () => { // ── the detectors must actually detect ─────────────────────────────────────────────────────── // Each detector is proven against the real pre-fix line it was written for. +test('CONTROL: a secret on a child\'s stdin is not interpolation, and the same secret in the command still is', () => { + // Arrange + const INTERP = new RegExp(`(?:execSync|exec|spawnSync|spawn)\\([^)]*\\$\\{[^}]*${SECRET_NAME}[^}]*\\}`, 'i'); + const stdin = "exec('security', ['-i'], { input: `add-generic-password -U -s ${ITEM} -a ${ITEM} -w ${token}\\n` });"; + const shell = 'execSync(`curl -H "Authorization: ${token}" https://x`, { input: `x` });'; + // Act + const stdinFlagged = INTERP.test(withoutStdin(stdin)); + const shellFlagged = INTERP.test(withoutStdin(shell)); + // Assert + assert.equal(stdinFlagged, false, 'a value written to stdin must not be flagged'); + assert.equal(shellFlagged, true, 'a value interpolated into the command string must still be flagged'); +}); + test('CONTROL: the detectors catch the real lines they were written for', () => { // ci/run-ci.sh, before the fix. assert.ok(DOCKER_E_ASSIGN.test(' -e SPRING_DATASOURCE_USERNAME=$CI_DB_USER -e SPRING_DATASOURCE_PASSWORD=$CI_DB_PASS \\'), diff --git a/bin/test/sandbox-net-refusal-pipes.test.mjs b/bin/test/sandbox-net-refusal-pipes.test.mjs new file mode 100644 index 00000000..d5c1b62c --- /dev/null +++ b/bin/test/sandbox-net-refusal-pipes.test.mjs @@ -0,0 +1,31 @@ +// bin/lib/sandbox-net.mjs on a refusal: it SIGKILLs bwrap, and bwrap may die holding data the helper +// wrote to it (resolv.conf on fd 5) unread. Node's pipes are socketpairs, so the helper's end then +// fails with ECONNRESET. The helper must still exit NOT_RUN with its reason, never 1 with a stack. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync, symlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { NOT_RUN } from '../lib/sandbox-net.mjs'; + +const HELPER = join(dirname(fileURLToPath(import.meta.url)), '..', 'lib', 'sandbox-net.mjs'); +const which = (c) => spawnSync('sh', ['-c', `command -v ${c}`], { encoding: 'utf8' }).stdout.trim(); + +test('a refusal while bwrap holds unread pipe data exits NOT_RUN with the reason, every time', { skip: process.platform !== 'linux' && 'the helper runs on Linux only' }, () => { + // Arrange: a fake bwrap that announces a child pid on fd 3 and never reads fd 5; no pasta on PATH + const bin = mkdtempSync(join(tmpdir(), 'cw-snet-')); + for (const c of ['sh', 'sleep']) symlinkSync(which(c), join(bin, c)); + writeFileSync(join(bin, 'bwrap'), `#!${which('sh')}\nprintf '{"child-pid": %d}' $$ >&3\nexec sleep 30\n`, { mode: 0o755 }); + const argv = [HELPER, '--', join(bin, 'bwrap'), '--unshare-net', '--info-fd', '3', '--block-fd', '4', '--ro-bind-data', '5', '/etc/resolv.conf', '--', 'true']; + // Act + const runs = []; + for (let i = 0; i < 15; i++) runs.push(spawnSync(process.execPath, argv, { encoding: 'utf8', env: { PATH: bin }, timeout: 20_000 })); + // Assert + for (const r of runs) { + assert.equal(r.status, NOT_RUN, r.stderr); + assert.match(r.stderr, /pasta is not installed/); + assert.doesNotMatch(r.stderr, /ECONNRESET|EPIPE|Unhandled/); + } +}); diff --git a/bin/test/schema-required.test.mjs b/bin/test/schema-required.test.mjs index c5878d8d..84017e48 100644 --- a/bin/test/schema-required.test.mjs +++ b/bin/test/schema-required.test.mjs @@ -17,6 +17,7 @@ const CLOSED_SCHEMA_DEBT = [ 'bin/comment-baseline.json', 'cra/controls.json', 'lib/memory-layer-contract.json', + 'manifests/wp-weakness-catalogue.json', 'monitor/approach-taxonomy.json', 'monitor/config-correctness-ledger.example.json', 'monitor/detector-bindings.json', diff --git a/bin/test/setup-release.test.mjs b/bin/test/setup-release.test.mjs index 317b55cb..2e530fdb 100644 --- a/bin/test/setup-release.test.mjs +++ b/bin/test/setup-release.test.mjs @@ -30,7 +30,7 @@ async function rig(t, { tamper = false, signing, platform = 'linux', brew = fals if (licenses === 'shipped') writeFileSync(join(pkg, 'LICENSE'), 'MIT licence text'); if (licenses === 'missing') members.pop(); const ASSET = platform === 'win32' ? 'zz-leaks_1.0.0_windows.zip' : `zz-leaks_1.0.0_${platform}.tar.gz`; - if (platform === 'win32') execFileSync('zip', ['-qj', join(d, ASSET), join(pkg, bin), join(pkg, 'README.md')]); + if (platform === 'win32') execFileSync('zip', ['-qj', join(d, ASSET), ...members.map((m) => join(pkg, m))]); else execFileSync('tar', ['-czf', join(d, ASSET), '-C', pkg, ...members]); const body = readFileSync(join(d, ASSET)); const sha256 = createHash('sha256').update(tamper ? Buffer.concat([body, Buffer.from('x')]) : body).digest('hex'); @@ -109,6 +109,18 @@ test('a Windows pin is a zip holding the .exe, read by the same tar verb and ins assert.match(out, /installed: zz-leaks/); }); +test('a Windows pin\x27s licence file is copied out of the zip, and a zip holding none is refused', { skip: !POSIX && 'sh stubs' }, async (t) => { + const shipped = await rig(t, { platform: 'win32', signing: 'none', licenses: 'shipped' }); + const missing = await rig(t, { platform: 'win32', signing: 'none', licenses: 'missing' }); + const a = await shipped.run(['--yes']); + const b = await missing.run(['--yes']); + assert.ok(existsSync(join(shipped.binDir, 'zz-leaks.exe')), a.out); + assert.equal(readFileSync(join(shipped.licenseDir, 'zz-leaks', 'LICENSE'), 'utf8'), 'MIT licence text'); + assert.equal(existsSync(join(shipped.licenseDir, 'zz-leaks', 'README.md')), false); + assert.match(b.out, /zz-leaks: no licence file in zz-leaks_1\.0\.0_windows\.zip/); + assert.equal(existsSync(join(missing.binDir, 'zz-leaks.exe')), false); +}); + test('a pinned release asset is verified, its binary alone installed, and the off-PATH directory named', { skip: !POSIX && 'tar archives and sh stubs' }, async (t) => { const { binDir, run } = await rig(t); const { out } = await run(['--yes']); diff --git a/bin/test/unit-lib-touch-ledger-append-touch-appender.test.mjs b/bin/test/unit-lib-touch-ledger-append-touch-appender.test.mjs index a3aa7d3a..b285195c 100644 --- a/bin/test/unit-lib-touch-ledger-append-touch-appender.test.mjs +++ b/bin/test/unit-lib-touch-ledger-append-touch-appender.test.mjs @@ -1,12 +1,21 @@ // A ledger appender that writes one session touch row per call and returns 1 or 0 (bin/lib/touch-ledger-append.mjs touchAppender). -import { test } from 'node:test'; +import { test, after } from 'node:test'; import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { touchAppender } from '../lib/touch-ledger-append.mjs'; const AT = '2026-01-02T03:04:05.000Z'; +// The appender's default is the checkout's own .claude/store/touches.jsonl. A probe that names no +// ledger would write there, and bin/test/sidecar-paths.test.mjs reads that as a store written with +// nothing versioning it, so every probe gets a scratch ledger unless it names its own. +const TMP = mkdtempSync(join(tmpdir(), 'cw-touch-appender-')); +const LEDGER = join(TMP, 'touches.jsonl'); +after(() => rmSync(TMP, { recursive: true, force: true })); const pick = (row) => Object.fromEntries(['s', 'p', 'at'].filter((k) => Object.hasOwn(row, k)).map((k) => [k, row[k]])); -const probe = ({ session, repo = '/repo', pid, ledger }) => { +const probe = ({ session, repo = '/repo', pid, ledger = LEDGER }) => { const saved = process.env.CLAUDE_PID; if (pid === undefined) delete process.env.CLAUDE_PID; else process.env.CLAUDE_PID = pid; diff --git a/bin/test/unit-reference-collect-env.test.mjs b/bin/test/unit-reference-collect-env.test.mjs index 4752bbe2..9c94cd8c 100644 --- a/bin/test/unit-reference-collect-env.test.mjs +++ b/bin/test/unit-reference-collect-env.test.mjs @@ -1,14 +1,20 @@ // Which CW_ environment variables each JS or shell source reads, directly or by quoted name (bin/reference.mjs collectEnv). -import { test } from 'node:test'; +import { test, after } from 'node:test'; import assert from 'node:assert/strict'; import { collectEnv } from '../reference.mjs'; -import { basename, dirname } from 'node:path'; +import { copyFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; const here = dirname(fileURLToPath(import.meta.url)); const real = basename(fileURLToPath(import.meta.url)); const selfOtherCase = real.replace('.test.', '.Test.'); const selfDotted = './' + selfOtherCase; +// A real copy under the other-case name: opening this file by that name works only on a case-insensitive volume. +const copies = mkdtempSync(join(tmpdir(), 'collectEnv-')); +copyFileSync(fileURLToPath(import.meta.url), join(copies, selfOtherCase)); +after(() => rmSync(copies, { recursive: true, force: true })); const row = (name, how, readers) => ({ name, how, readers, description: null }); const rowsFor = (readers) => [ row('CW_BRACKET_2', 'direct', readers), @@ -30,19 +36,19 @@ test('a source or script that cannot be read contributes nothing', () => { }); test("this file's env.CW_PLAIN_ONE and env['CW_BRACKET_2'] are direct reads, and its quoted 'CW_ONLY_QUOTED' is indirect because env[key] also appears", () => { - assert.deepEqual(collectEnv(here, [selfOtherCase]), rowsFor([selfOtherCase])); + assert.deepEqual(collectEnv(copies, [selfOtherCase]), rowsFor([selfOtherCase])); }); test('a reader listed twice appears once and readers are sorted by path with ./ before letters', () => { assert.deepEqual( - collectEnv(here, [selfOtherCase, selfDotted, selfOtherCase]), + collectEnv(copies, [selfOtherCase, selfDotted, selfOtherCase]), rowsFor([selfDotted, selfOtherCase]), ); }); test('skipped and unreadable entries do not change what a readable source reports', () => { assert.deepEqual( - collectEnv(here, ['no-such-script.sh', real, selfOtherCase, 'notes.md']), + collectEnv(copies, ['no-such-script.sh', real, selfOtherCase, 'notes.md']), rowsFor([selfOtherCase]), ); }); diff --git a/bin/test/unit-secrets-sweep-scan-buffer.test.mjs b/bin/test/unit-secrets-sweep-scan-buffer.test.mjs index 21c83d31..4572a880 100644 --- a/bin/test/unit-secrets-sweep-scan-buffer.test.mjs +++ b/bin/test/unit-secrets-sweep-scan-buffer.test.mjs @@ -6,6 +6,10 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +// Built at run time: a provider-shaped literal in a tracked file blocks the push that publishes it +// (bin/test/push-protection-shapes.test.mjs). +const AWS_KEY = 'AKIA' + 'ABCDEFGHIJKLMNOP'; + test('returns binary-extension unscanned for a png path', () => { const dir = mkdtempSync(join(tmpdir(), 'scanBuffer-')); try { @@ -53,9 +57,9 @@ test('returns empty findings for a clean text buffer', () => { test('reports an AWS key on the first line with line number one', () => { const dir = mkdtempSync(join(tmpdir(), 'scanBuffer-')); try { - const buf = Buffer.from('key=AKIAABCDEFGHIJKLMNOP\n'); + const buf = Buffer.from('key=' + AWS_KEY + '\n'); const r = scanBuffer(buf, 'a.txt'); - assert.deepEqual(r, { rel: "a.txt", findings: [{ file: "a.txt", line: 1, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: "AKIAABCDEFGHIJKLMNOP", fingerprint: "16cbbe3918a1dc83" }] }); + assert.deepEqual(r, { rel: "a.txt", findings: [{ file: "a.txt", line: 1, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: AWS_KEY, fingerprint: "16cbbe3918a1dc83" }] }); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -64,9 +68,9 @@ test('reports an AWS key on the first line with line number one', () => { test('reports an AWS key on the second line with line number two', () => { const dir = mkdtempSync(join(tmpdir(), 'scanBuffer-')); try { - const buf = Buffer.from('clean line\nkey=AKIAABCDEFGHIJKLMNOP\n'); + const buf = Buffer.from('clean line\nkey=' + AWS_KEY + '\n'); const r = scanBuffer(buf, 'a.txt'); - assert.deepEqual(r, { rel: "a.txt", findings: [{ file: "a.txt", line: 2, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: "AKIAABCDEFGHIJKLMNOP", fingerprint: "16cbbe3918a1dc83" }] }); + assert.deepEqual(r, { rel: "a.txt", findings: [{ file: "a.txt", line: 2, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: AWS_KEY, fingerprint: "16cbbe3918a1dc83" }] }); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -75,9 +79,9 @@ test('reports an AWS key on the second line with line number two', () => { test('marks the result as fixture when the marker is declared in the first twenty-five lines', () => { const dir = mkdtempSync(join(tmpdir(), 'scanBuffer-')); try { - const buf = Buffer.from('secrets-sweep-fixture\nkey=AKIAABCDEFGHIJKLMNOP\n'); + const buf = Buffer.from('secrets-sweep-fixture\nkey=' + AWS_KEY + '\n'); const r = scanBuffer(buf, 'a.txt'); - assert.deepEqual(r, { rel: "a.txt", findings: [{ file: "a.txt", line: 2, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: "AKIAABCDEFGHIJKLMNOP", fingerprint: "16cbbe3918a1dc83" }], fixture: true }); + assert.deepEqual(r, { rel: "a.txt", findings: [{ file: "a.txt", line: 2, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: AWS_KEY, fingerprint: "16cbbe3918a1dc83" }], fixture: true }); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/bin/test/unit-secrets-sweep-scan-line.test.mjs b/bin/test/unit-secrets-sweep-scan-line.test.mjs index 8e122111..7477e521 100644 --- a/bin/test/unit-secrets-sweep-scan-line.test.mjs +++ b/bin/test/unit-secrets-sweep-scan-line.test.mjs @@ -3,16 +3,21 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { scanLine } from '../secrets-sweep.mjs'; +// Built at run time: a provider-shaped literal in a tracked file blocks the push that publishes it +// (bin/test/push-protection-shapes.test.mjs). +const AWS_KEY = 'AKIA' + 'ABCDEFGHIJKLMNOP'; +const GITHUB_TOKEN = 'ghp_' + 'a'.repeat(36); + test('detects an AWS access key as a real secret with correct column', () => { - const line = 'key=AKIAABCDEFGHIJKLMNOP'; + const line = 'key=' + AWS_KEY; const r = scanLine(line, 1); - assert.deepEqual(r, [{ line: 1, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: "AKIAABCDEFGHIJKLMNOP", fingerprint: "16cbbe3918a1dc83" }]); + assert.deepEqual(r, [{ line: 1, col: 5, cls: "provider-key/aws", verdict: "FALSE-POSITIVE", reason: "placeholder", match: AWS_KEY, fingerprint: "16cbbe3918a1dc83" }]); }); test('detects a GitHub token as a real secret', () => { - const line = 'token=ghp_' + 'a'.repeat(36); + const line = 'token=' + GITHUB_TOKEN; const r = scanLine(line, 1); - assert.deepEqual(r, [{ line: 1, col: 7, cls: "provider-key/github", verdict: "FALSE-POSITIVE", reason: "placeholder", match: "ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", fingerprint: "7bdccadd83905810" }]); + assert.deepEqual(r, [{ line: 1, col: 7, cls: "provider-key/github", verdict: "FALSE-POSITIVE", reason: "placeholder", match: GITHUB_TOKEN, fingerprint: "7bdccadd83905810" }]); }); test('downgrades a quoted placeholder value to false-positive', () => { diff --git a/docs/PLATFORM-SEAMS.md b/docs/PLATFORM-SEAMS.md index 7745d314..3d70c719 100644 --- a/docs/PLATFORM-SEAMS.md +++ b/docs/PLATFORM-SEAMS.md @@ -24,7 +24,7 @@ Measured 2026-10-08: | Class | Files | Lines | What it matches | |---|---:|---:|---| | `launchd` | 10 | 41 | `launchctl`, `LaunchAgents`/`LaunchDaemons`, plist schedule keys | -| `keychain` | 7 | 14 | spawning `security`, `security find-/add-/dump-…`, `keychain:` refs, `osxkeychain` | +| `keychain` | 8 | 17 | spawning `security`, `security find-/add-/dump-…`, `keychain:` refs, `osxkeychain` | | `zsh` | 1 | 2 | zsh shebangs, `'zsh'` literals, `.zshrc`/`.zprofile` | | `users-path` | 3 | 3 | `/Users/` | | `homebrew` | 21 | 47 | `/opt/homebrew`, Cellar, `brew `, `'brew'` literals | @@ -86,6 +86,7 @@ an amd64 runner when its inputs change; its three runs so far, all on 2026-10-08 | `monitor/credential-scope.mjs` | Parses `security dump-keychain` to inventory stored items | Returns `null` off darwin (unknown, not empty); `CW_CRED_DUMP` supplies a fixture | | `monitor/images.mjs` | Names `docker-credential-osxkeychain` in a fix hint | Portable already: the hint is chosen per platform, with a Linux line for `secretservice`/`pass` | | `monitor/journey.mjs` | Names `keychain:/` as the example reference in the optional credentials step's CLI hint | Portable already: the step is optional and `lib/secrets.mjs` also resolves `file:` references; only the example text names the keychain | +| `lib/update/token.mjs` | Stores, reads and deletes the update token as the Keychain item `PORTLL_UPDATE_TOKEN` (`security -i` on stdin, `find-generic-password`, `delete-generic-password`) | Portable already: off darwin the token is a 0600 `update-token` file in commitwork's config directory, and `PORTLL_UPDATE_TOKEN` in the environment or `gh auth token` are read first on every platform | ## zsh @@ -164,6 +165,7 @@ adds a seam, and say in the sections above what the portable path is. | keychain | `bin/addsecret.mjs` | | keychain | `bin/secrets.mjs` | | keychain | `lib/secrets.mjs` | +| keychain | `lib/update/token.mjs` | | keychain | `monitor/archive-batches.mjs` | | keychain | `monitor/credential-scope.mjs` | | keychain | `monitor/images.mjs` | diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 5fb729ae..27bbe9e5 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -2723,8 +2723,8 @@ cra/watch.mjs — exploited-vulnerability watch with CRA Art. ## Environment variables -931 `CW_*` names read by tracked non-test sources: 804 by name, and -127 indirect (quoted in a file that reads the environment by a computed key, so the read +931 `CW_*` names read by tracked non-test sources: 803 by name, and +128 indirect (quoted in a file that reads the environment by a computed key, so the read is likely but not shown). At most 6 readers are listed per name. A description is taken from a reader's `env` header section when one has it, otherwise from another file's, which is named. @@ -3379,7 +3379,7 @@ a reader's `env` header section when one has it, otherwise from another file's, | `CW_RELEASE_REVIEWS_SCHEMA` | direct | `bin/lib/release-reviews.mjs` | undocumented | | `CW_RELEASE_SCRATCH` | direct | `bin/release-candidate.mjs` | the directory the tests gate makes its scratch HOME in (default ~/.cache/commitwork/release-candidate), read at call time | | `CW_RELEASE_SOURCE` | direct | `bin/release-candidate.mjs`, `bin/web-roadmap.mjs` | the repository --work reads commits from; default this checkout | -| `CW_RELEASE_SOURCES` | direct | `bin/release-report.mjs` | undocumented | +| `CW_RELEASE_SOURCES` | indirect | `monitor/store-paths.mjs` | undocumented | | `CW_RELEASE_TAG_PREFIX` | direct | `bin/release-tag.mjs` | the release tag prefix, default v (`bin/changelog.mjs`) | | `CW_RELEASE_TERMS` | direct | `bin/lib/release-terms.mjs` | a list file; overrides the sidecar | | `CW_REMEDIATION_INPUT` | indirect | `monitor/remediation-pr.mjs` | undocumented | diff --git a/lib/test/claude-spawn-inventory.test.mjs b/lib/test/claude-spawn-inventory.test.mjs index 4eef9a7d..caac345a 100644 --- a/lib/test/claude-spawn-inventory.test.mjs +++ b/lib/test/claude-spawn-inventory.test.mjs @@ -33,6 +33,7 @@ const VIA_BUILDER = [ 'bin/issue-loop.mjs', 'lib/claude-spawn.mjs', 'lib/cobolwork-remediation-engines.mjs', + 'lib/judge.mjs', 'monitor/divergence-review.mjs', 'monitor/envelope-witness.mjs', 'monitor/settings.mjs', @@ -42,6 +43,8 @@ const VIA_BUILDER = [ const NOT_A_SPAWN = new Map([ ['admin/routes/issue-detail.mjs', 'the literal is the route path /api/issue/claude; the spawn it reaches is launchClaudeSession in admin/routes/remediation.mjs'], ['bin/hooks/guard-destructive.mjs', 'the header records what was measured about an answered ask and names the flags that decide it; a hook spawns no agent'], + ['bin/disclosure-review.mjs', 'the literal is the default value of --backend, a judge backend name passed to judge(); the spawn it reaches is runClaude in lib/judge.mjs'], + ['bin/judge.mjs', 'the literal is the default value of --backend, a judge backend name passed to judge(); the spawn it reaches is runClaude in lib/judge.mjs'], ['bin/agent-config.mjs','the literal is a key in the harness config-path map (.mcp.json, .claude/settings.json); this file READS agent configuration'], ['monitor/agent-run-judge.mjs', 'the literal is an exe pattern and the flags are subjects it matches in RECORDED runs; it judges logs and starts nothing'], ['monitor/process-ancestry.mjs', 'the literal is a process name in the harness-process list; this file reads ps output'], diff --git a/lib/test/unit-cobolwork-remediation-jobs-clear-job.test.mjs b/lib/test/unit-cobolwork-remediation-jobs-clear-job.test.mjs index 0c7efc7f..19e593fd 100644 --- a/lib/test/unit-cobolwork-remediation-jobs-clear-job.test.mjs +++ b/lib/test/unit-cobolwork-remediation-jobs-clear-job.test.mjs @@ -2,10 +2,13 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { clearJob } from '../cobolwork-remediation-jobs.mjs'; -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync, unlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +// unlink(2) on a directory is EPERM on macOS and EISDIR on Linux; clearJob returns whichever message the OS gives. +const unlinkRefusal = (p) => { try { unlinkSync(p); } catch (e) { return e; } return assert.fail(`unlink removed ${p}`); }; + test('returns ok true when job file exists and is removed', async () => { const dir = mkdtempSync(join(tmpdir(), 'clearJob-')); try { @@ -32,7 +35,9 @@ test('returns ok false with error when unlink fails with non-ENOENT error', asyn try { const filePath = join(dir, 'job3.json'); mkdirSync(filePath); - const expectedError = `EPERM: operation not permitted, unlink '${filePath}'`; + const refusal = unlinkRefusal(filePath); + assert.ok(['EPERM', 'EISDIR'].includes(refusal.code), `unlink of a directory gave ${refusal.code}`); + const expectedError = refusal.message; const result = await clearJob(dir, { id: 'job3' }); assert.deepEqual(result, { ok: false, error: expectedError }); } finally { diff --git a/lib/test/unit-memory-store-open-for-read.test.mjs b/lib/test/unit-memory-store-open-for-read.test.mjs index f73cdd3c..98bd9e95 100644 --- a/lib/test/unit-memory-store-open-for-read.test.mjs +++ b/lib/test/unit-memory-store-open-for-read.test.mjs @@ -2,6 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { openForRead } from '../memory-store.mjs'; +import { DatabaseSync } from 'node:sqlite'; import { mkdtempSync, rmSync, writeFileSync, mkdirSync, chmodSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -17,13 +18,22 @@ test('returns null when the file does not exist', () => { } }); +// Identity and behaviour, never the constructor's name: Node 26 renamed the class to Database and kept +// DatabaseSync as an alias, so `constructor.name` is 'DatabaseSync' on 22 and 'Database' on 26. test('returns a database handle when the file exists', () => { + // Arrange const dir = mkdtempSync(join(tmpdir(), 'openForRead-')); try { const p = join(dir, 'ok.db'); writeFileSync(p, ''); + + // Act const r = openForRead({ path: p }); - assert.equal(r.constructor.name, "DatabaseSync"); + + // Assert + assert.ok(r instanceof DatabaseSync, 'a node:sqlite DatabaseSync handle'); + assert.deepEqual({ ...r.prepare('SELECT 1 AS one').get() }, { one: 1 }, 'and it answers a query'); + r.close(); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/monitor/mcp-tools-probe.mjs b/monitor/mcp-tools-probe.mjs index 4507420d..25b62126 100644 --- a/monitor/mcp-tools-probe.mjs +++ b/monitor/mcp-tools-probe.mjs @@ -137,6 +137,9 @@ export function speak(argv, { env, cwd, timeout }) { } }); child.on('exit', (code, signal) => finish({ ok: false, why: `exited ${signal || code} before answering tools/list${said()}` })); + // A server that exits before reading its stdin turns these writes into EPIPE. Its exit is the + // verdict ('exited … before answering'), so the write error is expected and must not throw. + child.stdin.on('error', (e) => { if (e.code !== 'EPIPE' && e.code !== 'ECONNRESET') finish({ ok: false, why: `stdin: ${e.code || e.message}` }); }); for (const f of RPC) child.stdin.write(`${JSON.stringify(f)}\n`); }); } diff --git a/monitor/nondeterministic-store.mjs b/monitor/nondeterministic-store.mjs index 133039b7..54e1070a 100644 --- a/monitor/nondeterministic-store.mjs +++ b/monitor/nondeterministic-store.mjs @@ -73,13 +73,36 @@ const heldRecords = new Set(); * withRecordLock instead of consulting it. */ export const holdsRecordLock = (path) => heldRecords.has(resolve(path)); -/** Serialise one record file's load/mutate/write, exactly as withIssuesLock does. */ -export function withRecordLock(path, fn, { attempts = 50 } = {}) { +// A WAITER WAITS FOR AS LONG AS A HOLDER MAY LIVE. The budget used to be 50 rounds of a 20 ms spin, +// about one second, against a 30 s stale window: a contended writer gave up long before the lock +// could be anything but live and busy, and its observation was dropped (thrown, never written) on a +// slow runner with eight writers. The wait is now wall-clock and spans the stale window plus a +// margin, so within the budget a waiter either gets the lock or takes over a dead holder's; it fails +// loud only past that, which no live holder of one short read-modify-write explains. +// +// It SLEEPS between rounds (Atomics.wait), never burns: seven spinning waiters on a two- or +// three-core runner starve the one process that can release the lock. The backoff is jittered so +// waiters that collided once do not retry in lockstep and collide again. +const RECORD_STALE_MS = 30_000; +export const RECORD_WAIT_MS = RECORD_STALE_MS + 5_000; +const sleep = (ms) => { if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }; +const backoffMs = (round) => { const cap = Math.min(100, 2 ** Math.min(round, 7)); return cap / 2 + Math.random() * (cap / 2); }; + +/** Serialise one record file's load/mutate/write. `attempts: 1` is single shot (refuse if held); + * otherwise it waits up to `waitMs` of wall clock. */ +export function withRecordLock(path, fn, { attempts = Infinity, waitMs = RECORD_WAIT_MS } = {}) { const target = resolve(path); - const held = acquireLock(`${target}.lock`, { - staleMs: 30_000, label: 'nondeterministic-store', attempts, spinMs: 20, - onStale: (ms) => console.warn(`[nondet] breaking a stale record lock (${Math.round(ms / 1000)}s old)`), - }); + const deadline = Date.now() + waitMs; + let held; + for (let round = 1; ; round++) { + held = acquireLock(`${target}.lock`, { + staleMs: RECORD_STALE_MS, label: 'nondeterministic-store', attempts: 1, + onStale: (ms) => console.warn(`[nondet] breaking a stale record lock (${Math.round(ms / 1000)}s old)`), + }); + const left = deadline - Date.now(); + if (held.ok || round >= attempts || left <= 0) break; + sleep(Math.min(backoffMs(round), left)); + } if (!held.ok) throw new Error(`record ${target} is locked by another process (${target}.lock); try again`); heldRecords.add(target); // Deregister before release so writeRows never believes a handed-on lock is held. diff --git a/monitor/perf-tuning.mjs b/monitor/perf-tuning.mjs index d6493b3e..7b176390 100644 --- a/monitor/perf-tuning.mjs +++ b/monitor/perf-tuning.mjs @@ -72,12 +72,14 @@ export const INTENSITY_LEVELS = () => loadProfiles().intensityLevels; // it is reported as `unknown` rather than guessed at `nvme` — guessing the fast case is exactly the // direction that produces a flattering number, and the panel asks the operator instead. -export function detectHardware({ env = process.env } = {}) { - const list = cpus() || []; +// `probe` replaces the node:os readings ({ cpus, totalmem, arch, platform }) so a test can describe +// a machine instead of asserting whichever one it happens to run on. +export function detectHardware({ env = process.env, probe = {} } = {}) { + const list = (probe.cpus || cpus)() || []; const cores = Number(env.CW_PERF_CORES) || list.length || 1; - const ramGB = Number(env.CW_PERF_RAM_GB) || Math.round(totalmem() / (1024 ** 3)); - const arch = env.CW_PERF_ARCH || osArch(); - const platform = env.CW_PERF_PLATFORM || osPlatform(); + const ramGB = Number(env.CW_PERF_RAM_GB) || Math.round((probe.totalmem || totalmem)() / (1024 ** 3)); + const arch = env.CW_PERF_ARCH || (probe.arch || osArch)(); + const platform = env.CW_PERF_PLATFORM || (probe.platform || osPlatform)(); const model = list[0]?.model || 'unknown'; const diskClass = env.CW_PERF_DISK || 'unknown'; const containers = env.CW_PERF_CONTAINERS === undefined ? null : env.CW_PERF_CONTAINERS !== '0'; diff --git a/monitor/registry-inventory.json b/monitor/registry-inventory.json index 6104656c..935d751b 100644 --- a/monitor/registry-inventory.json +++ b/monitor/registry-inventory.json @@ -198,6 +198,13 @@ "schema": "schema/wire-layer.schema.json", "binding": "validator" }, + { + "path": "manifests/wp-weakness-catalogue.json", + "kind": "declaration", + "schema": "schema/wp-weakness-catalogue.schema.json", + "binding": "mirrored", + "note": "bin/wp-posture.mjs loadCatalogue validates it by hand and never consults the schema; bin/test/schema-required.test.mjs holds the shipped catalogue to the schema." + }, { "path": "monitor/annotations.example.json", "kind": "store", diff --git a/monitor/store-paths.mjs b/monitor/store-paths.mjs index 14cd43f1..a1e70a53 100644 --- a/monitor/store-paths.mjs +++ b/monitor/store-paths.mjs @@ -306,6 +306,9 @@ export const privateDocsiteDirFor = privateRecord('CW_DOCSITE_PRIVATE', 'docsite /** Warnings an operator saw and set aside (monitor/disregarded-warnings.mjs). CW_DISREGARDED_WARNINGS. Absent = none set aside. */ export const disregardedWarningsPathFor = privateRecord('CW_DISREGARDED_WARNINGS', 'disregarded-warnings.json'); +/** The deploy boxes bin/release-report.mjs --fetch pulls release evidence from. CW_RELEASE_SOURCES. Absent = nothing to fetch. */ +export const releaseSourcesPathFor = privateRecord('CW_RELEASE_SOURCES', 'releases.json'); + /** Actions refused under unattended posture C, one JSON line each (lib/security-posture.mjs). CW_APPROVAL_QUEUE. Absent = nothing refused yet. */ export const approvalQueuePathFor = privateRecord('CW_APPROVAL_QUEUE', 'approval-queue.jsonl'); diff --git a/monitor/taxonomy-editions.json b/monitor/taxonomy-editions.json index 28e462fe..367137ea 100644 --- a/monitor/taxonomy-editions.json +++ b/monitor/taxonomy-editions.json @@ -820,6 +820,45 @@ } ], "countAfter": 207 + }, + { + "version": 18, + "label": "The register described without its environment", + "date": "2026-11-04", + "form": "JSON registry, sqlite projection", + "artifact": "monitor/failure-taxonomy.json", + "tracked": true, + "headline": "No class was minted or retired and no score moved: the prose of 123 of the 207 classes was reworded to name the role a thing played rather than the product, tool or session involved, and the scanner redaction roster shipped empty.", + "body": "Reconstructed from the commit that bumped the registry from 17 to 18, because the version moved and its entry was not authored with it. Every statement here was derived by comparing the registry on either side of that commit; the reasons behind individual rewordings are not given, because the commit does not carry them. The class count does not move: 207 before and after, the same ids in the same order, with every closure and gain score unchanged. What changed is wording. The example text of 117 classes, the score basis of 23, the predicate of four, the description of two and the name of one were rewritten, and nine classes had mitigation or proposal text reworded. Outside the classes, eight of the 26 remediation rows, six cycle log entries, two attribution plan rows, and the registry note and provenance fields were reworded the same way. Three statements changed in substance rather than in wording: the second-pass agreement figure now counts changed classes and changed scores separately, the mitigation provenance no longer restates per-product coverage counts and says the page computes them from the register, and the recorded count of classes without evidence under the published predicate fell from 64 to 61, which re-running that predicate on each side of the commit reproduces. The same commit emptied the scanner redaction roster, because the register no longer holds the text those redactions matched; it relaxed the roster schema to allow an empty list and moved the redaction tests onto a fixture roster, with a new test that refuses a catalogued scanner name in any rendered class field. The substrate map and the roster note stamp this work 2026-10-29; this entry carries the date of the commit.", + "changes": [ + { + "kind": "classes", + "date": "2026-11-04", + "ids": [], + "note": "No class minted or retired; the count stands at 207 from edition 17, with the same ids in the same order and no closure or gain score changed." + }, + { + "kind": "field", + "date": "2026-11-04", + "note": "Class prose reworded in 123 of 207 classes to name roles rather than products, tools or sessions: example in 117, scoreBasis in 23, predicate in four (A10, P2, W7, K4), description in two (R12, K11), name in one (K4), and mitigation or proposal text in nine. The same rewording reached eight of 26 remediation rows, six cycle log entries, two attribution plan rows and the note, sourceProvenance, stpaVocabulary and stpaProvenance fields." + }, + { + "kind": "field", + "date": "2026-11-04", + "note": "scoreProvenance restated the second-pass agreement figure from 28 scores changed and 171 confirmed to 28 classes changed (45 of 398 scores: closure 22, gain 23) and 171 confirmed. No score moved in this edition; the figure describes the earlier pass." + }, + { + "kind": "field", + "date": "2026-11-04", + "note": "mitigationProvenance dropped its hand-written per-product coverage, no-record and proposal counts and states that the page computes them from the register as rendered; evidencePredicate.count moved from 64 to 61 with the rule and matchers unchanged." + }, + { + "kind": "field", + "date": "2026-11-04", + "note": "The substrate map took the reworded predicates of A10, P2 and W7 and its verification stamp moved to registry version 18, with 207 classes, 201 mapped and 6 unmapped unchanged. The scanner redaction roster went from six redactions to none and its schema now permits an empty list." + } + ], + "countAfter": 207 } ] } diff --git a/monitor/test/host-role.test.mjs b/monitor/test/host-role.test.mjs index fefd1a53..02b98880 100644 --- a/monitor/test/host-role.test.mjs +++ b/monitor/test/host-role.test.mjs @@ -84,7 +84,7 @@ describe('resolveHostRole', () => { }); test('the presence threshold is env-overridable at call time', async () => { - const probe = { inputIdleSec: idleFor(120), onBattery: () => ({ value: false }), machineBusy: quiet }; + const probe = { localHour: day, inputIdleSec: idleFor(120), onBattery: () => ({ value: false }), machineBusy: quiet }; assert.equal((await resolveHostRole({ setting: auto, env: {}, probe })).role, 'shared'); assert.equal((await resolveHostRole({ setting: auto, env: { CW_HOST_PRESENCE_IDLE_S: '60' }, probe })).role, 'server'); }); diff --git a/monitor/test/install-agents-docker-gc.test.mjs b/monitor/test/install-agents-docker-gc.test.mjs index a3cb4f34..3c265615 100644 --- a/monitor/test/install-agents-docker-gc.test.mjs +++ b/monitor/test/install-agents-docker-gc.test.mjs @@ -77,8 +77,9 @@ test('paths are derived: DOCKER_CONFIG follows CW_DOCKER_CONFIG at call time, no test('the home-derived parts follow HOME, so nothing is pinned to one machine', () => { const fake = mkdtempSync(join(tmpdir(), 'cw-gc-home-')); try { - const code = 'import(process.argv[1]).then((m) => process.stdout.write(m.plist(m.AGENTS.find((a) => a.label === process.argv[2]))))'; - const run = () => spawnSync(process.execPath, ['-e', code, MODULE, LABEL], { + // The label goes first: with the module as argv[1] the child counts as running it directly, and the CLI block runs. + const code = 'import(process.argv[2]).then((m) => process.stdout.write(m.plist(m.AGENTS.find((a) => a.label === process.argv[1]))))'; + const run = () => spawnSync(process.execPath, ['-e', code, LABEL, MODULE], { encoding: 'utf8', env: { ...process.env, HOME: fake, CW_DOCKER_CONFIG: '', CW_SELF_SWEEP: '0' }, }); const a = run(); diff --git a/monitor/test/nondeterministic-store-lock.test.mjs b/monitor/test/nondeterministic-store-lock.test.mjs index 8d5d3a3c..ff6c9b9f 100644 --- a/monitor/test/nondeterministic-store-lock.test.mjs +++ b/monitor/test/nondeterministic-store-lock.test.mjs @@ -14,7 +14,8 @@ // 1. The writers must actually overlap. `spawnSync` in a loop does not overlap — it blocks until // each child exits, so twelve "concurrent" writers run one after another and the contention // under test never occurs. The writers below are started with async `spawn` AND hold at a -// shared wall-clock barrier, so the collision is arranged rather than hoped for. +// barrier (each reports READY, all are released together), so the collision is arranged rather +// than hoped for. // 2. The counting must be able to fail. A conservation check that cannot catch a loss is // decoration, so the control test points the same counting at the unlocked read-modify-write // this change replaced, with the interleaving forced: it must lose a row. If the first test @@ -42,32 +43,58 @@ const storeEnv = (store) => ({ ...process.env, CW_NONDET_STORE: store, CW_NONDET const recordPath = (store) => join(store, DIMENSION, `${SUBJECT}.jsonl`); const rowsIn = (p) => readFileSync(p, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); +// Each writer reports READY once booted and imported, then holds until the parent writes the go file: +// a fixed wall-clock barrier is missed by a writer that boots late on a slow runner, and a missed +// barrier quietly turns the concurrent writers back into a sequence the lock is never tested against. +const spawnWriter = (prog, args, env) => { + const kid = spawn(process.execPath, [prog, ...args], { env, stdio: ['ignore', 'pipe', 'pipe'] }); + let err = ''; + kid.stderr.on('data', (d) => { err += d; }); + const ready = new Promise((ok, fail) => { + kid.stdout.on('data', (d) => { if (String(d).includes('READY')) ok(); }); + kid.on('error', fail); + kid.on('exit', (code) => fail(new Error(`writer exited ${code} before READY: ${err}`))); + }); + const exited = new Promise((ok, fail) => { + kid.on('error', fail); + kid.on('exit', (code) => (code === 0 ? ok() : fail(new Error(`writer ${args[0]} exited ${code}: ${err}`)))); + }); + exited.catch(() => {}); // observed by the caller's await; a failure before READY is reported by `ready` + return { ready, exited }; +}; +// The child must have existsSync and go in scope. +const HOLD_UNTIL_GO = ` +process.stdout.write('READY\\n'); +while (!existsSync(go)) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 2); +`; + test(`CONSERVATION — ${WRITERS} processes x ${PER_WRITER} observations, each present exactly once`, async () => { + // Arrange const dir = scratch(); try { const store = join(dir, 'store'); const prog = join(dir, 'w.mjs'); + const go = join(dir, 'go'); // pathToFileURL is named AT the interpolation, not behind a const: bin/test/esm-specifier-shape.mjs // matches this shape by spelling on purpose, because a variable holding `C:\\...` looks identical // to one holding a URL and the child rejects the first with ERR_UNSUPPORTED_ESM_URL_SCHEME. writeFileSync(prog, ` +import { existsSync } from 'node:fs'; import { record } from ${JSON.stringify(pathToFileURL(STORE_PATH).href)}; -const [writer, n, startAt] = process.argv.slice(2); -while (Date.now() < Number(startAt)) { /* hold at the barrier so every writer arrives together */ } +const [writer, n, go] = process.argv.slice(2); +${HOLD_UNTIL_GO} for (let i = 0; i < Number(n); i++) { record({ subject: '${SUBJECT}', dimension: '${DIMENSION}', score: 0.5, detail: { writer: Number(writer), i } }); } `); - const startAt = Date.now() + 700; // enough for every child to boot and reach the barrier - const kids = Array.from({ length: WRITERS }, (_, w) => new Promise((done, fail) => { - const kid = spawn(process.execPath, [prog, String(w), String(PER_WRITER), String(startAt)], { env: storeEnv(store), stdio: ['ignore', 'ignore', 'pipe'] }); - let err = ''; - kid.stderr.on('data', (d) => { err += d; }); - kid.on('error', fail); - kid.on('exit', (code) => (code === 0 ? done() : fail(new Error(`writer ${w} exited ${code}: ${err}`)))); - })); - await Promise.all(kids); + const kids = Array.from({ length: WRITERS }, (_, w) => spawnWriter(prog, [String(w), String(PER_WRITER), go], storeEnv(store))); + await Promise.all(kids.map((k) => k.ready)); + + // Act: release every writer at once. + writeFileSync(go, ''); + await Promise.all(kids.map((k) => k.exited)); + // Assert const rows = rowsIn(recordPath(store)); assert.equal(rows.length, WRITERS * PER_WRITER, `expected ${WRITERS * PER_WRITER} observations, found ${rows.length} — a short count is a lost update, and the file is well-formed either way`); @@ -78,6 +105,47 @@ for (let i = 0; i < Number(n); i++) { } }); +test('a writer WAITS OUT a live holder that outlasts the old one-second budget, and its observation lands', async () => { + // Arrange: another process takes the record lock and keeps it for HOLD_MS. + const HOLD_MS = 1_500; + const dir = scratch(); + try { + const store = join(dir, 'store'); + const p = recordPath(store); + mkdirSync(dirname(p), { recursive: true }); + const prog = join(dir, 'holder.mjs'); + const go = join(dir, 'go'); + writeFileSync(prog, ` +import { existsSync } from 'node:fs'; +import { withRecordLock } from ${JSON.stringify(pathToFileURL(STORE_PATH).href)}; +const [label, p, go] = process.argv.slice(2); +withRecordLock(p, () => { + ${HOLD_UNTIL_GO} + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ${HOLD_MS}); +}, { attempts: 1 }); +`); + const holder = spawnWriter(prog, ['holder', p, go], storeEnv(store)); + await holder.ready; + writeFileSync(go, ''); + process.env.CW_NONDET_STORE = store; + process.env.CW_NONDET_HOT = '1000'; + + // Act + const started = Date.now(); + record({ subject: SUBJECT, dimension: DIMENSION, score: 0.2 }); + const waited = Date.now() - started; + await holder.exited; + + // Assert + assert.ok(waited >= HOLD_MS / 2, `the lock was held while record() ran (waited ${waited} ms)`); + assert.equal(read(SUBJECT, DIMENSION).length, 1, 'the contended observation is written, not dropped'); + } finally { + delete process.env.CW_NONDET_STORE; + delete process.env.CW_NONDET_HOT; + rmSync(dir, { recursive: true, force: true }); + } +}); + test('the control — the same counting DOES catch the unlocked read-modify-write it replaced', () => { const dir = scratch(); try { diff --git a/monitor/test/unit-defect-classify-group-by-evidence.test.mjs b/monitor/test/unit-defect-classify-group-by-evidence.test.mjs index 9a57adfc..0e541bf9 100644 --- a/monitor/test/unit-defect-classify-group-by-evidence.test.mjs +++ b/monitor/test/unit-defect-classify-group-by-evidence.test.mjs @@ -120,7 +120,8 @@ test('handles a closure whose evidence is a bigint by stringifying it', () => { }); test('handles a closure whose evidence is a date by stringifying it', () => { - assert.deepEqual(groupByEvidence([{ evidence: new Date(0), issueId: 'i1' }]), [{ evidence: "Thu Jan 01 1970 09:30:00 GMT+0930 (Australian Central Standard Time)", issueIds: ["i1"], unusable: false, sha: null }]); + const epoch = new Date(0); + assert.deepEqual(groupByEvidence([{ evidence: epoch, issueId: 'i1' }]), [{ evidence: String(epoch), issueIds: ["i1"], unusable: false, sha: null }]); }); test('handles a closure whose evidence is a regex by stringifying it', () => { diff --git a/monitor/test/unit-images-explain-pull-failure.test.mjs b/monitor/test/unit-images-explain-pull-failure.test.mjs index 5830fa72..bf489ab3 100644 --- a/monitor/test/unit-images-explain-pull-failure.test.mjs +++ b/monitor/test/unit-images-explain-pull-failure.test.mjs @@ -3,6 +3,13 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { explainPullFailure } from '../images.mjs'; +// The fix is chosen from process.platform at call time, so each expectation names the platform it is for. +const onPlatform = (name, fn) => { + const saved = Object.getOwnPropertyDescriptor(process, 'platform'); + Object.defineProperty(process, 'platform', { ...saved, value: name }); + try { return fn(); } finally { Object.defineProperty(process, 'platform', saved); } +}; + test('returns the raw value unchanged when it is null', () => { assert.equal(explainPullFailure(null), null); }); @@ -12,7 +19,7 @@ test('returns the raw value unchanged when it is an empty string', () => { }); test('returns the raw value unchanged when the text lacks credential keywords', () => { - assert.equal(explainPullFailure('docker-credential-osxkeychain failed'), "docker-credential-osxkeychain failed -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); + assert.equal(onPlatform('darwin', () => explainPullFailure('docker-credential-osxkeychain failed')), "docker-credential-osxkeychain failed -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); }); test('returns the raw value unchanged when the text has keywords but no helper name', () => { @@ -20,7 +27,7 @@ test('returns the raw value unchanged when the text has keywords but no helper n }); test('returns the raw value unchanged when the text has a helper name but no keywords', () => { - assert.equal(explainPullFailure('docker-credential-osxkeychain'), "docker-credential-osxkeychain -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); + assert.equal(onPlatform('darwin', () => explainPullFailure('docker-credential-osxkeychain')), "docker-credential-osxkeychain -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); }); test('returns the raw value unchanged when the text has neither keywords nor a helper name', () => { @@ -28,11 +35,15 @@ test('returns the raw value unchanged when the text has neither keywords nor a h }); test('appends the platform-specific fix when the text has both keywords and a helper name', () => { - assert.equal(explainPullFailure('error getting credentials for docker-credential-osxkeychain'), "error getting credentials for docker-credential-osxkeychain -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); + assert.equal(onPlatform('darwin', () => explainPullFailure('error getting credentials for docker-credential-osxkeychain')), "error getting credentials for docker-credential-osxkeychain -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); }); test('appends the platform-specific fix when the text has credential keyword and a helper name', () => { - assert.equal(explainPullFailure('credential helper docker-credential-wincred not found'), "credential helper docker-credential-wincred not found -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); + assert.equal(onPlatform('darwin', () => explainPullFailure('credential helper docker-credential-wincred not found')), "credential helper docker-credential-wincred not found -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: brew install docker-credential-helper (provides docker-credential-osxkeychain)"); +}); + +test('appends the Linux fix, not the brew one, when the same text is explained on linux', () => { + assert.equal(onPlatform('linux', () => explainPullFailure('error getting credentials for docker-credential-osxkeychain')), 'error getting credentials for docker-credential-osxkeychain -- the config names a credential helper that is not installed, and these images are PUBLIC and need none. Fix: install docker-credential-secretservice or docker-credential-pass, or remove "credsStore" from ~/.docker/config.json — these images are public and need no credentials'); }); test('returns the raw value unchanged when the text is a non-string object', () => { diff --git a/monitor/test/unit-mcp-tools-probe-arg-read-allowed.test.mjs b/monitor/test/unit-mcp-tools-probe-arg-read-allowed.test.mjs index 8de74214..4a180d31 100644 --- a/monitor/test/unit-mcp-tools-probe-arg-read-allowed.test.mjs +++ b/monitor/test/unit-mcp-tools-probe-arg-read-allowed.test.mjs @@ -20,10 +20,11 @@ test('returns false when path equals home', () => { test('returns false when home is inside path', () => { const dir = mkdtempSync(join(tmpdir(), 'argReadAllowed-')); try { - const home = join(dir, 'sub'); + const parent = realpathSync(dir); + const home = join(parent, 'sub'); mkdirSync(home, { recursive: true }); - const r = argReadAllowed(dir, home); - assert.equal(r, true); + const r = argReadAllowed(parent, home); + assert.equal(r, false); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -86,10 +87,11 @@ test('returns false for path outside home that is not mount allowed', () => { test('returns false when path is home parent and home is child', () => { const dir = mkdtempSync(join(tmpdir(), 'argReadAllowed-')); try { - const home = join(dir, 'a', 'b'); + const ancestor = realpathSync(dir); + const home = join(ancestor, 'a', 'b'); mkdirSync(home, { recursive: true }); - const r = argReadAllowed(dir, home); - assert.equal(r, true); + const r = argReadAllowed(ancestor, home); + assert.equal(r, false); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/monitor/test/unit-mcp-tools-probe-speak.test.mjs b/monitor/test/unit-mcp-tools-probe-speak.test.mjs new file mode 100644 index 00000000..fa533277 --- /dev/null +++ b/monitor/test/unit-mcp-tools-probe-speak.test.mjs @@ -0,0 +1,19 @@ +// speak() against a server that exits before it reads a frame. The three writes then hit a closed +// pipe; the verdict must be the server's exit, and the EPIPE must not escape as an unhandled error. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { speak } from '../mcp-tools-probe.mjs'; + +test('a server that exits at once is reported as exited, and the write to its closed stdin does not throw', async () => { + // Arrange + const argv = [process.execPath, '-e', 'process.stdin.destroy(); process.exit(3)']; + // Act + const runs = []; + for (let i = 0; i < 20; i++) runs.push(await speak(argv, { env: process.env, cwd: process.cwd(), timeout: 5000 })); + // Assert + for (const r of runs) { + assert.equal(r.ok, false); + assert.match(r.why, /^(exited 3 before answering tools\/list|stdin: )/); + assert.doesNotMatch(r.why, /^stdin: (EPIPE|ECONNRESET)/); + } +}); diff --git a/monitor/test/unit-perf-tuning-detect-hardware.test.mjs b/monitor/test/unit-perf-tuning-detect-hardware.test.mjs index aa27f720..06d2e2e9 100644 --- a/monitor/test/unit-perf-tuning-detect-hardware.test.mjs +++ b/monitor/test/unit-perf-tuning-detect-hardware.test.mjs @@ -3,28 +3,41 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { detectHardware } from '../perf-tuning.mjs'; +// The machine under test is DESCRIBED here and handed in as `probe`; nothing below reads the host. +const host = { + cpus: () => Array.from({ length: 18 }, () => ({ model: 'Apple M5 Pro' })), + totalmem: () => 48 * 1024 ** 3, + arch: () => 'arm64', + platform: () => 'darwin', +}; + test('returns cores from env when CW_PERF_CORES is set', () => { const r = detectHardware({ env: { CW_PERF_CORES: '8' } }); assert.equal(r.cores, 8); }); test('returns cores from cpus length when CW_PERF_CORES is unset', () => { - const r = detectHardware({ env: {} }); + const r = detectHardware({ env: {}, probe: host }); assert.equal(r.cores, 18); }); -test('returns cores as 1 when CW_PERF_CORES is 0 and cpus is empty', () => { - const r = detectHardware({ env: { CW_PERF_CORES: '0' } }); +test('returns cores from cpus length when CW_PERF_CORES is 0', () => { + const r = detectHardware({ env: { CW_PERF_CORES: '0' }, probe: host }); assert.equal(r.cores, 18); }); +test('returns cores as 1 when CW_PERF_CORES is 0 and cpus is empty', () => { + const r = detectHardware({ env: { CW_PERF_CORES: '0' }, probe: { ...host, cpus: () => [] } }); + assert.equal(r.cores, 1); +}); + test('returns ramGB from env when CW_PERF_RAM_GB is set', () => { const r = detectHardware({ env: { CW_PERF_RAM_GB: '16' } }); assert.equal(r.ramGB, 16); }); test('returns ramGB from totalmem when CW_PERF_RAM_GB is unset', () => { - const r = detectHardware({ env: {} }); + const r = detectHardware({ env: {}, probe: host }); assert.equal(r.ramGB, 48); }); @@ -34,7 +47,7 @@ test('returns arch from env when CW_PERF_ARCH is set', () => { }); test('returns arch from osArch when CW_PERF_ARCH is unset', () => { - const r = detectHardware({ env: {} }); + const r = detectHardware({ env: {}, probe: host }); assert.equal(r.arch, "arm64"); }); @@ -44,12 +57,12 @@ test('returns platform from env when CW_PERF_PLATFORM is set', () => { }); test('returns platform from osPlatform when CW_PERF_PLATFORM is unset', () => { - const r = detectHardware({ env: {} }); + const r = detectHardware({ env: {}, probe: host }); assert.equal(r.platform, "darwin"); }); test('returns model from cpus list when available', () => { - const r = detectHardware({ env: {} }); + const r = detectHardware({ env: {}, probe: host }); assert.equal(r.model, "Apple M5 Pro"); }); diff --git a/monitor/test/unit-sweep-calendar-calendar-lateness.test.mjs b/monitor/test/unit-sweep-calendar-calendar-lateness.test.mjs index f215fc08..e9915914 100644 --- a/monitor/test/unit-sweep-calendar-calendar-lateness.test.mjs +++ b/monitor/test/unit-sweep-calendar-calendar-lateness.test.mjs @@ -3,6 +3,11 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { calendarLateness } from '../sweep-calendar.mjs'; +// A calendar slot is the machine's LOCAL wall-clock time (launchd StartCalendarInterval), so every +// expected instant below depends on the zone. Pinned to a half-hour offset on purpose: under UTC a +// calendarLateness that wrongly read the slot as UTC would pass unnoticed. +process.env.TZ = 'Australia/Adelaide'; + test('returns null when calendar is invalid', () => { const r = calendarLateness({ calendar: '25:00', now: 1700000000000 }); assert.equal(r, null); diff --git a/package.json b/package.json index b2ce45ed..15fef494 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "admin/", "provenance/", "mcp/", + "codegraph/", + "chunk-diff/", "LICENSING.md", "LICENSE-APACHE-2.0", "docs/AGPL-SCOPE.md", diff --git a/schema/wp-weakness-catalogue.schema.json b/schema/wp-weakness-catalogue.schema.json new file mode 100644 index 00000000..127304f3 --- /dev/null +++ b/schema/wp-weakness-catalogue.schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://commitwork.portll.net/schema/wp-weakness-catalogue.schema.json", + "title": "WordPress weakness catalogue", + "$comment": "MIRRORED, not applied. bin/wp-posture.mjs loadCatalogue is what runs: it throws when entries is not an array, and sets aside (as invalid, counted in the report) an entry whose id, sev, cwe, predicate verb or type, count threshold, given type, where operators, enables routes or detail template it refuses. Required fields and enumerations here are the ones that validator enforces, NOT every field the current entries happen to carry. Two of its checks are not expressible here and stay with it: an id is unique, and every name in enables is a key of routes.", + "type": "object", + "required": ["entries"], + "properties": { + "schema": { "type": "integer", "minimum": 1 }, + "name": { "type": "string" }, + "note": { "type": "string" }, + "routes": { + "type": "object", + "$comment": "Route id to the sentence describing it. An entry's enables may only name a key of this object.", + "additionalProperties": { "type": "string" } + }, + "componentClasses": { + "type": "object", + "$comment": "Passed to bin/lib/wp-observables.mjs collectObservables as `classes`; that module owns the shape of each class." + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "sev", "cwe", "predicate", "detail"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,63}$" }, + "name": { "type": "string" }, + "sev": { "enum": ["crit", "high", "med", "low"] }, + "cwe": { "type": "string", "pattern": "^CWE-\\d+$" }, + "predicate": { + "type": "object", + "required": ["verb", "type"], + "$comment": "count-gt and count-lt additionally need a finite numeric `than`; loadCatalogue enforces that pairing.", + "properties": { + "verb": { "enum": ["exists", "absent", "count-gt", "count-lt"] }, + "type": { "enum": ["wp-install", "wp-component", "wp-option", "wp-setting", "wp-user", "audit-source", "served-dir", "file", "server-directive"] }, + "where": { "type": "object", "$comment": "Field to a scalar, a list (membership) or an operator object: neq, in, notIn, gt, gte, lt, lte, contains, notContains, startsWith, endsWith, regex, exists. An unknown operator is refused by loadCatalogue." }, + "than": { "type": "number" }, + "given": { + "type": "object", + "required": ["type"], + "properties": { + "type": { "enum": ["wp-install", "wp-component", "wp-option", "wp-setting", "wp-user", "audit-source", "served-dir", "file", "server-directive"] }, + "where": { "type": "object" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "detail": { "type": "string", "minLength": 1 }, + "enables": { "type": "array", "items": { "type": "string" } }, + "refs": { "type": "object" }, + "course": { "type": "string" } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +}