diff --git a/admin/routes/dfd.mjs b/admin/routes/dfd.mjs new file mode 100644 index 00000000..29ca3293 --- /dev/null +++ b/admin/routes/dfd.mjs @@ -0,0 +1,146 @@ +// admin/routes/dfd.mjs — the data flow diagram behind the panel login. +// +// GET /dfd/ the page (renderHtml), sent under the page's own hash-pinned CSP +// GET /api/dfd the filtered dfd@1 document as JSON +// +// Source, read at call time: the dfd@1 file CW_DFD_FILE names, else the flow store's graph +// (/flow.json) through the adapter. The query mirrors the command's flags — +// prefix, focus, hops, direction, collapse, collapse-all-groups-over, orientation, max, tests, +// imports — and goes through the command's own parser and loader (flow/dfd-cli.mjs), so the page +// and `node flow/dfd-cli.mjs` read, filter and refuse alike. collapse and orientation shape the +// drawing only; /api/dfd returns the document before either. +// +// `repo=` draws a scanned repository instead: the static pass over the checkout the panel's +// own fleet resolver (admin/lib/core.mjs resolvedRepos, the sweep's resolver) names `name`. Only a +// name is taken — never a path from the query: a value that is not a registry name (letters, +// digits, `.`, `_`, `-`, not `.` or `..`) is refused 400 before any lookup, and a name the +// registry does not know is refused 400 with the reason. The pass runs in a child (`node +// --experimental-vm-modules flow/dfd-cli.mjs --repo --emit model`): the panel +// has no vm modules, and the child reads the target's sources without running them. The model it +// returns is filtered and collapsed here, through the same prepare() as every other source. +// +// Dispatched below the login gate (admin/serve.mjs MODULAR_ROUTES), so every route here already +// has a session. + +import { execFile } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { parseArgs, prepare, EXIT, DEFAULT_MAX } from '../../flow/dfd-cli.mjs'; +import { renderHtml } from '../../lib/dfd-render.mjs'; +import { MARK_ICON } from '../../lib/brand-tokens.mjs'; +import { esc } from '../../lib/html-escape.mjs'; +import { resolvedRepos } from '../lib/core.mjs'; + +const CLI = fileURLToPath(new URL('../../flow/dfd-cli.mjs', import.meta.url)); +export const REPO_NAME_RE = /^[A-Za-z0-9._-]{1,128}$/; +export const PASS_TIMEOUT_MS = 180_000; + +const FILTER_PARAMS = ['prefix', 'focus', 'hops', 'direction']; +const DRAWING_PARAMS = ['collapse', 'collapse-all-groups-over', 'orientation', 'max']; +const BOOL_PARAMS = ['tests', 'imports']; +const STATUS = { [EXIT.USAGE]: 400, [EXIT.INVALID]: 422, [EXIT.INPUT]: 404, [EXIT.SIZE]: 422 }; + +/** + * `repo=` -> { path } of the registered checkout, or { error } (always a 400). The name is a + * key into the registry and nothing else: it is never joined to a directory. + */ +export function registeredRepo(name, repos = resolvedRepos) { + if (!REPO_NAME_RE.test(name) || name === '.' || name === '..') { + return { error: `repo must name a repository the panel knows (letters, digits, . _ -), not a path: ${JSON.stringify(name.slice(0, 80))}` }; + } + const list = repos() || []; + const hit = list.find((r) => r && r.name === name && typeof r.path === 'string' && r.path); + if (!hit) return { error: `unknown repository ${name}: the panel's registry names ${list.length} repositor${list.length === 1 ? 'y' : 'ies'}, and not this one` }; + return { path: hit.path }; +} + +/** Run the static pass on a registered checkout in a child -> { model } or { status, error }. */ +export function passInChild(path, { env = process.env, timeoutMs = PASS_TIMEOUT_MS, tests = false, imports = false, exec = execFile } = {}) { + const argv = ['--experimental-vm-modules', '--no-warnings', CLI, '--repo', path, '--emit', 'model', ...(tests ? ['--tests'] : []), ...(imports ? ['--imports'] : [])]; + const childEnv = { ...env }; + delete childEnv.NODE_TEST_CONTEXT; + return new Promise((resolveP) => { + exec(process.execPath, argv, { env: childEnv, timeout: timeoutMs, maxBuffer: 256 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { + if (err) { + const code = typeof err.code === 'number' ? err.code : null; + const why = String(stderr || err.message).trim().replace(/^dfd: /, '').split('\n')[0]; + return resolveP({ status: code === EXIT.INPUT ? 404 : 500, error: `the static pass over the repository failed${code === null ? ` (${err.signal || err.code})` : ''}: ${why}` }); + } + try { resolveP({ model: JSON.parse(stdout) }); } catch (e) { resolveP({ status: 500, error: `the static pass returned no readable model: ${e.message}` }); } + }); + }); +} + +/** The request's query -> the command's argv, its source fixed by the environment. */ +export function argvFor(url, env = process.env, { drawing = true } = {}) { + const q = new URL(url, 'http://panel.invalid').searchParams; + const argv = env.CW_DFD_FILE ? ['--from', env.CW_DFD_FILE] : ['--graph']; + for (const k of [...FILTER_PARAMS, ...(drawing ? DRAWING_PARAMS : [])]) if (q.has(k)) argv.push(`--${k}`, q.get(k)); + for (const k of BOOL_PARAMS) if (q.get(k) === '1' || q.get(k) === 'true') argv.push(`--${k}`); + return argv; +} + +/** + * -> { status, opts, model, collapse, drawn } or { status, error }. `drawing: false` ignores the + * drawing keys. With `repo=` in the query the answer is a Promise (the pass runs in a child); + * without it, as before, synchronous. `repos` and `pass` are the registry and the child, for tests. + */ +export function load(url, env = process.env, { drawing = true, repos = resolvedRepos, pass = passInChild } = {}) { + // a value flag always takes the next argument as its value, so a query value cannot become a flag + const { opts, error } = parseArgs(argvFor(url, env, { drawing })); + if (error) return { status: 400, error }; + const q = new URL(url, 'http://panel.invalid').searchParams; + if (q.has('repo')) { + const reg = registeredRepo(q.get('repo'), repos); + if (reg.error) return { status: 400, error: reg.error }; + return pass(reg.path, { env, tests: !!opts.tests, imports: !!opts.imports }).then((built) => { + if (built.error) return { status: built.status, error: built.error }; + const got = prepare(opts, { env, source: built.model }); + if (got.error) return { status: STATUS[got.code], error: got.error }; + return { status: 200, opts, ...got }; + }); + } + const got = prepare(opts, { env }); + if (got.error) return { status: STATUS[got.code], error: got.error }; + return { status: 200, opts, ...got }; +} + +const errorPage = (status, msg) => `${MARK_ICON}data flow diagram unavailable` + + `

data flow diagram unavailable

${esc(msg)}

Status ${status}.

`; + +/** Answer now, or when the pass does: `then` only when `load` returned a Promise. */ +const settle = (got, answer) => (typeof got?.then === 'function' ? got.then(answer) : answer(got)); + +function page(ctx) { + return settle(load(ctx.req.url), (got) => answerPage(ctx, got)); +} + +function answerPage(ctx, got) { + const max = got.opts?.max ?? DEFAULT_MAX; + if (got.status === 200 && got.drawn > max) { + got.status = 422; + got.error = `${got.drawn} elements is more than max ${max}: narrow it with prefix or focus, collapse groups, or raise max`; + } + if (got.status !== 200) return ctx.send(got.status, errorPage(got.status, got.error), 'text/html; charset=utf-8', { 'cache-control': 'no-store' }); + const { html, csp } = renderHtml(got.model, { fonts: 'served', collapse: got.collapse, orientation: got.opts.orientation || 'auto' }); + return ctx.send(200, html, 'text/html; charset=utf-8', { + 'content-security-policy': csp, + 'content-security-policy-report-only': csp, + 'cache-control': 'no-store', + 'x-robots-tag': 'noindex, nofollow', + }); +} + +function api(ctx) { + return settle(load(ctx.req.url, process.env, { drawing: false }), (got) => answerApi(ctx, got)); +} + +function answerApi(ctx, got) { + if (got.status !== 200) return ctx.send(got.status, { ok: false, error: got.error }); + return ctx.send(200, got.model, 'application/json', { 'cache-control': 'no-store' }); +} + +export const routes = [ + { method: 'GET', path: '/dfd/', handle: page }, + { method: 'GET', path: '/dfd', handle: page }, + { method: 'GET', path: '/api/dfd', handle: api }, +]; diff --git a/admin/serve.mjs b/admin/serve.mjs index 538c1a91..00984e66 100644 --- a/admin/serve.mjs +++ b/admin/serve.mjs @@ -93,6 +93,7 @@ import { routeFlagOff, offBody } from '../lib/feature-flags.mjs'; import { routes as overwatchLayerRoutes } from './routes/overwatch-layer.mjs'; import { routes as docsiteRoutes, docsiteHandle, docsiteOrigins, docsiteHosts } from './routes/docsite.mjs'; import { routes as launchlistRoutes, launchlistHosts } from './routes/launchlist.mjs'; +import { routes as dfdRoutes } from './routes/dfd.mjs'; import { routes as offboxRoutes, offboxHosts, handleIngest as offboxIngest } from './routes/offbox.mjs'; import { routes as jobRoutes, initJobRoutes } from './routes/jobs.mjs'; import { routes as panelProcessRoutes, initPanelProcessRoutes } from './routes/panel-process.mjs'; @@ -853,7 +854,7 @@ function errorPage(code, title, detail, { back = '/' } = {}) { // throw there took the process down for everyone rather than refusing one request. Three unguarded // decodeURIComponent calls have the same shape. Fail closed, per request, not per process. // Every modular route group, in dispatch order. Adding a group is one import and one entry here. -const MODULAR_ROUTES = [...secretsRoutes, ...leaksCheckRoutes, ...leaksVerifyRoutes, ...postureRoutes, ...reportRoutes, ...remediationRoutes, ...a11yRoutes, ...profileRoutes, ...configEditRoutes, ...scanConfigRoutes, ...ingestRoutes, ...issueDetailRoutes, ...craRoutes, ...determinationRoutes, ...annotationRoutes, ...oversightRoutes, ...correlationRoutes, ...remediationPolicyRoutes, ...llmRuntimeRoutes, ...hostRoutes, ...verdictRoutes, ...packageRoutes, ...updateRoutes, ...codeqlRemediationRoutes, ...cobolworkRemediationRoutes, ...projectsViewRoutes, ...fleetOverviewRoutes, ...settingsRoutes, ...agentSurfaceRoutes, ...perfRoutes, ...scannerRoutes, ...scanPathRoutes, ...turnsRoutes, ...commentRoutes, ...learningRoutes, ...featuresRoutes, ...overwatchLayerRoutes, ...docsiteRoutes, ...launchlistRoutes, ...offboxRoutes, ...rollupsRoutes, ...jobRoutes, ...panelProcessRoutes, ...panelStateRoutes, ...issuesRoutes, ...dailyRoutes, ...renovateRoutes, ...paletteRoutes, ...feedRoutes, ...journeyRoutes, ...graphRoutes, ...pullRequestRoutes, ...inboxRoutes, ...runnerRoutes]; +const MODULAR_ROUTES = [...secretsRoutes, ...leaksCheckRoutes, ...leaksVerifyRoutes, ...postureRoutes, ...reportRoutes, ...remediationRoutes, ...a11yRoutes, ...profileRoutes, ...configEditRoutes, ...scanConfigRoutes, ...ingestRoutes, ...issueDetailRoutes, ...craRoutes, ...determinationRoutes, ...annotationRoutes, ...oversightRoutes, ...correlationRoutes, ...remediationPolicyRoutes, ...llmRuntimeRoutes, ...hostRoutes, ...verdictRoutes, ...packageRoutes, ...updateRoutes, ...codeqlRemediationRoutes, ...cobolworkRemediationRoutes, ...projectsViewRoutes, ...fleetOverviewRoutes, ...settingsRoutes, ...agentSurfaceRoutes, ...perfRoutes, ...scannerRoutes, ...scanPathRoutes, ...turnsRoutes, ...commentRoutes, ...learningRoutes, ...featuresRoutes, ...overwatchLayerRoutes, ...docsiteRoutes, ...launchlistRoutes, ...dfdRoutes, ...offboxRoutes, ...rollupsRoutes, ...jobRoutes, ...panelProcessRoutes, ...panelStateRoutes, ...issuesRoutes, ...dailyRoutes, ...renovateRoutes, ...paletteRoutes, ...feedRoutes, ...journeyRoutes, ...graphRoutes, ...pullRequestRoutes, ...inboxRoutes, ...runnerRoutes]; // Server copy of the client's VALID_VIEWS — panel-view-paths.test.mjs pins the two sets equal. // Includes RETIRED view names still served as aliases ('leaks' -> the secrets view). The server's diff --git a/admin/test/dfd-route-repo.test.mjs b/admin/test/dfd-route-repo.test.mjs new file mode 100644 index 00000000..3770a35a --- /dev/null +++ b/admin/test/dfd-route-repo.test.mjs @@ -0,0 +1,203 @@ +// admin/routes/dfd.mjs `repo=`: a scanned repository drawn by name. The name is only a key +// into the panel's registry — a path, a traversal or an unknown name is a 400 with its reason — and +// the static pass runs in a child over the registered checkout. The repository is built per test +// (flow/test/dfd-fixture-repo.mjs); the registry is a temp projects.json named by CW_REGISTRY. +import { test, describe, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fixtureRepo } from '../../flow/test/dfd-fixture-repo.mjs'; + +const TMP = mkdtempSync(join(tmpdir(), 'cw-dfd-repo-route-')); +after(() => rmSync(TMP, { recursive: true, force: true })); +const { root: FIXTURE } = fixtureRepo(); +const NOT_GIT = mkdtempSync(join(tmpdir(), 'cw-dfd-not-git-')); + +// The registry is read by the panel's own resolver: set before the route module is imported. +writeFileSync(join(TMP, 'projects.json'), JSON.stringify({ + reportsRoot: join(TMP, 'reports'), monitorOutput: 'fixarea', defaultManifest: 'security-baseline', roots: [], + projects: [{ name: 'fixrepo', area: 'fixarea', path: FIXTURE, manifest: 'security-baseline' }], + areas: [{ slug: 'fixarea', label: 'fixarea', out: 'fixarea', primary: true, members: ['fixrepo'] }], +})); +process.env.CW_REGISTRY = join(TMP, 'projects.json'); +const { routes, load, registeredRepo, passInChild, REPO_NAME_RE } = await import('../routes/dfd.mjs'); + +const handler = (method, path) => routes.find((r) => r.method === method && r.path === path).handle; +async function call(path, url) { + let out = null; + await handler('GET', path)({ req: { url }, send: (status, body, ct, headers) => { out = { status, body, ct, headers }; } }); + assert.ok(out, 'the handler answered nothing'); + return out; +} +const known = () => [{ name: 'fixrepo', path: FIXTURE }, { name: 'nogit', path: NOT_GIT }, { name: 'nopath' }]; + +describe('dfd repo= — only a registered name', () => { + test('a path, a traversal, an empty or overlong value is refused before any lookup', () => { + // arrange + const attempts = ['../../etc', '/etc/passwd', '..', '.', 'a/b', 'a\\b', '', 'x'.repeat(129), 'fix repo', 'fixrepo\0']; + let looked = 0; + const repos = () => { looked++; return known(); }; + + // act + const got = attempts.map((a) => registeredRepo(a, repos)); + + // assert + for (const g of got) assert.match(g.error, /^repo must name a repository the panel knows \(letters, digits, \. _ -\), not a path: /); + assert.equal(looked, 0); + assert.equal(REPO_NAME_RE.test('fixrepo'), true); + }); + + test('a name the registry does not know, or knows without a path, is refused with the count it does know', () => { + // arrange + const names = ['other', 'nopath']; + + // act + const got = names.map((n) => registeredRepo(n, known)); + const one = registeredRepo('other', () => [{ name: 'fixrepo', path: FIXTURE }]); + const none = registeredRepo('other', () => null); + + // assert + assert.equal(got[0].error, "unknown repository other: the panel's registry names 3 repositories, and not this one"); + assert.match(got[1].error, /^unknown repository nopath/); + assert.match(one.error, /names 1 repository, and not this one/); + assert.match(none.error, /names 0 repositories/); + }); + + test('a registered name resolves to its registered path, and only that', () => { + // arrange + const name = 'fixrepo'; + + // act + const got = registeredRepo(name, known); + + // assert + assert.deepEqual(got, { path: FIXTURE }); + }); +}); + +describe('dfd repo= — load', () => { + test('the pass is handed the registered path and the query\'s tests/imports; its model is filtered here', async () => { + // arrange + let seen = null; + const model = { v: 1, kind: 'dfd', title: 't', source: { producer: 'flow-graph' }, + nodes: [{ id: 'a', kind: 'process', label: 'src/a.ts' }, { id: 'b', kind: 'store', label: 'out/b' }, { id: 'c', kind: 'store', label: 'out/c' }], + flows: [{ from: 'a', to: 'b', label: 'w', mode: 'sync' }] }; + const pass = async (path, o) => { seen = { path, ...o }; return { model }; }; + + // act + const got = await load('/api/dfd?repo=fixrepo&focus=out%2Fb&hops=0&tests=1', {}, { drawing: false, repos: known, pass }); + + // assert + assert.equal(got.status, 200); + assert.deepEqual([seen.path, seen.tests, seen.imports], [FIXTURE, true, false]); + assert.deepEqual(got.model.nodes.map((n) => n.id), ['b']); + }); + + test('a refused name is a synchronous 400; a failed pass keeps its status; a filter that matches nothing is 422', async () => { + // arrange + const failing = async () => ({ status: 500, error: 'boom' }); + const model = { v: 1, kind: 'dfd', nodes: [{ id: 'a', kind: 'process', label: 'a' }], flows: [] }; + + // act + const bad = load('/dfd/?repo=..%2F..%2Fetc', {}, { repos: known, pass: failing }); + const failed = await load('/dfd/?repo=fixrepo', {}, { repos: known, pass: failing }); + const empty = await load('/dfd/?repo=fixrepo&focus=zzz', {}, { repos: known, pass: async () => ({ model }) }); + const usage = load('/dfd/?repo=fixrepo&hops=x', {}, { repos: known, pass: failing }); + + // assert + assert.deepEqual([bad.status, failed.status, empty.status, usage.status], [400, 500, 422, 400]); + assert.match(bad.error, /not a path: "\.\.\/\.\.\/etc"/); + assert.equal(failed.error, 'boom'); + }); +}); + +describe('dfd repo= — the child', () => { + test('passInChild runs the static pass over a checkout and returns its unfiltered model', async () => { + // arrange + const path = FIXTURE; + + // act + const got = await passInChild(path, { env: process.env, imports: true, tests: true }); + + // assert + assert.equal(got.error, undefined); + assert.equal(got.model.source.producer, 'flow-graph'); + assert.equal(got.model.source.unknown.unanalysable, 3); + assert.ok(got.model.flows.some((f) => f.label === 'imports')); + }); + + test('passInChild: a directory that is not a checkout is 404, a pass that times out 500 with the signal', async () => { + // arrange + const env = process.env; + + // act + const notGit = await passInChild(NOT_GIT, { env }); + const slow = await passInChild(FIXTURE, { env, timeoutMs: 1 }); + + // assert + assert.equal(notGit.status, 404); + assert.match(notGit.error, /^the static pass over the repository failed: --repo .* is not a git checkout/); + assert.equal(slow.status, 500); + assert.match(slow.error, /failed \(SIGTERM\)/); + }); + + test('passInChild: a child that fails before the pass, or exits 0 with no model, is a 500, never an empty diagram', async () => { + // arrange + const env = { ...process.env, NODE_OPTIONS: '--require=/nonexistent-preload.cjs' }; + const silent = (file, argv, opts, cb) => cb(null, 'not json', ''); + + // act + const crashed = await passInChild(FIXTURE, { env }); + const garbled = await passInChild(FIXTURE, { exec: silent }); + + // assert + assert.equal(crashed.status, 500); + assert.match(crashed.error, /^the static pass over the repository failed: /); + assert.equal(garbled.status, 500); + assert.match(garbled.error, /^the static pass returned no readable model: /); + }); +}); + +describe('dfd repo= — the routes, through the panel\'s registry', () => { + test('the page draws a registered repository, its banner saying what was not read', async () => { + // arrange + const url = '/dfd/?repo=fixrepo'; + + // act + const r = await call('/dfd/', url); + + // assert + assert.equal(r.status, 200, r.body.slice(0, 400)); + assert.match(r.ct, /^text\/html/); + assert.match(r.body, /Unknown, not clean:<\/strong> 3 of 8 source file\(s\) not analysed/); + assert.ok(r.headers['content-security-policy']); + }); + + test('the document of a registered repository is its dfd@1 model with source.unknown', async () => { + // arrange + const url = '/api/dfd?repo=fixrepo'; + + // act + const r = await call('/api/dfd', url); + + // assert + assert.equal(r.status, 200); + assert.equal(r.body.kind, 'dfd'); + assert.deepEqual(r.body.source.unknown.unsupportedLanguages, { python: 1 }); + }); + + test('a traversal and an unknown name are 400s, as a page and as JSON', async () => { + // arrange + const urls = [['/dfd/', '/dfd/?repo=..%2F..%2Fetc%2Fpasswd'], ['/api/dfd', '/api/dfd?repo=%2Fetc'], ['/api/dfd', '/api/dfd?repo=nosuchrepo']]; + + // act + const got = await Promise.all(urls.map(([p, u]) => call(p, u))); + + // assert + assert.deepEqual(got.map((r) => r.status), [400, 400, 400]); + assert.match(got[0].body, /not a path: "\.\.\/\.\.\/etc\/passwd"/); + assert.deepEqual(got[1].body, { ok: false, error: 'repo must name a repository the panel knows (letters, digits, . _ -), not a path: "/etc"' }); + assert.match(got[2].body.error, /^unknown repository nosuchrepo: the panel's registry names \d+ repositor/); + }); +}); diff --git a/admin/test/dfd-route.test.mjs b/admin/test/dfd-route.test.mjs new file mode 100644 index 00000000..ee1e7d83 --- /dev/null +++ b/admin/test/dfd-route.test.mjs @@ -0,0 +1,135 @@ +// admin/routes/dfd.mjs over a really-spawned panel: the page and the document sit behind the login +// gate, and a signed-in request gets the page under its own hash-pinned policy. CW_AUTH_STORE / +// CW_REGISTRY / CW_DFD_FILE point at temp files and the synthetic fixture. + +import { test, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { createServer } from 'node:net'; +import { request } from 'node:http'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SERVE = join(HERE, '..', 'serve.mjs'); +const FIXTURE = join(HERE, '..', '..', 'lib', 'test', 'fixtures', 'dfd-synthetic.json'); +const TMP = mkdtempSync(join(tmpdir(), 'cw-dfd-route-')); + +let port, localPort, child, csrf, cookie = ''; + +const freePort = () => new Promise((res, rej) => { + const s = createServer(); + s.on('error', rej); + s.listen(0, '127.0.0.1', () => { const p = s.address().port; s.close(() => res(p)); }); +}); + +function hit(path, { method = 'GET', headers = {}, body = null, operator = true, auth = false } = {}) { + return new Promise((resolve, reject) => { + const h = { host: 'localhost', ...headers }; + if (auth && cookie) h.cookie = cookie; + if (body != null) { h['content-type'] = 'application/json'; h['content-length'] = Buffer.byteLength(body); } + const req = request({ host: '127.0.0.1', port: operator ? localPort : port, path, method, headers: h }, (res) => { + let buf = ''; + res.setEncoding('utf8'); + res.on('data', (d) => { buf += d; }); + res.on('end', () => { let json = null; try { json = JSON.parse(buf); } catch { /* html */ } resolve({ status: res.statusCode, headers: res.headers, body: buf, json }); }); + }); + req.on('error', reject); + if (body != null) req.write(body); + req.end(); + }); +} +const post = (path, payload, opts = {}) => + hit(path, { ...opts, method: 'POST', body: JSON.stringify(payload), headers: { 'x-cw-csrf': csrf, ...(opts.headers || {}) } }); + +before(async () => { + writeFileSync(join(TMP, 'projects.json'), JSON.stringify({ + reportsRoot: join(TMP, 'reports'), monitorOutput: 'fixarea', defaultManifest: 'security-baseline', roots: [], + projects: [{ name: 'fixrepo', area: 'fixarea', path: join(TMP, 'src'), manifest: 'security-baseline' }], + areas: [{ slug: 'fixarea', label: 'fixarea', out: 'fixarea', primary: true, members: ['fixrepo'] }], + })); + mkdirSync(join(TMP, 'src'), { recursive: true }); + + let lastErr = ''; + for (let attempt = 1; attempt <= 3; attempt++) { + port = await freePort(); + localPort = await freePort(); + let err = ''; + child = spawn(process.execPath, [SERVE], { + env: { + ...process.env, + CW_AUTH_STORE: join(TMP, 'users.json'), CW_REGISTRY: join(TMP, 'projects.json'), CW_DFD_FILE: FIXTURE, + // the panel stamps its code at boot; under plain `node --test` nothing redirects it out of .claude/store + CW_PANEL_CODE_STAMP: join(TMP, 'panel-code-stamp.json'), + CW_ADMIN_PORT: String(port), CW_ADMIN_LOCAL_PORT: String(localPort), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stderr.on('data', (d) => { err += String(d); }); + for (let i = 0; i < 100 && !csrf; i++) { + try { const r = await hit('/api/csrf'); if (r.status === 200) { csrf = r.json.token; break; } } catch { /* not up yet */ } + await new Promise((r) => setTimeout(r, 100)); + } + if (csrf) break; + child.kill('SIGKILL'); + lastErr = err; + } + assert.ok(csrf, `panel did not come up${lastErr ? ` — last child stderr:\n${lastErr}` : ''}`); + // a user must exist first: with none, the zero-user posture refuses every remote caller before the gate + const boot = await post('/auth/bootstrap', { email: 'op@example.com', password: 'correct horse battery' }); + assert.equal(boot.status, 200, `bootstrap failed: ${boot.body}`); + const login = await post('/auth/login', { email: 'op@example.com', password: 'correct horse battery' }); + assert.equal(login.status, 200, `login failed: ${login.body}`); + cookie = String(login.headers['set-cookie'] || '').split(';')[0]; + assert.ok(cookie, 'login set no cookie'); +}); + +after(() => { child?.kill('SIGKILL'); rmSync(TMP, { recursive: true, force: true }); }); + +describe('dfd route — gated', () => { + test('without a session the published port answers neither the page nor the document', async () => { + // arrange + const paths = ['/dfd/', '/api/dfd']; + + // act + const rs = await Promise.all(paths.map((p) => hit(p, { operator: false }))); + + // assert + for (const r of rs) { + assert.notEqual(r.status, 200); + assert.doesNotMatch(r.body, / { + test('GET /dfd/ serves the diagram under the page\'s own hash-pinned CSP', async () => { + // arrange + const path = '/dfd/?orientation=TB'; + + // act + const r = await hit(path, { operator: false, auth: true }); + + // assert + assert.equal(r.status, 200, r.body.slice(0, 200)); + assert.match(r.body, / { + // arrange + const path = '/api/dfd?prefix=val&hops=0'; + + // act + const r = await hit(path, { operator: false, auth: true }); + + // assert + assert.equal(r.status, 200, r.body.slice(0, 200)); + assert.equal(r.json.kind, 'dfd'); + assert.deepEqual(r.json.nodes.map((nd) => nd.id), ['validate']); + }); +}); diff --git a/admin/test/dfd-routes-entry.test.mjs b/admin/test/dfd-routes-entry.test.mjs new file mode 100644 index 00000000..5c83fb4e --- /dev/null +++ b/admin/test/dfd-routes-entry.test.mjs @@ -0,0 +1,175 @@ +// admin/routes/dfd.mjs through its handlers: the source the environment names, the query as the +// command's flags, and the status each refusal becomes. The login gate is tested over a spawned +// panel in dfd-route.test.mjs. +import { test, describe, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { routes, argvFor, load } from '../routes/dfd.mjs'; +import { makeNode, makeEdge, nodeId } from '../../flow/graph.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FIXTURE = join(HERE, '..', '..', 'lib', 'test', 'fixtures', 'dfd-synthetic.json'); +const TMP = mkdtempSync(join(tmpdir(), 'cw-dfd-entry-')); +after(() => rmSync(TMP, { recursive: true, force: true })); + +const handler = (method, path) => routes.find((r) => r.method === method && r.path === path).handle; + +function call(path, url, env) { + const saved = { ...process.env }; + for (const k of ['CW_DFD_FILE', 'CW_HARNESS_STORE']) delete process.env[k]; + Object.assign(process.env, env); + let out = null; + try { + handler('GET', path)({ req: { url }, send: (status, body, ct, headers) => { out = { status, body, ct, headers }; } }); + } finally { + for (const k of ['CW_DFD_FILE', 'CW_HARNESS_STORE']) delete process.env[k]; + Object.assign(process.env, { ...saved }); + } + assert.ok(out, 'the handler answered nothing'); + return out; +} + +describe('dfd routes — declaration', () => { + test('the page and the document are declared once each, the page with and without its slash', () => { + // arrange + const want = ['GET /dfd/', 'GET /dfd', 'GET /api/dfd']; + + // act + const got = routes.map((r) => `${r.method} ${r.path}`); + + // assert + assert.deepEqual(got, want); + assert.equal(handler('GET', '/dfd'), handler('GET', '/dfd/')); + }); +}); + +describe('dfd routes — the query is the command line', () => { + test('a dfd file named by CW_DFD_FILE is the source, and each known query key becomes its flag', () => { + // arrange + const url = '/dfd/?prefix=api&focus=x&hops=2&direction=out&collapse=g,h&collapse-all-groups-over=3&orientation=TB&max=9&tests=1&imports=true&title=nope&from=/etc/passwd'; + + // act + const argv = argvFor(url, { CW_DFD_FILE: '/data/model.json' }); + + // assert + assert.deepEqual(argv, ['--from', '/data/model.json', '--prefix', 'api', '--focus', 'x', '--hops', '2', '--direction', 'out', + '--collapse', 'g,h', '--collapse-all-groups-over', '3', '--orientation', 'TB', '--max', '9', '--tests', '--imports']); + }); + + test('without the drawing keys only the filter reaches the command', () => { + // arrange + const url = '/api/dfd?prefix=api&collapse=nope&orientation=XX&max=1'; + + // act + const argv = argvFor(url, {}, { drawing: false }); + + // assert + assert.deepEqual(argv, ['--graph', '--prefix', 'api']); + }); + + test('with no dfd file the source is the flow store, and a flag-shaped value stays a value', () => { + // arrange + const url = '/api/dfd?prefix=--graph&tests=0'; + + // act + const argv = argvFor(url, {}); + const got = load(url, { CW_DFD_FILE: FIXTURE }); + + // assert + assert.deepEqual(argv, ['--graph', '--prefix', '--graph']); + assert.equal(got.status, 422); + assert.match(got.error, /the filter matched no node \(prefix "--graph"/); + }); +}); + +describe('GET /dfd/ — the page', () => { + test('serves the diagram under the page\'s own policy, with served fonts', () => { + // arrange + const env = { CW_DFD_FILE: FIXTURE }; + + // act + const r = call('/dfd/', '/dfd/?orientation=LR&collapse=platform', env); + + // assert + assert.equal(r.status, 200); + assert.equal(r.ct, 'text/html; charset=utf-8'); + assert.match(r.body, / { + // arrange + const store = join(TMP, 'store'); + mkdirSync(store, { recursive: true }); + writeFileSync(join(store, 'flow.json'), JSON.stringify({ v: 1, generatedAt: '2026-11-12T00:00:00.000Z', + nodes: [makeNode('module', 'bin/a.mjs', { analysis: 'analysed' }), makeNode('store', 'reports/x.json')], + edges: [makeEdge(nodeId('module', 'bin/a.mjs'), nodeId('store', 'reports/x.json'), 'writes', { evidence: ['bin/a.mjs:1 writeFileSync'] })] })); + + // act + const r = call('/dfd/', '/dfd/', { CW_HARNESS_STORE: store }); + + // assert + assert.equal(r.status, 200, r.body.slice(0, 300)); + assert.match(r.body, /Producer: flow-graph<\/code>/); + }); + + test('each refusal is a status and an escaped page, never a diagram', () => { + // arrange + const bad = join(TMP, 'bad.json'); + writeFileSync(bad, JSON.stringify({ v: 1, kind: 'dfd', nodes: [], flows: [{ from: 'a', to: 'b', mode: 'sync' }] })); + + // act + const usage = call('/dfd/', '/dfd/?direction=sideways&prefix=a', { CW_DFD_FILE: FIXTURE }); + const absent = call('/dfd/', '/dfd/', { CW_HARNESS_STORE: join(TMP, 'empty') }); + const invalid = call('/dfd/', '/dfd/', { CW_DFD_FILE: bad }); + const nomatch = call('/dfd/', '/dfd/?prefix= + +
+

${esc(title)}

+

Data flow diagram, Gane–Sarson notation · dfd@${esc(model.v)} · ${model.nodes.length} elements, ${model.flows.length} flows · ranks ${lay.orientation === 'TB' ? 'top to bottom' : 'left to right'}

+${sourceBanner(model)} +
${renderSvg(model, lay)}
+${LEGEND} +

What the analysis found

+
+${facts} +
+

Elements

+ +${nodeRows} +
refkindlabelboundarylayerhop rankroles
+

Flows

+ +${flowRows} +
#fromtomodeclasslabelevidence
+
+`; + const stats = { orientation: lay.orientation, width: lay.width, height: lay.height, crossings: lay.crossings, collisions: lay.collisions, elements: model.nodes.length, flows: model.flows.length }; + return { html, csp: policy(["frame-ancestors 'self'"]), stats }; +} diff --git a/lib/dfd.mjs b/lib/dfd.mjs new file mode 100644 index 00000000..85fedc6d --- /dev/null +++ b/lib/dfd.mjs @@ -0,0 +1,375 @@ +// lib/dfd.mjs — dfd@1: a source-independent data flow diagram model, its validation and its +// analysis. Rendering is lib/dfd-render.mjs. The adapters that build a model from the flow graph or +// from CodeQL code-flow records are flow/dfd-adapter.mjs: lib/ ships and flow/ does not, so nothing +// here imports from flow/. +// +// Three element kinds and no fourth. A message broker or queue is a STORE with subtype +// broker|queue|topic|stream: data rests in it between a producer and a consumer, the classic +// notations have no fourth symbol, and asynchrony is carried by the flow's `mode`, so a kind of its +// own would say the same thing twice and let the two disagree. +// +// Everything here is a pure function over the model. Nothing reads the clock, the environment or +// the disk except validateDfd, which opens schema/dfd.schema.json. +// +// What analysis never does: an undirected flow (`directed: false`, the flow graph's `touches`) takes +// no part in ranks, cycles, feedback or paths. Its direction was never determined, and a cycle +// built on a guessed direction is a fabricated feedback loop. + +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { validateAgainstSchema } from './json-schema.mjs'; + +export const DFD_VERSION = 1; +export const NODE_KINDS = Object.freeze(['process', 'store', 'external']); +export const MODES = Object.freeze(['sync', 'async', 'event']); +// `log` is a broker: an append-only file one process appends to and another tails holds data +// between them exactly as a topic does, and the reader is decoupled from the writer in time. +export const BROKER_SUBTYPES = Object.freeze(['broker', 'queue', 'topic', 'stream', 'log']); +export const STORE_SUBTYPES = Object.freeze([...BROKER_SUBTYPES, 'database', 'file']); +export const EXTERNAL_SUBTYPES = Object.freeze(['endpoint', 'scheduler', 'actor', 'system', 'env', 'binary']); +// A flow's role says what it carries back. Only `data` takes part in the two-way collapse: a +// feedback, error or retry flow is a return path even when its pair is adjacent. +export const ROLES = Object.freeze(['data', 'feedback', 'error', 'retry']); +// flow.direction: what the runtime saw a module do across a static `touches`, which never had one +export const FLOW_DIRECTIONS = Object.freeze(['reads', 'writes', 'both']); +// filterModel's walk: forward along flows, backward against them, or both +export const FILTER_DIRECTIONS = Object.freeze(['in', 'out', 'both']); +export const FLOW_CLASSES = Object.freeze(['forward', 'bidirectional', 'feedback', 'self-loop', 'undirected']); + +export const schemaPath = () => join(dirname(fileURLToPath(import.meta.url)), '..', 'schema', 'dfd.schema.json'); + +export const isBroker = (node) => !!node && node.kind === 'store' && BROKER_SUBTYPES.includes(node.subtype); +export const isDirected = (flow) => flow.directed !== false; +export const roleOf = (flow) => flow.role || 'data'; +const SUBTYPES_BY_KIND = { store: STORE_SUBTYPES, external: EXTERNAL_SUBTYPES }; + +/** -> { ok, errors }. Schema first; the referential checks run only on a document of the right shape. */ +export function validateDfd(doc, { path = schemaPath() } = {}) { + const errors = validateAgainstSchema(doc, { path }).errors.slice(); + if (errors.length) return { ok: false, errors }; + if (Object.keys(doc)[0] !== 'v') errors.push('dfd: "v" must be the first field of the document — an old-shape record is a silently different meaning, not a parse failure'); + const ids = new Set(); + for (const n of doc.nodes) { + if (ids.has(n.id)) errors.push(`dfd: node id ${JSON.stringify(n.id)} is declared more than once`); + ids.add(n.id); + } + const layers = doc.layers ? new Set(doc.layers) : null; + const groups = doc.groups ? new Set(doc.groups.map((g) => g.id)) : null; + if (groups && groups.size !== doc.groups.length) errors.push('dfd: a group id is declared more than once'); + for (const n of doc.nodes) { + if (layers && n.layer !== undefined && !layers.has(n.layer)) errors.push(`dfd: node ${JSON.stringify(n.id)} names layer ${JSON.stringify(n.layer)}, which "layers" does not declare`); + if (groups && n.group !== undefined && !groups.has(n.group)) errors.push(`dfd: node ${JSON.stringify(n.id)} names group ${JSON.stringify(n.group)}, which "groups" does not declare`); + // the schema admits the union of the subtypes; which kind may carry which is checked here + if (n.subtype !== undefined) { + const allowed = SUBTYPES_BY_KIND[n.kind]; + if (!allowed) errors.push(`dfd: node ${JSON.stringify(n.id)} is a process and carries subtype ${JSON.stringify(n.subtype)}: only a store or an external has one`); + else if (!allowed.includes(n.subtype)) errors.push(`dfd: node ${JSON.stringify(n.id)} is a ${n.kind} and carries subtype ${JSON.stringify(n.subtype)}, which is a subtype of the other kind (${n.kind} takes ${allowed.join('|')})`); + } + } + doc.flows.forEach((f, i) => { + for (const end of ['from', 'to']) { + if (!ids.has(f[end])) errors.push(`dfd: flows[${i}].${end} names ${JSON.stringify(f[end])}, which is not a node`); + } + // a resolved direction on a flow that still says it has none is two answers to one question + if (f.direction !== undefined && !isDirected(f)) errors.push(`dfd: flows[${i}] carries direction ${JSON.stringify(f.direction)} and directed:false — a resolved touch is directed`); + }); + return { ok: errors.length === 0, errors }; +} + +// ── graph primitives (index-based, iterative: the repository's own graph has 11,000 nodes) ──────── + +/** Tarjan. -> { comp, count }: comp[i] is the component of node i, numbered by first member. */ +export function stronglyConnected(out) { + const n = out.length; + const index = new Array(n).fill(-1); + const low = new Array(n).fill(0); + const onStack = new Array(n).fill(false); + const raw = new Array(n).fill(-1); + const stack = []; + let next = 0; + let count = 0; + const open = (v, work) => { index[v] = low[v] = next++; stack.push(v); onStack[v] = true; work.push([v, 0]); }; + for (let s = 0; s < n; s++) { + if (index[s] !== -1) continue; + const work = []; + open(s, work); + while (work.length) { + const top = work[work.length - 1]; + const v = top[0]; + if (top[1] < out[v].length) { + const w = out[v][top[1]++][0]; + if (index[w] === -1) open(w, work); + else if (onStack[w]) low[v] = Math.min(low[v], index[w]); + continue; + } + work.pop(); + if (work.length) { const p = work[work.length - 1][0]; low[p] = Math.min(low[p], low[v]); } + if (low[v] !== index[v]) continue; + let w; + do { w = stack.pop(); onStack[w] = false; raw[w] = count; } while (w !== v); + count++; + } + } + const renumber = new Map(); + const comp = raw.map((c) => { if (!renumber.has(c)) renumber.set(c, renumber.size); return renumber.get(c); }); + return { comp, count }; +} + +/** + * The arcs that close a cycle, chosen deterministically: inside each component, a depth-first walk + * from its ENTRY (the first member something outside the component flows into, else its first + * member) marks every arc that returns to a node still being walked. An arc between components is + * never one. -> Set of arc keys. + */ +export function backArcs(out, comp, count) { + const members = Array.from({ length: count }, () => []); + comp.forEach((c, i) => members[c].push(i)); + const entered = new Array(out.length).fill(false); + out.forEach((arcs, v) => { for (const [w] of arcs) if (comp[w] !== comp[v]) entered[w] = true; }); + const back = new Set(); + const state = new Array(out.length).fill(0); + for (const ms of members) { + if (ms.length < 2) continue; + const root = ms.find((m) => entered[m]) ?? ms[0]; + const work = [[root, 0]]; + state[root] = 1; + while (work.length) { + const top = work[work.length - 1]; + const v = top[0]; + if (top[1] >= out[v].length) { state[v] = 2; work.pop(); continue; } + const [w, key] = out[v][top[1]++]; + if (comp[w] !== comp[v]) continue; + if (state[w] === 1) back.add(key); + else if (state[w] === 0) { state[w] = 1; work.push([w, 0]); } + } + } + return back; +} + +/** Longest-path layering of an acyclic arc list. -> { rank, order }. Throws on a cycle. */ +export function longestPathRanks(n, arcs) { + const out = Array.from({ length: n }, () => []); + const indeg = new Array(n).fill(0); + for (const [a, b] of arcs) { out[a].push(b); indeg[b]++; } + const rank = new Array(n).fill(0); + const order = []; + for (let i = 0; i < n; i++) if (indeg[i] === 0) order.push(i); + for (let q = 0; q < order.length; q++) { + const a = order[q]; + for (const b of out[a]) { + if (rank[b] < rank[a] + 1) rank[b] = rank[a] + 1; + if (--indeg[b] === 0) order.push(b); + } + } + if (order.length !== n) throw new Error('dfd: longestPathRanks was given a cyclic arc list'); + return { rank, order }; +} + +// ── analysis ───────────────────────────────────────────────────────────────────────────────────── + +/** + * Pure analysis of a VALID dfd@1 model. Flow references are indexes into model.flows. + * + * Classification, in priority order — every flow takes exactly one class: + * undirected directed:false; excluded from everything below + * self-loop from === to (a retry on the same process) + * feedback role feedback|error|retry: a declared return path, even when its pair is + * adjacent — A->B data with B->A error is a loop, not a synchronised pair + * bidirectional role data both ways: A->B and B->A; collapses to one two-headed flow + * feedback a back arc inside a strongly connected component of the data flows (backArcs) + * forward everything else + * + * Ranks and paths run over the data flows less their back arcs; a declared return path is set + * aside as a back arc is. Cycles are the components of every directed flow, declared returns + * included, so a loop closed only by an error flow is still reported as a cycle. + */ +export function analyse(model, { maxPaths = 200 } = {}) { + const ids = model.nodes.map((n) => n.id); + const ix = new Map(ids.map((id, i) => [id, i])); + const n = ids.length; + const flows = model.flows; + const flowClass = flows.map(() => 'forward'); + const arcs = []; + const data = []; + flows.forEach((f, k) => { + if (!isDirected(f)) flowClass[k] = 'undirected'; + else if (f.from === f.to) flowClass[k] = 'self-loop'; + else { + const arc = { a: ix.get(f.from), b: ix.get(f.to), k }; + arcs.push(arc); + if (roleOf(f) === 'data') data.push(arc); + else flowClass[k] = 'feedback'; + } + }); + + const adjacency = (list) => { + const out = Array.from({ length: n }, () => []); + for (const { a, b, k } of list) out[a].push([b, k]); + return out; + }; + const dataOut = adjacency(data); + const dataScc = stronglyConnected(dataOut); + const back = backArcs(dataOut, dataScc.comp, dataScc.count); + for (const { k } of arcs) if (flowClass[k] === 'feedback') back.add(k); + const { comp, count } = stronglyConnected(adjacency(arcs)); + + const byPair = new Map(); + for (const { a, b, k } of data) { + const key = `${a}>${b}`; + if (!byPair.has(key)) byPair.set(key, []); + byPair.get(key).push(k); + } + const bidirectional = []; + for (const [key, ks] of byPair) { + const [a, b] = key.split('>').map(Number); + const reverse = byPair.get(`${b}>${a}`); + if (!reverse || a > b) continue; + const all = [...ks, ...reverse].sort((x, y) => x - y); + for (const k of all) flowClass[k] = 'bidirectional'; + bidirectional.push({ a: ids[a], b: ids[b], flows: all, modes: [...new Set(all.map((k) => flows[k].mode))] }); + } + for (const k of back) if (flowClass[k] === 'forward') flowClass[k] = 'feedback'; + + const dag = arcs.filter(({ k }) => !back.has(k)); + const { rank, order } = longestPathRanks(n, dag.map(({ a, b }) => [a, b])); + const condensed = new Set(arcs.filter(({ a, b }) => comp[a] !== comp[b]).map(({ a, b }) => `${comp[a]}>${comp[b]}`)); + const sccRank = longestPathRanks(count, [...condensed].map((s) => s.split('>').map(Number))).rank; + + const succ = Array.from({ length: n }, () => new Set()); + const pred = Array.from({ length: n }, () => new Set()); + for (const { a, b } of arcs) { succ[a].add(b); pred[b].add(a); } + const dagSucc = Array.from({ length: n }, () => new Set()); + const dagIn = new Array(n).fill(0); + for (const { a, b } of dag) if (!dagSucc[a].has(b)) { dagSucc[a].add(b); dagIn[b]++; } + + const sizes = new Array(count).fill(0); + comp.forEach((c) => { sizes[c]++; }); + const cycles = []; + for (let c = 0; c < count; c++) { + if (sizes[c] < 2) continue; + const inside = arcs.filter(({ a, b }) => comp[a] === c && comp[b] === c).map(({ k }) => k); + const fb = inside.filter((k) => flowClass[k] === 'feedback'); + cycles.push({ + members: ids.filter((_, i) => comp[i] === c), + feedback: fb, + // every arc in the component belongs to a two-way pair: a synchronised pair, not a loop + bidirectionalOnly: inside.every((k) => flowClass[k] === 'bidirectional'), + // a cycle through an element the producer could not vouch for may not exist + uncertain: model.nodes.some((nd, i) => comp[i] === c && nd.unknown === true), + }); + } + + const sources = []; + const sinks = []; + for (let i = 0; i < n; i++) { + if (dagIn[i] === 0 && dagSucc[i].size > 0) sources.push(i); + if (dagIn[i] > 0 && dagSucc[i].size === 0) sinks.push(i); + } + const ways = new Array(n).fill(0); + for (let q = order.length - 1; q >= 0; q--) { + const v = order[q]; + ways[v] = dagSucc[v].size === 0 ? 1 : [...dagSucc[v]].reduce((s, w) => s + ways[w], 0); + } + const total = sources.reduce((s, v) => s + ways[v], 0); + const listed = []; + for (const s of sources) { + const work = [[s, [...dagSucc[s]], 0]]; + const trail = [s]; + while (work.length && listed.length < maxPaths) { + const top = work[work.length - 1]; + if (top[2] >= top[1].length) { work.pop(); trail.pop(); continue; } + const w = top[1][top[2]++]; + trail.push(w); + if (dagSucc[w].size === 0) { listed.push(trail.map((i) => ids[i])); trail.pop(); continue; } + work.push([w, [...dagSucc[w]], 0]); + } + } + + const node = (id) => model.nodes[ix.get(id)]; + const asyncHops = []; + flows.forEach((f, k) => { + const byMode = f.mode !== 'sync'; + const byBroker = isBroker(node(f.from)) || isBroker(node(f.to)); + if (byMode || byBroker) asyncHops.push({ flow: k, from: f.from, to: f.to, reason: byMode && byBroker ? 'mode+broker' : byMode ? 'mode' : 'broker' }); + }); + const brokered = []; + model.nodes.forEach((b, bi) => { + if (!isBroker(b)) return; + for (const p of pred[bi]) for (const c of succ[bi]) brokered.push({ from: ids[p], via: b.id, to: ids[c] }); + }); + + return { + v: DFD_VERSION, + flowClass, + nodes: ids.map((id, i) => ({ id, rank: rank[i], sccRank: sccRank[comp[i]], cyclic: sizes[comp[i]] > 1, fanOut: succ[i].size, fanIn: pred[i].size })), + bidirectional, + selfLoops: flows.map((f, k) => ({ node: f.from, flow: k })).filter((_, k) => flowClass[k] === 'self-loop'), + feedback: flows.map((f, k) => ({ flow: k, from: f.from, to: f.to, role: roleOf(f) })).filter((_, k) => flowClass[k] === 'feedback'), + undirected: flowClass.map((c, k) => (c === 'undirected' ? k : -1)).filter((k) => k >= 0), + cycles, + branches: ids.map((id, i) => ({ node: id, to: [...succ[i]].map((j) => ids[j]) })).filter((b) => b.to.length > 1), + merges: ids.map((id, i) => ({ node: id, from: [...pred[i]].map((j) => ids[j]) })).filter((m) => m.from.length > 1), + sources: sources.map((i) => ids[i]), + sinks: sinks.map((i) => ids[i]), + // total is exact while it is a safe integer and null beyond: a rounded count is not a count + paths: { + total: Number.isSafeInteger(total) ? total : null, + listed, + truncated: listed.length !== total, + multiHop: listed.filter((p) => p.length > 2).length, + }, + asyncHops, + brokered, + }; +} + +// ── filter ─────────────────────────────────────────────────────────────────────────────────────── + +/** + * Keep the seeds and everything within `hops` flows of them. Seeds are the nodes whose label or id + * starts with `prefix`, plus the node whose id or label is `around` (`focus` is its older name and + * is still read). `direction` chooses the walk: 'out' follows flows forward from the seeds (what + * they feed), 'in' follows them backward (what feeds them), 'both' (the default) ignores direction. + * An undirected flow is walked in every direction: which way it runs was never determined, so + * neither walk may drop it. With no seed criterion the model is returned as it is; when nothing + * matches, null — never an empty diagram. What was cut is counted in source.filter: a filtered + * diagram that does not say so reads as the whole system. + */ +export function filterModel(model, { prefix, around, focus, hops = 1, direction = 'both' } = {}) { + const centre = around ?? focus; + if (!FILTER_DIRECTIONS.includes(direction)) throw new Error(`dfd: filterModel direction must be one of ${FILTER_DIRECTIONS.join(', ')}, got ${JSON.stringify(direction)}`); + if (prefix === undefined && centre === undefined) return model; + const seeds = new Set(); + for (const n of model.nodes) { + if (prefix !== undefined && (n.label.startsWith(prefix) || n.id.startsWith(prefix))) seeds.add(n.id); + if (centre !== undefined && (n.id === centre || n.label === centre)) seeds.add(n.id); + } + if (seeds.size === 0) return null; + const near = new Map(model.nodes.map((n) => [n.id, []])); + for (const f of model.flows) { + const both = direction === 'both' || !isDirected(f); + if (both || direction === 'out') near.get(f.from).push(f.to); + if (both || direction === 'in') near.get(f.to).push(f.from); + } + const keep = new Set(seeds); + let edge = [...seeds]; + for (let h = 0; h < hops; h++) { + const next = []; + for (const id of edge) for (const other of near.get(id)) if (!keep.has(other)) { keep.add(other); next.push(other); } + edge = next; + } + const nodes = model.nodes.filter((n) => keep.has(n.id)); + const flows = model.flows.filter((f) => keep.has(f.from) && keep.has(f.to)); + const { v, kind, title, source, ...rest } = model; + return { + v, kind, ...(title === undefined ? {} : { title }), + source: { + ...(source || { producer: 'authored' }), + filter: { prefix: prefix ?? null, focus: centre ?? null, hops, direction, seeds: seeds.size, + nodesKept: nodes.length, nodesOmitted: model.nodes.length - nodes.length, flowsOmitted: model.flows.length - flows.length }, + }, + ...rest, + nodes, + flows, + }; +} diff --git a/lib/test/dfd-render-unknown.test.mjs b/lib/test/dfd-render-unknown.test.mjs new file mode 100644 index 00000000..35605af8 --- /dev/null +++ b/lib/test/dfd-render-unknown.test.mjs @@ -0,0 +1,45 @@ +// The page banner's statement of what a producer could not read (lib/dfd-render.mjs unknownBanner): +// unknown is never clean, so each count is stated, zero included. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { unknownBanner, renderHtml } from '../dfd-render.mjs'; + +test('unknownBanner states every count and names the unread files, the rest counted', () => { + // arrange + const u = { files: 9, unanalysable: 2, byState: { unparseable: 1, 'unsupported-language': 1 }, unsupportedLanguages: { python: 3 }, + computedSites: 2, computedSitesByKind: { 'endpoint: computed call': 2 }, filesUnread: ['a.ts: unparseable', 'b.tsx: unsupported-language'], filesUnreadNotListed: 4 }; + + // act + const html = unknownBanner(u); + + // assert + assert.match(html, /^Unknown, not clean:<\/strong> 2 of 9 source file\(s\) not analysed \(unparseable 1, unsupported-language 1\)/); + assert.match(html, /files in languages this pass does not read: python 3/); + assert.match(html, /2 call site\(s\) whose path, endpoint, database or script is computed and could not be named \(endpoint: computed call 2\)/); + assert.match(html, /Not analysed: a\.ts: unparseable; b<x>\.tsx: unsupported-language; and 4 more\./); +}); + +test('unknownBanner with nothing unread still says so, and an empty block reads as zeros', () => { + // arrange + const blocks = [{ files: 3, unanalysable: 0, unsupportedLanguages: {}, computedSites: 0, filesUnread: [] }, {}]; + + // act + const got = blocks.map(unknownBanner); + + // assert + assert.match(got[0], /0 of 3 source file\(s\) not analysed; no tracked file in a language this pass does not read; 0 call site\(s\)/); + assert.doesNotMatch(got[0], /Not analysed:/); + assert.match(got[1], /0 of 0 source file\(s\)/); +}); + +test('the page banner carries the unknown block of the model\'s source', () => { + // arrange + const model = { v: 1, kind: 'dfd', title: 't', source: { producer: 'flow-graph', unknown: { files: 1, unanalysable: 1, byState: { unparseable: 1 }, filesUnread: ['x.js: unparseable'] } }, + nodes: [{ id: 'a', kind: 'process', label: 'a' }, { id: 'b', kind: 'store', label: 'b' }], flows: [{ from: 'a', to: 'b', label: 'w', mode: 'sync' }] }; + + // act + const { html } = renderHtml(model, { fonts: 'none' }); + + // assert + assert.match(html, /