From 0509af9550aa57e04ecf3231d9e186a3190cdcf2 Mon Sep 17 00:00:00 2001 From: Portll Date: Thu, 12 Nov 2026 00:26:10 +1030 Subject: [PATCH 01/36] feat(dfd): data flow diagram model, analysis, renderer and CLI dfd@1 (schema/dfd.schema.json): processes, stores (broker, queue, topic, stream subtypes) and external entities, with groups, maturity layers and sync, async or event flows. lib/dfd.mjs validates and analyses: two-way pairs, feedback edges, self-loops, hop ranks, branch and merge points, source-to-sink paths, async hops. lib/dfd-render.mjs draws Gane-Sarson notation as one self-contained page with a deterministic layered layout. flow/dfd-cli.mjs renders a dfd@1 file or the flow graph. --- flow/README.md | 102 +++++ flow/dfd-cli.mjs | 109 ++++++ flow/test/dfd-cli.test.mjs | 279 ++++++++++++++ lib/dfd-render.mjs | 499 ++++++++++++++++++++++++ lib/dfd.mjs | 428 +++++++++++++++++++++ lib/test/dfd-render.test.mjs | 504 ++++++++++++++++++++++++ lib/test/dfd.test.mjs | 554 +++++++++++++++++++++++++++ lib/test/fixtures/dfd-synthetic.json | 54 +++ schema/dfd.schema.json | 86 +++++ 9 files changed, 2615 insertions(+) create mode 100644 flow/dfd-cli.mjs create mode 100644 flow/test/dfd-cli.test.mjs create mode 100644 lib/dfd-render.mjs create mode 100644 lib/dfd.mjs create mode 100644 lib/test/dfd-render.test.mjs create mode 100644 lib/test/dfd.test.mjs create mode 100644 lib/test/fixtures/dfd-synthetic.json create mode 100644 schema/dfd.schema.json diff --git a/flow/README.md b/flow/README.md index bc480cf36..ba19ebeb1 100644 --- a/flow/README.md +++ b/flow/README.md @@ -157,6 +157,108 @@ asked a question — `monitor/rollup.mjs` republishes when run with no argument, treats an unknown first argument as a whole-fleet sweep — so `DEFAULT_COMMANDS` is a checked read-only set and widening it is a deliberate act, not a default. +## Data flow diagrams (`dfd@1`) + +`flow/` printed text. `flow/dfd*.mjs` draws: a data flow diagram of this repository's own graph, or +of any system described in a `dfd@1` file — a scanned repository's flows, a threat-model diagram. + +| file | what it is | +|---|---| +| `dfd.mjs` | the `dfd@1` model, `validateDfd`, the analysis (`analyse`), the adapter (`fromFlowGraph`), the filter (`filterModel`) — pure functions | +| `dfd-render.mjs` | deterministic layered layout and one self-contained HTML page with an inline SVG | +| `dfd-cli.mjs` | the command | +| `../schema/dfd.schema.json` | the document's shape; ids, flow ends, declared layers and groups are checked in `validateDfd` | + +```sh +node flow/dfd-cli.mjs --from lib/test/fixtures/dfd-synthetic.json --out /tmp/dfd.html +node flow/static.mjs # writes /flow.json +node flow/dfd-cli.mjs --graph --prefix flow/ --out /tmp/flow-dfd.html +node flow/dfd-cli.mjs --graph --focus monitor/rollup.mjs --hops 1 --emit model # the dfd@1 document +node --test --experimental-test-coverage flow/test/dfd.test.mjs flow/test/dfd-render.test.mjs flow/test/dfd-cli.test.mjs +``` + +The command is `node flow/dfd-cli.mjs`, as `flow/report.mjs` is, and not a `commitwork` subcommand: +`flow/` is not in `package.json` `files`, so the installed CLI cannot import it. + +### The model + +```jsonc +{ "v": 1, "kind": "dfd", "title": "…", + "source": { "producer": "authored|flow-graph|…", "omitted": {…}, "filter": {…} }, + "layers": ["bronze", "silver", "gold"], // maturity order, optional + "groups": [{ "id": "internet", "label": "Internet", "boundary": "trust|system" }], + "nodes": [{ "id": "api", "kind": "process|store|external", "label": "Order API", + "subtype": "topic", "group": "internet", "layer": "bronze", "unknown": false, "note": "…" }], + "flows": [{ "from": "api", "to": "events", "label": "OrderPlaced", "mode": "sync|async|event", + "directed": true, + "evidence": { "witness": "static|runtime|both|authored", + "existence": "confirmed|contradicted|unknown", "refs": ["file:line how"] } }] } +``` + +Three element kinds. A broker or queue is a **store** with subtype `broker`, `queue`, `topic` or +`stream`: data rests in it between producer and consumer, the notation has no fourth symbol, and +asynchrony already lives on the flow's `mode`. `mode` says whether the two ends are coupled in time: +`sync` the sender waits, `async` it does not, `event` the receiver is triggered by a message. +`evidence` keeps the flow graph's two axes apart, unchanged. A flow with no `evidence` is *declared*; +neither it nor an `unknown` one is ever drawn or tabulated as confirmed. + +### What the analysis derives + +Every flow takes exactly one class, in this order: **undirected** (`directed: false`), +**self-loop** (`from === to`, a retry), **bidirectional** (A→B and B→A both exist; one two-headed +flow), **feedback** (an arc that returns inside a strongly connected component, found by walking the +component from the node the outside flows into), **forward**. From the same pass: hop rank per node +(longest path once the returning arcs are set aside) and the rank of its component, branch and merge +points, source-to-sink paths (counted exactly, listed up to a cap, the cut declared), async hops (by +mode, or by a broker at either end) and producer → broker → consumer triples. + +An undirected flow takes no part in ranks, cycles or paths: a cycle built on a guessed direction is +a fabricated feedback loop. A cycle through an element marked `unknown` is reported as uncertain. + +### The flow-graph adapter + +| flow graph | `dfd@1` | +|---|---| +| `module` | process; `group` is its first path segment; an unanalysable module is `unknown` | +| `artifact`, `store` | store, subtype `file`; a bare filename is `unknown` — a name, not a location | +| `env` | external, subtype `env` | +| `process` (spawn target) | external, subtype `binary`, when the target reads as one command name; the module's own process when it names a module; otherwise left out and counted | +| `reads` M←X | flow X→M · `writes` M→X: flow M→X · `touches`: flow with `directed: false` | +| `spawns` | flow M→P, `async` unless every witness is a `*Sync` call; left out when every witness is a bare `exec`, which a database handle also has | +| `reads` M←M′ (an import) | left out and counted: a code dependency, not a data flow (`--imports` keeps it) | + +Test modules and fixtures are left out unless `--tests`. Everything left out is counted in +`source.omitted` and printed on the page. + +Measured on this tree with the `dfd` files in it, 2026-11-11: the graph's 11,077 nodes and 21,770 +edges become 2,489 elements (547 processes, 1,140 stores, 802 externals: 755 env keys and 47 +binaries) and 2,973 flows, every one `static` / `unknown`, 1,445 of them undirected, 8 async. Left +out and counted: 3,613 test modules, 2,422 imports, 28 spawn targets that are not command names, 21 +bare-`exec` edges, 4,947 nodes with no remaining flow. The analysis finds 23 two-way pairs, 1 +feedback flow, no self-loop, 179 branch and 295 merge points, 5,281 source-to-sink paths, and 14 +cyclic components: 13 are one two-way pair, the fourteenth has ten members. **All 14 pass through a +bare-filename store**, so all 14 are reported as uncertain. Nothing in the graph is a broker, a +network endpoint or a scheduled trigger, because C1 does not extract them. + +### The page + +Gane–Sarson notation: a process is a rounded box, a store is a box open at the right (`D`; `Q` for a +broker), an external entity is a square box with a shadow. Solid is sync, dashed is async or event, +two heads is bidirectional, dotted and headless is a direction nobody determined. A feedback flow +is routed over the top on its own track; a retry is a loop on the process's corner. Maturity layers +are horizontal bands and hop rank runs left to right. A boundary is a hull when it encloses only its +members, and a caption under each member when it cannot. The same elements and flows are tabulated +under the diagram. + +Layout is rank assignment, one virtual point per rank a long flow crosses, four barycentre sweeps +and coordinates, with no clock and no randomness: one model is one sequence of bytes. The page +carries `houseCss()` and the theme follower, no other script, no `style` attribute, and a +`Content-Security-Policy` of hashes for exactly those. `renderHtml` returns `{ html, csp }` so a +panel route can send the same policy as a header. + +Not done: no panel route, no crossing minimisation beyond the sweeps, no label collision avoidance, +and a diagram over 150 elements is refused unless `--max` raises the limit. + ## Requests made while building, and where they stand The build owned `flow/**` only (spec §4) and recorded cross-cutting changes rather than making them. diff --git a/flow/dfd-cli.mjs b/flow/dfd-cli.mjs new file mode 100644 index 000000000..e97267d76 --- /dev/null +++ b/flow/dfd-cli.mjs @@ -0,0 +1,109 @@ +// flow/dfd-cli.mjs — draw a data flow diagram from a dfd@1 file or from the flow graph. +// +// usage: node flow/dfd-cli.mjs --from [options] +// node flow/dfd-cli.mjs --graph [] [--tests] [--imports] [options] +// +// --from a dfd@1 document, hand-authored or emitted by a tool (schema/dfd.schema.json) +// --graph [] the flow graph through the adapter; default /flow.json, +// which `node flow/static.mjs` or `node flow/report.mjs` writes +// --tests keep test modules and fixtures (left out by default) +// --imports keep module-to-module imports as flows (left out by default: not data flows) +// --prefix

seed the filter with every node whose label or id starts with

+// --focus seed the filter with one node, by id or label +// --hops keep everything within n flows of the seeds (default 1) +// --max refuse to draw more than n elements (default 150); filter, or raise it +// --emit html (default) | model (the dfd@1 document) | analysis (the analysis as JSON) +// --fonts inline (default) | none | served — see lib/house-css.mjs +// --title the diagram's title +// --out write there (tmp+rename) and print one summary line; default is stdout +// +// exit: 0 written · 2 usage · 20 the document is not a valid dfd@1 · 21 an input is absent, unreadable +// or not a flow graph · 22 the filter matched no node, or the model is larger than --max +// env, read at call time: CW_HARNESS_STORE (the flow store directory, default reports/harness; --graph +// without a path reads flow.json there) + +import { readJson, storePath } from './store.mjs'; +import { validateDfd, analyse, fromFlowGraph, filterModel } from '../lib/dfd.mjs'; +import { renderHtml } from '../lib/dfd-render.mjs'; +import { writeAtomic } from '../monitor/lockfile.mjs'; +import { isMainModule } from '../lib/is-main.mjs'; + +export const EXIT = Object.freeze({ OK: 0, USAGE: 2, INVALID: 20, INPUT: 21, SIZE: 22 }); +export const DEFAULT_MAX = 150; + +const VALUE_FLAGS = ['from', 'prefix', 'focus', 'hops', 'max', 'emit', 'fonts', 'title', 'out']; +const BOOL_FLAGS = ['tests', 'imports']; +const EMIT = ['html', 'model', 'analysis']; +const FONTS = ['inline', 'none', 'served']; + +/** -> { opts } or { error }. `--graph` takes an optional path. */ +export function parseArgs(argv) { + const opts = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + const name = a.startsWith('--') ? a.slice(2) : null; + const next = argv[i + 1]; + if (name === 'graph') opts.graph = next !== undefined && !next.startsWith('--') ? argv[++i] : true; + else if (BOOL_FLAGS.includes(name)) opts[name] = true; + else if (VALUE_FLAGS.includes(name)) { + if (next === undefined) return { error: `--${name} needs a value` }; + opts[name] = argv[++i]; + } else return { error: `unknown argument ${a}` }; + } + if ((opts.from === undefined) === (opts.graph === undefined)) return { error: 'give exactly one of --from and --graph []' }; + for (const k of ['hops', 'max']) { + if (opts[k] === undefined) continue; + if (!/^\d+$/.test(opts[k])) return { error: `--${k} takes a whole number, got ${opts[k]}` }; + opts[k] = Number(opts[k]); + } + if (opts.emit !== undefined && !EMIT.includes(opts.emit)) return { error: `--emit takes one of ${EMIT.join(', ')}` }; + if (opts.fonts !== undefined && !FONTS.includes(opts.fonts)) return { error: `--fonts takes one of ${FONTS.join(', ')}` }; + return { opts }; +} + +function load(path, what) { + let r; + try { r = readJson(path); } catch (e) { return { error: e.message }; } + if (r.state === 'absent') return { error: `${what} not found at ${path}` }; + return { data: r.data }; +} + +export function main(argv = process.argv.slice(2), { env = process.env, stdout = process.stdout, stderr = process.stderr } = {}) { + const fail = (code, msg) => { stderr.write(`dfd: ${msg}\n`); return code; }; + const { opts, error } = parseArgs(argv); + if (error) return fail(EXIT.USAGE, `${error}\n usage: node flow/dfd-cli.mjs --from | --graph [] [--prefix p] [--focus node] [--hops n] [--emit html|model|analysis] [--out file]`); + + let model; + if (opts.from !== undefined) { + const got = load(opts.from, 'dfd document'); + if (got.error) return fail(EXIT.INPUT, got.error); + model = got.data; + } else { + const path = opts.graph === true ? storePath('flow.json', env) : opts.graph; + const got = load(path, 'flow graph'); + if (got.error) return fail(EXIT.INPUT, `${got.error}${opts.graph === true ? ' — run `node flow/static.mjs` to write it' : ''}`); + if (!got.data || !Array.isArray(got.data.nodes) || !Array.isArray(got.data.edges)) return fail(EXIT.INPUT, `${path} is not a flow graph (no nodes and edges arrays)`); + model = fromFlowGraph(got.data, { tests: !!opts.tests, imports: !!opts.imports, title: opts.title }); + } + const valid = validateDfd(model); + if (!valid.ok) return fail(EXIT.INVALID, `not a valid dfd@1 document:\n${valid.errors.map((e) => ` ${e}`).join('\n')}`); + if (opts.title !== undefined) model = { ...model, title: opts.title }; + + model = filterModel(model, { prefix: opts.prefix, focus: opts.focus, hops: opts.hops ?? 1 }); + if (model === null) return fail(EXIT.SIZE, `the filter matched no node (prefix ${JSON.stringify(opts.prefix ?? null)}, focus ${JSON.stringify(opts.focus ?? null)})`); + + const emit = opts.emit || 'html'; + const max = opts.max ?? DEFAULT_MAX; + if (emit === 'html' && model.nodes.length > max) { + return fail(EXIT.SIZE, `${model.nodes.length} elements is more than --max ${max}: a diagram that size is not readable. Narrow it with --prefix or --focus (and --hops), or raise --max.`); + } + const text = emit === 'model' ? `${JSON.stringify(model, null, 2)}\n` + : emit === 'analysis' ? `${JSON.stringify(analyse(model), null, 2)}\n` + : renderHtml(model, { fonts: opts.fonts || 'inline' }).html; + if (opts.out === undefined) { stdout.write(text); return EXIT.OK; } + writeAtomic(opts.out, text, { mkdir: true }); + stdout.write(`dfd: ${model.nodes.length} elements, ${model.flows.length} flows -> ${opts.out} (${emit})\n`); + return EXIT.OK; +} + +if (isMainModule(import.meta.url)) process.exitCode = main(); diff --git a/flow/test/dfd-cli.test.mjs b/flow/test/dfd-cli.test.mjs new file mode 100644 index 000000000..76fa06b23 --- /dev/null +++ b/flow/test/dfd-cli.test.mjs @@ -0,0 +1,279 @@ +// The dfd command (flow/dfd-cli.mjs): its two sources, its filter, and the exit code for each way it +// declines. A refusal is asserted with what it wrote to stdout, which must be nothing. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { main, parseArgs, EXIT, DEFAULT_MAX } from '../dfd-cli.mjs'; +import { validateDfd } from '../../lib/dfd.mjs'; +import { makeNode, makeEdge, nodeId } from '../graph.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FIXTURE = join(HERE, 'fixtures', 'dfd-synthetic.json'); +const tmp = () => mkdtempSync(join(tmpdir(), 'flow-dfd-')); + +function run(argv, env = {}) { + const out = []; + const err = []; + const code = main(argv, { env, stdout: { write: (s) => out.push(s) }, stderr: { write: (s) => err.push(s) } }); + return { code, out: out.join(''), err: err.join('') }; +} + +function graphFile(dir, name = 'flow.json') { + const nodes = [makeNode('module', 'bin/a.mjs', { analysis: 'analysed' }), makeNode('module', 'lib/b.mjs', { analysis: 'analysed' }), + makeNode('store', 'reports/x.json'), makeNode('env', 'CW_X')]; + const edges = [ + makeEdge(nodeId('module', 'bin/a.mjs'), nodeId('store', 'reports/x.json'), 'writes', { evidence: ['bin/a.mjs:1 writeFileSync'] }), + makeEdge(nodeId('module', 'lib/b.mjs'), nodeId('store', 'reports/x.json'), 'reads', { evidence: ['lib/b.mjs:2 readFileSync'] }), + makeEdge(nodeId('module', 'lib/b.mjs'), nodeId('env', 'CW_X'), 'reads', { evidence: ['lib/b.mjs:3 env'] }), + ]; + const path = join(dir, name); + writeFileSync(path, JSON.stringify({ v: 1, generatedAt: '2026-11-11T00:00:00.000Z', nodes, edges })); + return path; +} + +// ── arguments ──────────────────────────────────────────────────────────────────────────────────── + +test('parseArgs reads a source, the filter and the numbers', () => { + // arrange + const argv = ['--graph', 'g.json', '--prefix', 'flow/', '--focus', 'n', '--hops', '2', '--max', '40', '--tests', '--imports', '--emit', 'model', '--fonts', 'none', '--title', 't', '--out', 'o.html']; + + // act + const { opts, error } = parseArgs(argv); + + // assert + assert.equal(error, undefined); + assert.deepEqual(opts, { graph: 'g.json', prefix: 'flow/', focus: 'n', hops: 2, max: 40, tests: true, imports: true, emit: 'model', fonts: 'none', title: 't', out: 'o.html' }); +}); + +test('parseArgs lets --graph stand without a path', () => { + // arrange + const last = ['--graph']; + const followed = ['--graph', '--tests']; + + // act + const a = parseArgs(last); + const b = parseArgs(followed); + + // assert + assert.deepEqual(a.opts, { graph: true }); + assert.deepEqual(b.opts, { graph: true, tests: true }); +}); + +test('parseArgs names each malformed invocation', () => { + // arrange + const cases = [ + [[], /exactly one of --from/], + [['--from', 'a', '--graph'], /exactly one of --from/], + [['--from'], /--from needs a value/], + [['stray'], /unknown argument stray/], + [['--nope'], /unknown argument --nope/], + [['--from', 'a', '--hops', 'two'], /--hops takes a whole number, got two/], + [['--from', 'a', '--max', '-1'], /--max takes a whole number/], + [['--from', 'a', '--emit', 'png'], /--emit takes one of html, model, analysis/], + [['--from', 'a', '--fonts', 'comic'], /--fonts takes one of inline, none, served/], + ]; + + // act + const errors = cases.map(([argv]) => parseArgs(argv).error); + + // assert + cases.forEach(([, re], i) => assert.match(errors[i], re)); +}); + +// ── from a dfd file ────────────────────────────────────────────────────────────────────────────── + +test('a dfd file renders to stdout as one HTML page', () => { + // arrange + const argv = ['--from', FIXTURE, '--fonts', 'none']; + + // act + const r = run(argv); + + // assert + assert.equal(r.code, EXIT.OK); + assert.equal(r.err, ''); + assert.match(r.out, /^/); + assert.match(r.out, / { + // arrange + const dir = tmp(); + const out = join(dir, 'deep', 'page.html'); + + // act + const r = run(['--from', FIXTURE, '--out', out]); + + // assert + assert.equal(r.code, EXIT.OK); + assert.equal(r.out, `dfd: 16 elements, 20 flows -> ${out} (html)\n`); + assert.match(readFileSync(out, 'utf8'), /url\("data:font\/woff2;base64,/, 'the default page carries its fonts'); + assert.deepEqual(readdirSync(join(dir, 'deep')), ['page.html']); +}); + +test('--emit model returns the document and --emit analysis the analysis', () => { + // arrange + const title = 'Renamed'; + + // act + const model = run(['--from', FIXTURE, '--emit', 'model', '--title', title]); + const analysis = run(['--from', FIXTURE, '--emit', 'analysis']); + + // assert + assert.equal(model.code, EXIT.OK); + assert.equal(JSON.parse(model.out).title, title); + assert.deepEqual(validateDfd(JSON.parse(model.out)).errors, []); + assert.deepEqual(JSON.parse(analysis.out).feedback.map((f) => f.flow), [10, 11]); +}); + +test('the filter narrows a dfd file and the page says it is a subset', () => { + // arrange + const argv = ['--from', FIXTURE, '--focus', 'ingest', '--hops', '1', '--emit', 'model']; + + // act + const r = run(argv); + const m = JSON.parse(r.out); + + // assert + assert.equal(r.code, EXIT.OK); + assert.deepEqual(m.nodes.map((n) => n.id), ['events', 'ingest', 'bronze']); + assert.equal(m.source.filter.nodesOmitted, 13); +}); + +test('an invalid document exits 20 and lists what is wrong', () => { + // arrange + const dir = tmp(); + const bad = join(dir, 'bad.json'); + writeFileSync(bad, JSON.stringify({ v: 1, kind: 'dfd', nodes: [{ id: 'a', kind: 'process', label: 'a' }], flows: [{ from: 'a', to: 'zz', mode: 'sync' }] })); + + // act + const r = run(['--from', bad]); + + // assert + assert.equal(r.code, EXIT.INVALID); + assert.equal(r.out, ''); + assert.match(r.err, /not a valid dfd@1 document:\n {2}dfd: flows\[0\]\.to names "zz"/); +}); + +test('an absent or corrupt input exits 21', () => { + // arrange + const dir = tmp(); + const corrupt = join(dir, 'corrupt.json'); + writeFileSync(corrupt, '{ "v": 1,'); + + // act + const absent = run(['--from', join(dir, 'none.json')]); + const broken = run(['--from', corrupt]); + + // assert + assert.equal(absent.code, EXIT.INPUT); + assert.match(absent.err, /dfd document not found at /); + assert.equal(broken.code, EXIT.INPUT); + assert.match(broken.err, /flow store corrupt/); + assert.equal(absent.out + broken.out, ''); +}); + +test('a usage error exits 2 and shows the usage line', () => { + // arrange + const argv = ['--from', FIXTURE, '--emit', 'pdf']; + + // act + const r = run(argv); + + // assert + assert.equal(r.code, EXIT.USAGE); + assert.match(r.err, /--emit takes one of[\s\S]*usage: node flow\/dfd-cli\.mjs/); +}); + +test('a filter that matches nothing, or a model past --max, exits 22 and draws nothing', () => { + // arrange + const nothing = ['--from', FIXTURE, '--prefix', 'no/such/']; + const tooBig = ['--from', FIXTURE, '--max', '5']; + + // act + const a = run(nothing); + const b = run(tooBig); + const asModel = run([...tooBig, '--emit', 'model']); + + // assert + assert.equal(a.code, EXIT.SIZE); + assert.match(a.err, /the filter matched no node \(prefix "no\/such\/", focus null\)/); + assert.equal(b.code, EXIT.SIZE); + assert.match(b.err, /16 elements is more than --max 5/); + assert.equal(a.out + b.out, ''); + assert.equal(asModel.code, EXIT.OK, 'the limit is on drawing, not on emitting the model'); + assert.equal(DEFAULT_MAX, 150); +}); + +// ── from the flow graph ────────────────────────────────────────────────────────────────────────── + +test('--graph runs the adapter and draws the result', () => { + // arrange + const path = graphFile(tmp()); + + // act + const page = run(['--graph', path, '--fonts', 'none']); + const model = JSON.parse(run(['--graph', path, '--emit', 'model', '--prefix', 'bin/', '--title', 'bin only']).out); + + // assert + assert.equal(page.code, EXIT.OK); + assert.match(page.out, /Producer: flow-graph<\/code>\. Flow graph of 4 nodes and 3 edges/); + assert.equal(model.title, 'bin only'); + assert.deepEqual(model.nodes.map((n) => n.label), ['bin/a.mjs', 'reports/x.json']); + assert.equal(model.source.filter.prefix, 'bin/'); +}); + +test('--graph without a path reads the store CW_HARNESS_STORE names', () => { + // arrange + const dir = tmp(); + graphFile(dir); + + // act + const found = run(['--graph', '--emit', 'model'], { CW_HARNESS_STORE: dir }); + const missing = run(['--graph'], { CW_HARNESS_STORE: join(dir, 'empty') }); + + // assert + assert.equal(found.code, EXIT.OK); + assert.equal(JSON.parse(found.out).nodes.length, 4); + assert.equal(missing.code, EXIT.INPUT); + assert.match(missing.err, /flow graph not found at .*flow\.json — run `node flow\/static\.mjs` to write it/); +}); + +test('a file that is not a flow graph exits 21, and a graph with nothing to draw exits 20', () => { + // arrange + const dir = tmp(); + const notGraph = join(dir, 'x.json'); + const empty = join(dir, 'empty.json'); + writeFileSync(notGraph, JSON.stringify({ nodes: {} })); + writeFileSync(empty, JSON.stringify({ v: 1, nodes: [], edges: [] })); + + // act + const a = run(['--graph', notGraph]); + const b = run(['--graph', empty]); + + // assert + assert.equal(a.code, EXIT.INPUT); + assert.match(a.err, /is not a flow graph \(no nodes and edges arrays\)/); + assert.doesNotMatch(a.err, /run `node flow\/static\.mjs`/); + assert.equal(b.code, EXIT.INVALID); + assert.match(b.err, /not a valid dfd@1 document/); +}); + +test('run as a command it exits with main\'s code', () => { + // arrange + const cli = join(HERE, '..', 'dfd-cli.mjs'); + + // act + const ok = spawnSync(process.execPath, [cli, '--from', FIXTURE, '--emit', 'analysis'], { encoding: 'utf8' }); + const bad = spawnSync(process.execPath, [cli], { encoding: 'utf8' }); + + // assert + assert.equal(ok.status, 0); + assert.equal(JSON.parse(ok.stdout).v, 1); + assert.equal(bad.status, 2); + assert.match(bad.stderr, /^dfd: give exactly one of/); +}); diff --git a/lib/dfd-render.mjs b/lib/dfd-render.mjs new file mode 100644 index 000000000..c8213d537 --- /dev/null +++ b/lib/dfd-render.mjs @@ -0,0 +1,499 @@ +// lib/dfd-render.mjs — a dfd@1 model as one self-contained HTML page with an inline SVG diagram. +// +// Notation: Gane–Sarson. A process is a rounded box with its reference in a top strip, a data store +// is a box open at the right with its reference in a left compartment (D for a store, Q for a +// broker or queue), an external entity is a square box with a shadow. Solid line = sync, dashed = +// async or event, a head at both ends = bidirectional, a dotted line with no head = direction never +// determined. A feedback flow leaves the diagram's left-to-right order: it is routed over the top on +// its own track, heavier and in the secondary accent. A retry on one process is a loop on its corner. +// +// Layout is a small layered (Sugiyama-style) pass and is deterministic: ranks by longest path, one +// virtual point per rank a long flow crosses, a fixed number of barycentre sweeps, then coordinates. +// Nothing reads the clock, so one model renders to the same bytes every time. Maturity layers +// (bronze, silver, gold) are horizontal bands; hop rank runs left to right. +// +// The page makes no request with `fonts: 'none'` or `'inline'`; it carries no script but the theme +// follower, no style attribute, and a Content-Security-Policy that admits exactly its own bytes. +// Under every diagram the same nodes and flows are tabulated: the drawing is never the only channel. + +import { createHash } from 'node:crypto'; +import { esc } from './html-escape.mjs'; +import { houseCss } from './house-css.mjs'; +import { MARK_ICON } from './brand-tokens.mjs'; +import { FOLLOWER_JS } from './theme-follower.mjs'; +import { analyse, isBroker, isDirected, longestPathRanks } from './dfd.mjs'; + +export const SWEEPS = 4; +const NODE_H = 44; +const SLOT = 84; +const CHAR = 6.7; +const NODE_MIN = 124; +const NODE_MAX = 260; +const LABEL_MAX = Math.floor((NODE_MAX - 46) / CHAR); +const TRACK = 16; +const LANE_PAD = 14; + +const sha256b64 = (s) => createHash('sha256').update(s, 'utf8').digest('base64'); +const num = (x) => String(Math.round(x * 10) / 10); +const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length; + +/** Middle ellipsis: a path's head and tail both identify it. */ +export function clip(label, max = LABEL_MAX) { + const s = String(label); + if (s.length <= max) return s; + const head = Math.ceil((max - 1) / 2); + return `${s.slice(0, head)}…${s.slice(s.length - (max - 1 - head))}`; +} + +/** A polyline with rounded corners, as an SVG path. */ +export function roundedPath(pts, r = 6) { + let d = `M${num(pts[0][0])},${num(pts[0][1])}`; + for (let i = 1; i < pts.length - 1; i++) { + const [px, py] = pts[i - 1]; + const [x, y] = pts[i]; + const [nx, ny] = pts[i + 1]; + const inLen = Math.hypot(x - px, y - py); + const outLen = Math.hypot(nx - x, ny - y); + const k = Math.min(r, inLen / 2, outLen / 2); + d += ` L${num(x - ((x - px) / inLen) * k)},${num(y - ((y - py) / inLen) * k)}`; + d += ` Q${num(x)},${num(y)} ${num(x + ((nx - x) / outLen) * k)},${num(y + ((ny - y) / outLen) * k)}`; + } + const last = pts[pts.length - 1]; + return `${d} L${num(last[0])},${num(last[1])}`; +} + +const curve = (x1, y1, x2, y2) => { + const k = (x2 - x1) / 2; + return `C${num(x1 + k)},${num(y1)} ${num(x2 - k)},${num(y2)} ${num(x2)},${num(y2)}`; +}; + +const modeStyle = (modes) => (modes.every((m) => m === 'sync') ? 'sync' : modes.every((m) => m !== 'sync') ? 'async' : 'mixed'); + +// contradicted outranks unknown outranks confirmed; a flow with no evidence at all is `declared` +function existenceOf(flows) { + const states = flows.map((f) => (f.evidence ? f.evidence.existence : 'declared')); + for (const s of ['contradicted', 'unknown', 'declared']) if (states.includes(s)) return s; + return 'confirmed'; +} + +/** + * Model + analysis -> geometry. Every flow lands in exactly one drawn edge (`edges[].flows`), and + * flows between the same two nodes share one edge. + */ +export function layout(model, analysis) { + const nodes = model.nodes; + const n = nodes.length; + const ix = new Map(nodes.map((nd, i) => [nd.id, i])); + const flows = model.flows; + const aRank = analysis.nodes.map((a) => a.rank); + + const counters = { P: 0, D: 0, Q: 0, E: 0 }; + const refs = nodes.map((nd) => { + const letter = nd.kind === 'process' ? 'P' : nd.kind === 'external' ? 'E' : isBroker(nd) ? 'Q' : 'D'; + return `${letter}${++counters[letter]}`; + }); + + // ── bundles: one drawn edge per unordered node pair, one per self-looping node ───────────────── + const bundleBy = new Map(); + flows.forEach((f, k) => { + const a = ix.get(f.from); + const b = ix.get(f.to); + const key = a === b ? `self:${a}` : `${Math.min(a, b)}|${Math.max(a, b)}`; + if (!bundleBy.has(key)) bundleBy.set(key, { self: a === b, lo: Math.min(a, b), hi: Math.max(a, b), flows: [] }); + bundleBy.get(key).flows.push(k); + }); + const bundles = [...bundleBy.values()]; + for (const b of bundles) { + const classes = b.flows.map((k) => analysis.flowClass[k]); + b.cls = b.self ? 'self' : classes.includes('bidirectional') ? 'bi' : classes.includes('feedback') ? 'fb' + : classes.includes('forward') ? 'fwd' : 'und'; + b.mode = modeStyle(b.flows.map((k) => flows[k].mode)); + b.existence = existenceOf(b.flows.map((k) => flows[k])); + b.label = clip([...new Set(b.flows.map((k) => flows[k].label).filter(Boolean))].join(' / '), 44); + } + + // ── ranks: the analysis layering, then undirected pairs oriented along its topological order ──── + const directedArcs = []; + for (const b of bundles) { + if (b.cls === 'fwd' || b.cls === 'bi') directedArcs.push(aRank[b.lo] < aRank[b.hi] ? [b.lo, b.hi] : [b.hi, b.lo]); + } + const topo = new Array(n); + longestPathRanks(n, directedArcs).order.forEach((v, t) => { topo[v] = t; }); + const hintArcs = bundles.filter((b) => b.cls === 'und').map((b) => (topo[b.lo] < topo[b.hi] ? [b.lo, b.hi] : [b.hi, b.lo])); + const rank = longestPathRanks(n, [...directedArcs, ...hintArcs]).rank; + const ranks = Math.max(...rank) + 1; + const feedbacks = bundles.filter((b) => b.cls === 'fb'); + feedbacks.forEach((b, i) => { + const k = b.flows.find((j) => analysis.flowClass[j] === 'feedback'); + b.from = ix.get(flows[k].from); + b.to = ix.get(flows[k].to); + b.track = i; + }); + + // ── lanes: unlayered first, then the declared maturity order ────────────────────────────────── + const layerNames = model.layers ? [...model.layers] : [...new Set(nodes.map((nd) => nd.layer).filter((l) => l !== undefined))]; + const laneNames = [...(nodes.some((nd) => nd.layer === undefined) ? [null] : []), ...layerNames]; + const laneOf = (nd) => laneNames.indexOf(nd.layer === undefined ? null : nd.layer); + + // ── columns: nodes and one virtual point per rank a long flow crosses ───────────────────────── + const cols = Array.from({ length: ranks }, () => []); + const items = nodes.map((nd, i) => ({ node: i, rank: rank[i], lane: laneOf(nd), group: nd.group === undefined ? `~n${i}` : `g${nd.group}`, up: [], down: [] })); + for (const it of items) cols[it.rank].push(it); + const link = (a, b) => { a.down.push(b); b.up.push(a); }; + bundles.forEach((b, bi) => { + if (b.self || b.cls === 'fb') return; + [b.a, b.b] = rank[b.lo] < rank[b.hi] ? [b.lo, b.hi] : [b.hi, b.lo]; + b.chain = [items[b.a]]; + for (let r = rank[b.a] + 1; r < rank[b.b]; r++) { + const d = { node: -1, rank: r, lane: items[b.b].lane, group: `~d${bi}:${r}`, up: [], down: [] }; + cols[r].push(d); + b.chain.push(d); + } + b.chain.push(items[b.b]); + for (let i = 1; i < b.chain.length; i++) link(b.chain[i - 1], b.chain[i]); + }); + const settle = (col) => { + col.sort((p, q) => (p.lane - q.lane) || (p.gb - q.gb) || (p.group < q.group ? -1 : p.group > q.group ? 1 : 0) || (p.bary - q.bary) || (p.pos - q.pos)); + col.forEach((it, i) => { it.pos = i; }); + }; + for (const col of cols) { + col.forEach((it, i) => { it.pos = i; it.bary = i; it.gb = 0; }); + const first = new Map(); + for (const it of col) if (!first.has(it.group)) first.set(it.group, it.pos); + for (const it of col) it.gb = first.get(it.group); + settle(col); + } + for (let s = 0; s < SWEEPS; s++) { + const down = s % 2 === 0; + for (let i = 1; i < ranks; i++) { + const col = cols[down ? i : ranks - 1 - i]; + for (const it of col) { + const near = down ? it.up : it.down; + it.bary = near.length ? mean(near.map((o) => o.pos)) : it.pos; + } + const sums = new Map(); + for (const it of col) { + const key = `${it.lane}|${it.group}`; + if (!sums.has(key)) sums.set(key, []); + sums.get(key).push(it.bary); + } + for (const it of col) it.gb = mean(sums.get(`${it.lane}|${it.group}`)); + settle(col); + } + } + + // ── coordinates ─────────────────────────────────────────────────────────────────────────────── + const widths = nodes.map((nd) => Math.max(NODE_MIN, Math.min(NODE_MAX, Math.round(clip(nd.label).length * CHAR + 46)))); + const colW = cols.map((col) => Math.max(24, ...col.filter((it) => it.node >= 0).map((it) => widths[it.node]))); + // a gutter holds the feedback risers and the label of every flow that starts on its left + const gutters = cols.map(() => Math.max(96, 52 + 12 * feedbacks.length)); + for (const b of bundles) { + if (!b.self && b.cls !== 'fb') gutters[rank[b.a]] = Math.max(gutters[rank[b.a]], Math.round(b.label.length * 6.1 + 30)); + } + const left = Math.max(40, 28 + 6 * feedbacks.length) + (layerNames.length ? 70 : 0); + const colX = []; + colW.forEach((w, r) => { colX.push(r === 0 ? left : colX[r - 1] + colW[r - 1] + gutters[r - 1]); }); + const width = colX[ranks - 1] + colW[ranks - 1] + Math.max(40, 28 + 6 * feedbacks.length); + const bodyTop = 22 + TRACK * feedbacks.length + 26; + + const rows = laneNames.map((_, l) => Math.max(1, ...cols.map((col) => col.filter((it) => it.lane === l).length))); + const laneTop = []; + rows.forEach((_, l) => { laneTop.push(l === 0 ? bodyTop : laneTop[l - 1] + rows[l - 1] * SLOT + 2 * LANE_PAD); }); + const height = laneTop[rows.length - 1] + rows[rows.length - 1] * SLOT + 2 * LANE_PAD + 8; + for (const col of cols) { + for (let l = 0; l < laneNames.length; l++) { + const inLane = col.filter((it) => it.lane === l); + inLane.forEach((it, k) => { it.cy = laneTop[l] + LANE_PAD + ((rows[l] - inLane.length) / 2 + k + 0.5) * SLOT; }); + } + } + const boxes = nodes.map((nd, i) => ({ + id: nd.id, ref: refs[i], kind: nd.kind, rank: rank[i], lane: laneNames[items[i].lane], + x: colX[rank[i]] + (colW[rank[i]] - widths[i]) / 2, y: items[i].cy - NODE_H / 2, w: widths[i], h: NODE_H, + })); + + // ── ports: flows leave on the right and arrive on the left, spread so they do not share a point ─ + const outs = nodes.map(() => []); + const ins = nodes.map(() => []); + bundles.forEach((b, bi) => { + if (b.self) return; + if (b.cls === 'fb') { outs[b.from].push({ bi, toward: -Infinity }); ins[b.to].push({ bi, toward: -Infinity }); return; } + outs[b.a].push({ bi, toward: b.chain[1].cy }); + ins[b.b].push({ bi, toward: b.chain[b.chain.length - 2].cy }); + }); + const portY = new Map(); + const spread = (list, i, side) => { + list.sort((p, q) => (p.toward - q.toward) || (p.bi - q.bi)); + const step = list.length > 1 ? Math.min(9, (NODE_H - 14) / (list.length - 1)) : 0; + list.forEach((p, k) => portY.set(`${side}${p.bi}`, items[i].cy + (k - (list.length - 1) / 2) * step)); + }; + outs.forEach((list, i) => spread(list, i, 'o')); + ins.forEach((list, i) => spread(list, i, 'i')); + + // ── edges ───────────────────────────────────────────────────────────────────────────────────── + const edges = bundles.map((b, bi) => { + const base = { flows: b.flows, cls: b.cls, mode: b.mode, existence: b.existence, label: b.label }; + if (b.self) { + const bx = boxes[b.lo]; + const x1 = bx.x + bx.w - 36; + const x2 = bx.x + bx.w - 10; + return { ...base, from: bx.id, to: bx.id, heads: 'end', lx: x2 + 6, ly: bx.y - 9, anchor: 'start', + d: `M${num(x1)},${num(bx.y)} C${num(x1)},${num(bx.y - 26)} ${num(x2)},${num(bx.y - 26)} ${num(x2)},${num(bx.y)}` }; + } + if (b.cls === 'fb') { + const u = boxes[b.from]; + const v = boxes[b.to]; + const yt = 22 + TRACK * b.track; + const xr = colX[u.rank] + colW[u.rank] + 16 + 6 * b.track; + const xl = colX[v.rank] - 16 - 6 * b.track; + const y1 = portY.get(`o${bi}`); + const y2 = portY.get(`i${bi}`); + return { ...base, from: u.id, to: v.id, heads: 'end', lx: (xr + xl) / 2, ly: yt - 4, anchor: 'middle', + d: roundedPath([[u.x + u.w, y1], [xr, y1], [xr, yt], [xl, yt], [xl, y2], [v.x, y2]]) }; + } + const a = boxes[b.a]; + const z = boxes[b.b]; + let x = a.x + a.w; + let y = portY.get(`o${bi}`); + let d = `M${num(x)},${num(y)}`; + let lx = null; + let ly = null; + const hop = (x2, y2) => { + if (lx === null) { lx = (x + x2) / 2; ly = (y + y2) / 2 - 5; } + d += ` ${curve(x, y, x2, y2)}`; + x = x2; y = y2; + }; + for (const dummy of b.chain.slice(1, -1)) { + hop(colX[dummy.rank], dummy.cy); + x = colX[dummy.rank] + colW[dummy.rank]; + d += ` L${num(x)},${num(y)}`; + } + hop(z.x, portY.get(`i${bi}`)); + // a forward flow always runs from the lower rank, so only a two-way pair takes a head at its start + return { ...base, from: a.id, to: z.id, heads: b.cls === 'bi' ? 'both' : b.cls === 'fwd' ? 'end' : 'none', lx, ly, anchor: 'middle', d }; + }); + + // ── bands and boundary hulls ────────────────────────────────────────────────────────────────── + const bands = laneNames.map((name, l) => ({ name, y: laneTop[l], h: rows[l] * SLOT + 2 * LANE_PAD })) + .filter((b) => b.name !== null); + const declared = new Map((model.groups || []).map((g) => [g.id, g])); + const hulls = []; + const loose = new Set(); + for (const g of [...new Set(nodes.map((nd) => nd.group).filter((x) => x !== undefined))]) { + const inside = boxes.filter((_, i) => nodes[i].group === g); + const hull = { + id: g, label: declared.get(g)?.label || g, boundary: declared.get(g)?.boundary || 'system', + x: Math.min(...inside.map((b) => b.x)) - 10, y: Math.min(...inside.map((b) => b.y)) - 20, + x2: Math.max(...inside.map((b) => b.x + b.w)) + 10, y2: Math.max(...inside.map((b) => b.y + b.h)) + 10, + }; + const hits = (r) => r.x < hull.x2 && r.x2 > hull.x && r.y < hull.y2 && r.y2 > hull.y; + const blocked = boxes.some((b, i) => nodes[i].group !== g && hits({ x: b.x, y: b.y, x2: b.x + b.w, y2: b.y + b.h })) || hulls.some(hits); + // a boundary that cannot be drawn without enclosing a stranger is named under each member instead + if (blocked) loose.add(g); + else hulls.push(hull); + } + boxes.forEach((b, i) => { if (loose.has(nodes[i].group)) b.caption = declared.get(nodes[i].group)?.label || nodes[i].group; }); + + return { width, height, nodes: boxes, edges, bands, hulls }; +} + +// ── SVG ──────────────────────────────────────────────────────────────────────────────────────── + +const MARKERS = ['ink', 'mut', 'fb', 'bad'].map((c) => ``).join(''); + +const markerOf = (e) => (e.existence === 'contradicted' ? 'bad' : e.cls === 'fb' ? 'fb' : e.existence === 'unknown' ? 'mut' : 'ink'); + +function nodeSvg(b, nd) { + const cx = b.x + b.w / 2; + const tip = `${b.ref} ${nd.kind}${nd.subtype ? ` (${nd.subtype})` : ''}: ${nd.label}${nd.group ? ` — boundary ${nd.group}` : ''}${nd.layer ? ` — layer ${nd.layer}` : ''}${nd.note ? ` — ${nd.note}` : ''}`; + const cls = `n ${nd.kind === 'process' ? 'proc' : nd.kind === 'store' ? 'store' : 'ext'}${nd.unknown ? ' unk' : ''}`; + let shape; + if (nd.kind === 'process') { + shape = `` + + `${b.ref}${esc(clip(nd.label))}`; + } else if (nd.kind === 'store') { + shape = `` + + `${b.ref}${esc(clip(nd.label))}`; + } else { + shape = `` + + `${b.ref}${esc(clip(nd.label))}`; + } + const cap = b.caption ? `‹${esc(clip(b.caption, 30))}›` : ''; + return `${esc(tip)}${shape}${cap}`; +} + +function edgeSvg(e) { + const m = markerOf(e); + const cls = ['e', e.cls, e.mode, ...(e.existence === 'unknown' ? ['unk'] : e.existence === 'contradicted' ? ['bad'] : [])].join(' '); + const end = e.heads === 'none' ? '' : ` marker-end="url(#dfd-a-${m})"`; + const start = e.heads === 'both' ? ` marker-start="url(#dfd-a-${m})"` : ''; + const word = { fwd: 'flow', bi: 'two-way flow', fb: 'feedback flow', self: 'retry loop', und: 'flow, direction not determined' }[e.cls]; + const label = e.label ? `${esc(e.label)}` : ''; + return `${esc(`${word}: ${e.from} → ${e.to} (${e.mode}, existence ${e.existence})`)}${label}`; +} + +export function renderSvg(model, lay) { + const title = model.title || 'data flow diagram'; + const bands = lay.bands.map((b, i) => `${esc(clip(b.name, 12))}`).join(''); + const hulls = lay.hulls.map((h) => `${esc(clip(h.label, Math.floor((h.x2 - h.x - 12) / 6.1)))}`).join(''); + return `` + + `${esc(title)}${lay.nodes.length} elements and ${model.flows.length} flows, drawn as ${lay.edges.length} edges. The tables after the diagram list every one.` + + `${MARKERS}${bands}${hulls}` + + `${lay.edges.map(edgeSvg).join('')}${lay.nodes.map((b, i) => nodeSvg(b, model.nodes[i])).join('')}`; +} + +// ── page ─────────────────────────────────────────────────────────────────────────────────────── + +const FONT_WEIGHTS = { sans: [400, 600], mono: [400, 600] }; + +export const pageCss = (fonts = 'none') => `${houseCss({ fonts, weights: FONT_WEIGHTS })} +body{font-size:.875rem;line-height:1.5} +main{padding:1rem 1.375rem 3.75rem;max-width:110rem;margin:0 auto} +h1{font-size:1.25rem;margin:0 0 .25rem} +h2{font-size:1rem;margin:1.75rem 0 .5rem} +.sub{color:var(--mut);font-size:.8125rem;margin:0 0 .75rem} +.banner{border:1px solid var(--line);border-left:4px solid var(--acc);background:var(--panel);padding:.5rem .75rem;margin:.625rem 0;font-size:.8125rem;max-width:none} +.dfd-wrap{overflow:auto;border:1px solid var(--line);background:var(--panel);max-height:85vh} +svg.dfd{display:block} +.dfd text{font:11px var(--mono);fill:var(--ink)} +.dfd .ref{font-weight:600;fill:var(--mut);font-size:10px} +.dfd .cap{fill:var(--mut);font-size:10px;font-style:italic} +.dfd .n rect,.dfd .n path{fill:var(--panel2);stroke:var(--ink);stroke-width:1.4} +.dfd .n line{stroke:var(--ink);stroke-width:1} +.dfd .n.ext .sh{fill:var(--line2);stroke:none} +.dfd .n.unk rect,.dfd .n.unk path{stroke-dasharray:4 3;stroke:var(--mut)} +.dfd .n.unk text{font-style:italic;fill:var(--mut)} +.dfd .e path{fill:none;stroke:var(--ink);stroke-width:1.5} +.dfd .e.async path{stroke-dasharray:7 4} +.dfd .e.mixed path{stroke-dasharray:10 3 2 3} +.dfd .e.und path{stroke-dasharray:1 5;stroke-linecap:round;stroke-width:2} +.dfd .e.unk path{stroke:var(--mut);stroke-width:1} +.dfd .e.und.unk path{stroke-width:1.6} +.dfd .e.fb path{stroke:var(--acc2);stroke-width:2.4} +.dfd .e.fb.unk path{stroke-width:1.4} +.dfd .e.bad path{stroke:var(--crit)} +.dfd .el{fill:var(--mut);font-size:10px;paint-order:stroke;stroke:var(--panel);stroke-width:3px;stroke-linejoin:round} +.dfd .e.fb .el{fill:var(--acc2);font-weight:600} +.dfd .mk-ink{fill:var(--ink)}.dfd .mk-mut{fill:var(--mut)}.dfd .mk-fb{fill:var(--acc2)}.dfd .mk-bad{fill:var(--crit)} +.dfd .band rect{fill:color-mix(in srgb,var(--acc) 5%,transparent);stroke:var(--line)} +.dfd .band.alt rect{fill:color-mix(in srgb,var(--acc) 10%,transparent)} +.dfd .band text{fill:var(--mut);font-weight:600;letter-spacing:.08em;text-transform:uppercase} +.dfd .hull rect{fill:none;stroke:var(--line2);stroke-width:1.2} +.dfd .hull.trust rect{stroke:var(--crit);stroke-dasharray:9 5} +.dfd .hull text{fill:var(--mut);font-size:10px} +.legend{display:flex;flex-wrap:wrap;gap:.375rem 1.25rem;list-style:none;padding:0;margin:.625rem 0;font-size:.75rem;color:var(--mut)} +.legend li{display:inline-flex;align-items:center;gap:.375rem;margin:0} +.facts{display:grid;grid-template-columns:max-content 1fr;gap:.25rem 1rem;margin:.5rem 0;font-size:.8125rem} +.facts dt{color:var(--mut)} +.facts dd{margin:0} +table{background:var(--panel);width:100%} +td.m{font-family:var(--mono);font-size:.75rem;overflow-wrap:anywhere} +.absent{color:var(--mut);font-style:italic} +`; + +const legendSvg = (inner) => ``; +const legendLine = (cls, marker, heads = 'end') => legendSvg(``); +const LEGEND = `

    +
  • ${legendSvg('')} process
    • +
  • ${legendSvg('')} data store (D), broker or queue (Q)
    • +
  • ${legendSvg('')} external entity
    • +
  • ${legendLine('fwd sync', 'ink')} sync flow
    • +
  • ${legendLine('fwd async', 'ink')} async or event flow
    • +
  • ${legendLine('fwd mixed', 'ink')} both modes on one pair
    • +
  • ${legendLine('bi sync', 'ink', 'both')} bidirectional
    • +
  • ${legendLine('fb sync', 'fb')} feedback (routed over the top)
    • +
  • ${legendSvg('')} retry loop
    • +
  • ${legendLine('und sync', 'ink', 'none')} direction not determined
    • +
  • ${legendLine('fwd sync unk', 'mut')} existence unknown
    • +
  • ${legendLine('fwd sync bad', 'bad')} contradicted
    • +
  • ${legendSvg('')} system boundary
    • +
  • ${legendSvg('')} trust boundary
    • +
  • ${legendSvg('')} maturity layer
    • +
`; + +const absent = (text) => `${esc(text)}`; +const list = (xs, empty = 'none') => (xs.length ? xs.map((x) => esc(x)).join('; ') : absent(empty)); + +function sourceBanner(model) { + const s = model.source; + if (!s) return ''; + const parts = [`Producer: ${esc(s.producer)}.`]; + if (s.note) parts.push(esc(s.note)); + if (s.graph) parts.push(`Flow graph of ${esc(s.graph.nodes)} nodes and ${esc(s.graph.edges)} edges${s.graph.generatedAt ? `, generated ${esc(s.graph.generatedAt)}` : ''}.`); + if (s.omitted) parts.push(`Left out by the adapter: ${Object.entries(s.omitted).map(([k, v]) => `${esc(k)} ${esc(v)}`).join(', ')}.`); + if (s.filter) { + const f = s.filter; + parts.push(`This is a subset. Filter: ${f.prefix === null ? '' : `prefix ${esc(f.prefix)}, `}${f.focus === null ? '' : `focus ${esc(f.focus)}, `}${esc(f.hops)} hop(s) from ${esc(f.seeds)} seed node(s); ${esc(f.nodesOmitted)} nodes and ${esc(f.flowsOmitted)} flows are not shown.`); + } + return ``; +} + +/** -> { html, csp }. `csp` is the header form; the page carries the same policy in a meta element. */ +export function renderHtml(model, { fonts = 'none', maxPaths = 200 } = {}) { + const analysis = analyse(model, { maxPaths }); + const lay = layout(model, analysis); + const ref = new Map(lay.nodes.map((b) => [b.id, b.ref])); + const labels = new Map(model.nodes.map((nd) => [nd.id, nd.label])); + const name = (id) => `${ref.get(id)} ${labels.get(id)}`; + const title = model.title || 'data flow diagram'; + const classWord = { forward: 'forward', bidirectional: 'bidirectional', feedback: 'feedback', 'self-loop': 'retry loop', undirected: 'direction not determined' }; + + const nodeRows = model.nodes.map((nd, i) => { + const a = analysis.nodes[i]; + const roles = [a.fanOut > 1 ? `branch ×${a.fanOut}` : '', a.fanIn > 1 ? `merge ×${a.fanIn}` : '', analysis.sources.includes(nd.id) ? 'source' : '', + analysis.sinks.includes(nd.id) ? 'sink' : '', a.cyclic ? 'in a cycle' : '', nd.unknown ? 'unknown' : ''].filter(Boolean); + return `${lay.nodes[i].ref}${esc(nd.kind)}${nd.subtype ? ` · ${esc(nd.subtype)}` : ''}${esc(nd.label)}${nd.group ? esc(nd.group) : absent('none')}${nd.layer ? esc(nd.layer) : absent('none')}${a.rank}${roles.length ? esc(roles.join(', ')) : ''}${nd.note ? ` ${esc(nd.note)}` : ''}`; + }).join('\n'); + const flowRows = model.flows.map((f, k) => { + const ev = f.evidence; + const refs = ev && ev.refs && ev.refs.length ? ` — ${esc(ev.refs[0])}${ev.refs.length > 1 ? ` (+${ev.refs.length - 1})` : ''}` : ''; + const evidence = !ev ? absent('declared, no evidence') + : `${ev.existence === 'unknown' ? absent('existence unknown') : esc(ev.existence)}${ev.witness ? ` · ${esc(ev.witness)}` : ''}${refs}`; + return `${k + 1}${esc(name(f.from))}${isDirected(f) ? '→' : '—'}${esc(name(f.to))}${esc(f.mode)}${classWord[analysis.flowClass[k]]}${f.label ? esc(f.label) : ''}${evidence}`; + }).join('\n'); + + const p = analysis.paths; + const facts = [ + ['Bidirectional pairs', list(analysis.bidirectional.map((b) => `${name(b.a)} ⇄ ${name(b.b)}`))], + ['Feedback flows', list(analysis.feedback.map((f) => `${name(f.from)} → ${name(f.to)}`))], + ['Retry loops', list(analysis.selfLoops.map((s) => name(s.node)))], + ['Cycles', list(analysis.cycles.map((c) => `{${c.members.map((id) => ref.get(id)).join(', ')}}${c.bidirectionalOnly ? ' two-way only' : ''}${c.uncertain ? ' uncertain: passes through an element marked unknown' : ''}`))], + ['Branch points (fan-out)', list(analysis.branches.map((b) => `${name(b.node)} → ${b.to.length}`))], + ['Merge points (fan-in)', list(analysis.merges.map((m) => `${name(m.node)} ← ${m.from.length}`))], + ['Async hops', list(analysis.asyncHops.map((h) => `${ref.get(h.from)} → ${ref.get(h.to)} (${h.reason})`))], + ['Through a broker', list(analysis.brokered.map((b) => `${ref.get(b.from)} → ${ref.get(b.via)} → ${ref.get(b.to)}`))], + ['Source-to-sink paths', `${p.total === null ? absent('more than can be counted exactly') : p.total}${p.truncated ? `, first ${p.listed.length} listed` : ''}; ${p.multiHop} of those listed are multi-hop` + + `${p.listed.length ? `; longest listed: ${esc(p.listed.reduce((m, x) => (x.length > m.length ? x : m)).map((id) => ref.get(id)).join(' → '))}` : ''}`], + ['Direction not determined', analysis.undirected.length ? `${analysis.undirected.length} flow(s), outside every figure above` : absent('none')], + ].map(([k, v]) => `
${k}
${v}
`).join('\n'); + + const css = pageCss(fonts); + const policy = (extra) => ["default-src 'none'", `style-src 'sha256-${sha256b64(css)}'`, `script-src 'sha256-${sha256b64(FOLLOWER_JS)}'`, + 'img-src data:', `font-src ${fonts === 'served' ? "'self'" : 'data:'}`, "base-uri 'none'", "form-action 'none'", ...extra].join('; '); + const html = ` + +${MARK_ICON} +${esc(title)} — data flow diagram + + +
+

${esc(title)}

+

Data flow diagram, Gane–Sarson notation · dfd@${esc(model.v)} · ${model.nodes.length} elements, ${model.flows.length} flows

+${sourceBanner(model)} +
${renderSvg(model, lay)}
+${LEGEND} +

What the analysis found

+
+${facts} +
+

Elements

+ +${nodeRows} +
refkindlabelboundarylayerhop rankroles
+

Flows

+ +${flowRows} +
#fromtomodeclasslabelevidence
+
+`; + return { html, csp: policy(["frame-ancestors 'self'"]) }; +} diff --git a/lib/dfd.mjs b/lib/dfd.mjs new file mode 100644 index 000000000..1faebe837 --- /dev/null +++ b/lib/dfd.mjs @@ -0,0 +1,428 @@ +// lib/dfd.mjs — dfd@1: a source-independent data flow diagram model, its validation, its analysis +// and the adapter from the flow graph (flow/graph.mjs). Rendering is lib/dfd-render.mjs. +// +// Three element kinds and no fourth. A message broker or queue is a STORE with subtype +// broker|queue|topic|stream: data rests in it between a producer and a consumer, the classic +// notations have no fourth symbol, and asynchrony is carried by the flow's `mode`, so a kind of its +// own would say the same thing twice and let the two disagree. +// +// Everything here is a pure function over the model. Nothing reads the clock, the environment or +// the disk except validateDfd, which opens schema/dfd.schema.json. +// +// What analysis never does: an undirected flow (`directed: false`, the flow graph's `touches`) takes +// no part in ranks, cycles, feedback or paths. Its direction was never determined, and a cycle +// built on a guessed direction is a fabricated feedback loop. + +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { validateAgainstSchema } from './json-schema.mjs'; +import { isTestModule } from '../flow/static.mjs'; + +export const DFD_VERSION = 1; +export const NODE_KINDS = Object.freeze(['process', 'store', 'external']); +export const MODES = Object.freeze(['sync', 'async', 'event']); +export const BROKER_SUBTYPES = Object.freeze(['broker', 'queue', 'topic', 'stream']); +export const FLOW_CLASSES = Object.freeze(['forward', 'bidirectional', 'feedback', 'self-loop', 'undirected']); + +export const schemaPath = () => join(dirname(fileURLToPath(import.meta.url)), '..', 'schema', 'dfd.schema.json'); + +export const isBroker = (node) => !!node && node.kind === 'store' && BROKER_SUBTYPES.includes(node.subtype); +export const isDirected = (flow) => flow.directed !== false; + +/** -> { ok, errors }. Schema first; the referential checks run only on a document of the right shape. */ +export function validateDfd(doc, { path = schemaPath() } = {}) { + const errors = validateAgainstSchema(doc, { path }).errors.slice(); + if (errors.length) return { ok: false, errors }; + if (Object.keys(doc)[0] !== 'v') errors.push('dfd: "v" must be the first field of the document — an old-shape record is a silently different meaning, not a parse failure'); + const ids = new Set(); + for (const n of doc.nodes) { + if (ids.has(n.id)) errors.push(`dfd: node id ${JSON.stringify(n.id)} is declared more than once`); + ids.add(n.id); + } + const layers = doc.layers ? new Set(doc.layers) : null; + const groups = doc.groups ? new Set(doc.groups.map((g) => g.id)) : null; + if (groups && groups.size !== doc.groups.length) errors.push('dfd: a group id is declared more than once'); + for (const n of doc.nodes) { + if (layers && n.layer !== undefined && !layers.has(n.layer)) errors.push(`dfd: node ${JSON.stringify(n.id)} names layer ${JSON.stringify(n.layer)}, which "layers" does not declare`); + if (groups && n.group !== undefined && !groups.has(n.group)) errors.push(`dfd: node ${JSON.stringify(n.id)} names group ${JSON.stringify(n.group)}, which "groups" does not declare`); + } + doc.flows.forEach((f, i) => { + for (const end of ['from', 'to']) { + if (!ids.has(f[end])) errors.push(`dfd: flows[${i}].${end} names ${JSON.stringify(f[end])}, which is not a node`); + } + }); + return { ok: errors.length === 0, errors }; +} + +// ── graph primitives (index-based, iterative: the repository's own graph has 11,000 nodes) ──────── + +/** Tarjan. -> { comp, count }: comp[i] is the component of node i, numbered by first member. */ +export function stronglyConnected(out) { + const n = out.length; + const index = new Array(n).fill(-1); + const low = new Array(n).fill(0); + const onStack = new Array(n).fill(false); + const raw = new Array(n).fill(-1); + const stack = []; + let next = 0; + let count = 0; + const open = (v, work) => { index[v] = low[v] = next++; stack.push(v); onStack[v] = true; work.push([v, 0]); }; + for (let s = 0; s < n; s++) { + if (index[s] !== -1) continue; + const work = []; + open(s, work); + while (work.length) { + const top = work[work.length - 1]; + const v = top[0]; + if (top[1] < out[v].length) { + const w = out[v][top[1]++][0]; + if (index[w] === -1) open(w, work); + else if (onStack[w]) low[v] = Math.min(low[v], index[w]); + continue; + } + work.pop(); + if (work.length) { const p = work[work.length - 1][0]; low[p] = Math.min(low[p], low[v]); } + if (low[v] !== index[v]) continue; + let w; + do { w = stack.pop(); onStack[w] = false; raw[w] = count; } while (w !== v); + count++; + } + } + const renumber = new Map(); + const comp = raw.map((c) => { if (!renumber.has(c)) renumber.set(c, renumber.size); return renumber.get(c); }); + return { comp, count }; +} + +/** + * The arcs that close a cycle, chosen deterministically: inside each component, a depth-first walk + * from its ENTRY (the first member something outside the component flows into, else its first + * member) marks every arc that returns to a node still being walked. An arc between components is + * never one. -> Set of arc keys. + */ +export function backArcs(out, comp, count) { + const members = Array.from({ length: count }, () => []); + comp.forEach((c, i) => members[c].push(i)); + const entered = new Array(out.length).fill(false); + out.forEach((arcs, v) => { for (const [w] of arcs) if (comp[w] !== comp[v]) entered[w] = true; }); + const back = new Set(); + const state = new Array(out.length).fill(0); + for (const ms of members) { + if (ms.length < 2) continue; + const root = ms.find((m) => entered[m]) ?? ms[0]; + const work = [[root, 0]]; + state[root] = 1; + while (work.length) { + const top = work[work.length - 1]; + const v = top[0]; + if (top[1] >= out[v].length) { state[v] = 2; work.pop(); continue; } + const [w, key] = out[v][top[1]++]; + if (comp[w] !== comp[v]) continue; + if (state[w] === 1) back.add(key); + else if (state[w] === 0) { state[w] = 1; work.push([w, 0]); } + } + } + return back; +} + +/** Longest-path layering of an acyclic arc list. -> { rank, order }. Throws on a cycle. */ +export function longestPathRanks(n, arcs) { + const out = Array.from({ length: n }, () => []); + const indeg = new Array(n).fill(0); + for (const [a, b] of arcs) { out[a].push(b); indeg[b]++; } + const rank = new Array(n).fill(0); + const order = []; + for (let i = 0; i < n; i++) if (indeg[i] === 0) order.push(i); + for (let q = 0; q < order.length; q++) { + const a = order[q]; + for (const b of out[a]) { + if (rank[b] < rank[a] + 1) rank[b] = rank[a] + 1; + if (--indeg[b] === 0) order.push(b); + } + } + if (order.length !== n) throw new Error('dfd: longestPathRanks was given a cyclic arc list'); + return { rank, order }; +} + +// ── analysis ───────────────────────────────────────────────────────────────────────────────────── + +/** + * Pure analysis of a VALID dfd@1 model. Flow references are indexes into model.flows. + * + * Classification, in priority order — every flow takes exactly one class: + * undirected directed:false; excluded from everything below + * self-loop from === to (a retry on the same process) + * bidirectional the pair A->B and B->A both exist; collapses to one two-headed flow + * feedback a back arc inside a strongly connected component (see backArcs) + * forward everything else + */ +export function analyse(model, { maxPaths = 200 } = {}) { + const ids = model.nodes.map((n) => n.id); + const ix = new Map(ids.map((id, i) => [id, i])); + const n = ids.length; + const flows = model.flows; + const flowClass = flows.map(() => 'forward'); + const arcs = []; + flows.forEach((f, k) => { + if (!isDirected(f)) flowClass[k] = 'undirected'; + else if (f.from === f.to) flowClass[k] = 'self-loop'; + else arcs.push({ a: ix.get(f.from), b: ix.get(f.to), k }); + }); + + const out = Array.from({ length: n }, () => []); + for (const { a, b, k } of arcs) out[a].push([b, k]); + const { comp, count } = stronglyConnected(out); + const back = backArcs(out, comp, count); + + const byPair = new Map(); + for (const { a, b, k } of arcs) { + const key = `${a}>${b}`; + if (!byPair.has(key)) byPair.set(key, []); + byPair.get(key).push(k); + } + const bidirectional = []; + for (const [key, ks] of byPair) { + const [a, b] = key.split('>').map(Number); + const reverse = byPair.get(`${b}>${a}`); + if (!reverse || a > b) continue; + const all = [...ks, ...reverse].sort((x, y) => x - y); + for (const k of all) flowClass[k] = 'bidirectional'; + bidirectional.push({ a: ids[a], b: ids[b], flows: all, modes: [...new Set(all.map((k) => flows[k].mode))] }); + } + for (const k of back) if (flowClass[k] === 'forward') flowClass[k] = 'feedback'; + + const dag = arcs.filter(({ k }) => !back.has(k)); + const { rank, order } = longestPathRanks(n, dag.map(({ a, b }) => [a, b])); + const condensed = new Set(arcs.filter(({ a, b }) => comp[a] !== comp[b]).map(({ a, b }) => `${comp[a]}>${comp[b]}`)); + const sccRank = longestPathRanks(count, [...condensed].map((s) => s.split('>').map(Number))).rank; + + const succ = Array.from({ length: n }, () => new Set()); + const pred = Array.from({ length: n }, () => new Set()); + for (const { a, b } of arcs) { succ[a].add(b); pred[b].add(a); } + const dagSucc = Array.from({ length: n }, () => new Set()); + const dagIn = new Array(n).fill(0); + for (const { a, b } of dag) if (!dagSucc[a].has(b)) { dagSucc[a].add(b); dagIn[b]++; } + + const sizes = new Array(count).fill(0); + comp.forEach((c) => { sizes[c]++; }); + const cycles = []; + for (let c = 0; c < count; c++) { + if (sizes[c] < 2) continue; + const inside = arcs.filter(({ a, b }) => comp[a] === c && comp[b] === c).map(({ k }) => k); + const fb = inside.filter((k) => flowClass[k] === 'feedback'); + cycles.push({ + members: ids.filter((_, i) => comp[i] === c), + feedback: fb, + // every arc in the component belongs to a two-way pair: a synchronised pair, not a loop + bidirectionalOnly: inside.every((k) => flowClass[k] === 'bidirectional'), + // a cycle through an element the producer could not vouch for may not exist + uncertain: model.nodes.some((nd, i) => comp[i] === c && nd.unknown === true), + }); + } + + const sources = []; + const sinks = []; + for (let i = 0; i < n; i++) { + if (dagIn[i] === 0 && dagSucc[i].size > 0) sources.push(i); + if (dagIn[i] > 0 && dagSucc[i].size === 0) sinks.push(i); + } + const ways = new Array(n).fill(0); + for (let q = order.length - 1; q >= 0; q--) { + const v = order[q]; + ways[v] = dagSucc[v].size === 0 ? 1 : [...dagSucc[v]].reduce((s, w) => s + ways[w], 0); + } + const total = sources.reduce((s, v) => s + ways[v], 0); + const listed = []; + for (const s of sources) { + const work = [[s, [...dagSucc[s]], 0]]; + const trail = [s]; + while (work.length && listed.length < maxPaths) { + const top = work[work.length - 1]; + if (top[2] >= top[1].length) { work.pop(); trail.pop(); continue; } + const w = top[1][top[2]++]; + trail.push(w); + if (dagSucc[w].size === 0) { listed.push(trail.map((i) => ids[i])); trail.pop(); continue; } + work.push([w, [...dagSucc[w]], 0]); + } + } + + const node = (id) => model.nodes[ix.get(id)]; + const asyncHops = []; + flows.forEach((f, k) => { + const byMode = f.mode !== 'sync'; + const byBroker = isBroker(node(f.from)) || isBroker(node(f.to)); + if (byMode || byBroker) asyncHops.push({ flow: k, from: f.from, to: f.to, reason: byMode && byBroker ? 'mode+broker' : byMode ? 'mode' : 'broker' }); + }); + const brokered = []; + model.nodes.forEach((b, bi) => { + if (!isBroker(b)) return; + for (const p of pred[bi]) for (const c of succ[bi]) brokered.push({ from: ids[p], via: b.id, to: ids[c] }); + }); + + return { + v: DFD_VERSION, + flowClass, + nodes: ids.map((id, i) => ({ id, rank: rank[i], sccRank: sccRank[comp[i]], cyclic: sizes[comp[i]] > 1, fanOut: succ[i].size, fanIn: pred[i].size })), + bidirectional, + selfLoops: flows.map((f, k) => ({ node: f.from, flow: k })).filter((_, k) => flowClass[k] === 'self-loop'), + feedback: flows.map((f, k) => ({ flow: k, from: f.from, to: f.to })).filter((_, k) => flowClass[k] === 'feedback'), + undirected: flowClass.map((c, k) => (c === 'undirected' ? k : -1)).filter((k) => k >= 0), + cycles, + branches: ids.map((id, i) => ({ node: id, to: [...succ[i]].map((j) => ids[j]) })).filter((b) => b.to.length > 1), + merges: ids.map((id, i) => ({ node: id, from: [...pred[i]].map((j) => ids[j]) })).filter((m) => m.from.length > 1), + sources: sources.map((i) => ids[i]), + sinks: sinks.map((i) => ids[i]), + // total is exact while it is a safe integer and null beyond: a rounded count is not a count + paths: { + total: Number.isSafeInteger(total) ? total : null, + listed, + truncated: listed.length !== total, + multiHop: listed.filter((p) => p.length > 2).length, + }, + asyncHops, + brokered, + }; +} + +// ── the flow-graph adapter ─────────────────────────────────────────────────────────────────────── + +// A spawn target the static pass captured is the first argument of exec/spawn AS WRITTEN. `exec` is +// also a database method, so SQL text arrives here as a "process". Only a single token that reads as +// a command name is drawn as an external entity; the rest are counted and left out. +const COMMAND_NAME = /^[A-Za-z0-9_@.+/-]+$/; +const SYNC_VERB = /Sync$/; +const verbOf = (ref) => String(ref).split(' ').pop(); + +/** + * Flow graph (flow/graph.mjs) -> dfd@1. The mapping: + * + * module -> process group = first path segment; unanalysable modules marked unknown + * artifact, store -> store subtype file; a bare-filename node is marked unknown + * env -> external subtype env: configuration supplied from outside the boundary + * process (spawn) -> external subtype binary, when the target reads as a command name; + * -> the module's own process when the target is a module path; + * otherwise omitted and counted + * reads M <- X -> flow X -> M writes M -> X -> flow M -> X + * spawns M -> P -> flow M -> P, async unless every witness is a *Sync call; omitted and + * counted when every witness is a bare `exec`, which a database handle + * also has + * touches M -- X -> flow M -- X, directed:false + * reads M <- M' -> a static or dynamic import: a code dependency, not a data flow. Omitted and + * counted unless { imports: true }. + * + * File reads and writes are `sync`: the mode says whether the two ends are coupled in time, and the + * static pass has no evidence that any of them is not. Every flow carries the edge's own witness and + * existence, unchanged. Nodes left with no flow are omitted and counted. + */ +export function fromFlowGraph(graph, { tests = false, imports = false, title } = {}) { + const src = new Map(graph.nodes.map((n) => [n.id, n])); + const modulePaths = new Set(graph.nodes.filter((n) => n.kind === 'module').map((n) => n.path)); + const omitted = { testModules: 0, importEdges: 0, spawnTargetsNotCommands: 0, ambiguousExecEdges: 0, edgesOfOmittedNodes: 0, isolatedNodes: 0 }; + const mapped = new Map(); + + const mapNode = (n) => { + if (n.kind === 'module') { + if (!tests && isTestModule(n.path)) { omitted.testModules++; return null; } + const unknown = n.analysis !== undefined && n.analysis !== 'analysed'; + return { id: n.id, kind: 'process', label: n.path, group: n.path.includes('/') ? n.path.split('/')[0] : '(root)', + ...(unknown ? { unknown: true, note: `not analysed (${n.analysis}): its flows are unknown, not absent` } : {}) }; + } + if (n.kind === 'artifact' || n.kind === 'store') { + return { id: n.id, kind: 'store', label: n.path, subtype: 'file', + ...(n.composed ? { unknown: true, note: 'bare filename: a name, not a location — several files may share this node' } : {}) }; + } + if (n.kind === 'env') return { id: n.id, kind: 'external', label: n.path, subtype: 'env' }; + if (modulePaths.has(n.path)) return mapped.get(`node:module:${n.path}`); + if (COMMAND_NAME.test(n.path)) return { id: n.id, kind: 'external', label: n.path, subtype: 'binary' }; + omitted.spawnTargetsNotCommands++; + return null; + }; + // modules first: a spawn target that names a module resolves to that module's process + for (const n of graph.nodes) if (n.kind === 'module') mapped.set(n.id, mapNode(n)); + for (const n of graph.nodes) if (n.kind !== 'module') mapped.set(n.id, mapNode(n)); + + const flows = []; + for (const e of graph.edges) { + const from = mapped.get(e.from); + const to = mapped.get(e.to); + if (!from || !to) { omitted.edgesOfOmittedNodes++; continue; } + const refs = [].concat(e.evidence || []); + const evidence = { witness: e.witness, existence: e.existence, refs }; + if (e.kind === 'reads') { + if (src.get(e.to).kind === 'module') { + if (!imports) { omitted.importEdges++; continue; } + flows.push({ from: to.id, to: from.id, label: 'imports', mode: 'sync', evidence }); + } else flows.push({ from: to.id, to: from.id, label: 'reads', mode: 'sync', evidence }); + } else if (e.kind === 'writes') flows.push({ from: from.id, to: to.id, label: 'writes', mode: 'sync', evidence }); + else if (e.kind === 'spawns') { + // a bare `exec` is child_process.exec or a database handle's exec: the witness cannot say which + if (refs.length > 0 && refs.every((r) => verbOf(r) === 'exec')) { omitted.ambiguousExecEdges++; continue; } + const sync = refs.length > 0 && refs.every((r) => SYNC_VERB.test(verbOf(r))); + flows.push({ from: from.id, to: to.id, label: 'spawns', mode: sync ? 'sync' : 'async', evidence }); + } else flows.push({ from: from.id, to: to.id, label: 'touches', mode: 'sync', directed: false, evidence }); + } + + const used = new Set(flows.flatMap((f) => [f.from, f.to])); + const nodes = []; + const seen = new Set(); + for (const m of mapped.values()) { + if (!m || seen.has(m.id)) continue; + seen.add(m.id); + if (used.has(m.id)) nodes.push(m); + else omitted.isolatedNodes++; + } + return { + v: DFD_VERSION, + kind: 'dfd', + title: title || 'flow graph', + source: { + producer: 'flow-graph', + graph: { v: graph.v ?? null, generatedAt: graph.generatedAt ?? null, nodes: graph.nodes.length, edges: graph.edges.length }, + options: { tests, imports }, + omitted, + }, + nodes, + flows, + }; +} + +// ── filter ─────────────────────────────────────────────────────────────────────────────────────── + +/** + * Keep the seeds and everything within `hops` flows of them, in either direction. Seeds are the + * nodes whose label or id starts with `prefix`, plus the node whose id or label is `focus`. With + * neither, the model is returned as it is; when nothing matches, null — never an empty diagram. + * What was cut is counted in source.filter: a filtered diagram that does not say so reads as the + * whole system. + */ +export function filterModel(model, { prefix, focus, hops = 1 } = {}) { + if (prefix === undefined && focus === undefined) return model; + const seeds = new Set(); + for (const n of model.nodes) { + if (prefix !== undefined && (n.label.startsWith(prefix) || n.id.startsWith(prefix))) seeds.add(n.id); + if (focus !== undefined && (n.id === focus || n.label === focus)) seeds.add(n.id); + } + if (seeds.size === 0) return null; + const near = new Map(model.nodes.map((n) => [n.id, []])); + for (const f of model.flows) { near.get(f.from).push(f.to); near.get(f.to).push(f.from); } + const keep = new Set(seeds); + let edge = [...seeds]; + for (let h = 0; h < hops; h++) { + const next = []; + for (const id of edge) for (const other of near.get(id)) if (!keep.has(other)) { keep.add(other); next.push(other); } + edge = next; + } + const nodes = model.nodes.filter((n) => keep.has(n.id)); + const flows = model.flows.filter((f) => keep.has(f.from) && keep.has(f.to)); + const { v, kind, title, source, ...rest } = model; + return { + v, kind, ...(title === undefined ? {} : { title }), + source: { + ...(source || { producer: 'authored' }), + filter: { prefix: prefix ?? null, focus: focus ?? null, hops, seeds: seeds.size, + nodesKept: nodes.length, nodesOmitted: model.nodes.length - nodes.length, flowsOmitted: model.flows.length - flows.length }, + }, + ...rest, + nodes, + flows, + }; +} diff --git a/lib/test/dfd-render.test.mjs b/lib/test/dfd-render.test.mjs new file mode 100644 index 000000000..8fdf6fdfc --- /dev/null +++ b/lib/test/dfd-render.test.mjs @@ -0,0 +1,504 @@ +// The dfd page (lib/dfd-render.mjs). What is asserted is what a reader relies on without checking: +// no two elements overlap, every flow in the model is drawn exactly once, one model is one sequence +// of bytes, and the page admits nothing but its own style and the theme follower. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyse } from '../dfd.mjs'; +import { layout, renderSvg, renderHtml, pageCss, clip, roundedPath, SWEEPS } from '../dfd-render.mjs'; +import { FOLLOWER_JS } from '../theme-follower.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const fixture = () => JSON.parse(readFileSync(join(HERE, 'fixtures', 'dfd-synthetic.json'), 'utf8')); +const sha = (s) => createHash('sha256').update(s, 'utf8').digest('base64'); +const lay = (m) => layout(m, analyse(m)); +const node = (id, kind = 'process', extra = {}) => ({ id, kind, label: id, ...extra }); +const flow = (from, to, extra = {}) => ({ from, to, mode: 'sync', ...extra }); +const doc = (nodes, flows, extra = {}) => ({ v: 1, kind: 'dfd', ...extra, nodes, flows }); + +const overlaps = (boxes) => { + const out = []; + for (let i = 0; i < boxes.length; i++) { + for (let j = i + 1; j < boxes.length; j++) { + const a = boxes[i]; + const b = boxes[j]; + if (a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h) out.push(`${a.id} / ${b.id}`); + } + } + return out; +}; + +// ── helpers ────────────────────────────────────────────────────────────────────────────────────── + +test('clip keeps a short label and cuts a long one in the middle', () => { + // arrange + const long = 'monitor/extractors/sast-lint/_deno-check-counts.mjs'; + + // act + const short = clip('bin/a.mjs'); + const cut = clip(long, 21); + + // assert + assert.equal(short, 'bin/a.mjs'); + assert.equal(cut, 'monitor/ex…counts.mjs'); + assert.equal(cut.length, 21); +}); + +test('roundedPath rounds each inner corner and never by more than half a segment', () => { + // arrange + const pts = [[0, 0], [10, 0], [10, 4], [30, 4]]; + + // act + const d = roundedPath(pts, 6); + + // assert + assert.equal(d, 'M0,0 L8,0 Q10,0 10,2 L10,2 Q10,4 12,4 L30,4'); +}); + +// ── layout ─────────────────────────────────────────────────────────────────────────────────────── + +test('no two elements of the fixture overlap and all sit inside the drawing', () => { + // arrange + const m = fixture(); + + // act + const g = lay(m); + + // assert + assert.deepEqual(overlaps(g.nodes), []); + assert.ok(g.nodes.every((b) => b.x >= 0 && b.y >= 0 && b.x + b.w <= g.width && b.y + b.h <= g.height)); +}); + +test('every flow of the fixture is drawn exactly once', () => { + // arrange + const m = fixture(); + + // act + const g = lay(m); + const drawn = g.edges.flatMap((e) => e.flows).sort((a, b) => a - b); + + // assert + assert.deepEqual(drawn, m.flows.map((_, k) => k)); + assert.ok(g.edges.every((e) => /^M[-\d.]+,[-\d.]+ /.test(e.d) && !/NaN|undefined|Infinity/.test(e.d))); +}); + +test('the fixture draws one edge of each kind the notation has', () => { + // arrange + const m = fixture(); + + // act + const g = lay(m); + const by = (cls) => g.edges.filter((e) => e.cls === cls); + + // assert + assert.deepEqual(by('bi').map((e) => [e.from, e.to, e.heads, e.label]), [['customer', 'api', 'both', 'order request / order status']]); + assert.deepEqual(by('self').map((e) => [e.from, e.heads]), [['ingest', 'end']]); + assert.deepEqual(by('fb').map((e) => [e.from, e.to, e.mode, e.existence]), [['validate', 'transform', 'sync', 'declared'], ['validate', 'bronze', 'async', 'unknown']]); + assert.deepEqual(by('fwd').filter((e) => e.mode === 'async').map((e) => [e.from, e.to]), [['api', 'events'], ['events', 'ingest']]); + assert.equal(by('fwd').find((e) => e.to === 'orders-db').existence, 'confirmed'); +}); + +test('a feedback edge runs over the top: its track is above every element', () => { + // arrange + const m = fixture(); + + // act + const g = lay(m); + const top = Math.min(...g.nodes.map((b) => b.y)); + const tracks = g.edges.filter((e) => e.cls === 'fb').map((e) => e.ly + 4); + + // assert + assert.equal(tracks.length, 2); + assert.ok(tracks.every((y) => y < top)); + assert.notEqual(tracks[0], tracks[1]); +}); + +test('ranks run left to right and maturity layers are bands in the declared order', () => { + // arrange + const m = fixture(); + + // act + const g = lay(m); + const box = Object.fromEntries(g.nodes.map((b) => [b.id, b])); + + // assert + assert.deepEqual(g.bands.map((b) => b.name), ['bronze', 'silver', 'gold']); + assert.ok(g.bands[0].y < g.bands[1].y && g.bands[1].y < g.bands[2].y); + assert.ok(box.ingest.x < box.bronze.x && box.bronze.x < box.transform.x && box.transform.x < box.validate.x); + for (const b of g.nodes.filter((n) => n.lane)) { + const band = g.bands.find((x) => x.name === b.lane); + assert.ok(b.y >= band.y && b.y + b.h <= band.y + band.h, `${b.id} sits inside ${b.lane}`); + } +}); + +test('element references follow kind and declaration order, with Q for a broker', () => { + // arrange + const m = fixture(); + + // act + const refs = Object.fromEntries(lay(m).nodes.map((b) => [b.id, b.ref])); + + // assert + assert.equal(refs.customer, 'E1'); + assert.equal(refs.api, 'P1'); + assert.equal(refs['orders-db'], 'D1'); + assert.equal(refs.events, 'Q1'); + assert.equal(refs.analyst, 'E2'); +}); + +test('a boundary is a hull when it encloses only its members', () => { + // arrange + const m = fixture(); + + // act + const g = lay(m); + + // assert + assert.deepEqual(g.hulls.map((h) => [h.id, h.label, h.boundary]), [['internet', 'Internet', 'trust'], ['platform', 'Order platform', 'system']]); + assert.ok(g.nodes.every((b) => b.caption === undefined)); +}); + +test('a boundary that would enclose a stranger is named under each member instead', () => { + // arrange: the two members of g sit in ranks 0 and 2, with an outsider between them + const m = doc([node('a', 'process', { group: 'g' }), node('mid'), node('z', 'store', { group: 'g' })], [flow('a', 'mid'), flow('mid', 'z')]); + + // act + const g = lay(m); + + // assert + assert.deepEqual(g.hulls, []); + assert.deepEqual(g.nodes.map((b) => b.caption), ['g', undefined, 'g']); + assert.match(renderSvg(m, g), /]*>‹g›<\/text>/); +}); + +test('two boundaries whose hulls would collide keep the first and caption the second', () => { + // arrange: h's only member sits between the two members of g in one column + const m = doc( + [node('src'), node('g1', 'store', { group: 'g' }), node('h1', 'store', { group: 'h' }), node('g2', 'store', { group: 'g' })], + [flow('src', 'g1'), flow('src', 'h1'), flow('src', 'g2')], + ); + + // act + const g = lay(m); + + // assert + assert.deepEqual(overlaps(g.nodes), []); + assert.equal(g.hulls.length + new Set(g.nodes.map((b) => b.caption).filter(Boolean)).size, 2); +}); + +test('a flow that skips a rank is routed through a point in the rank it crosses', () => { + // arrange + const m = doc([node('a'), node('b'), node('c')], [flow('a', 'b'), flow('b', 'c'), flow('a', 'c', { label: 'skip' })]); + + // act + const g = lay(m); + const skip = g.edges.find((e) => e.label === 'skip'); + const direct = g.edges.find((e) => e.from === 'a' && e.to === 'b'); + + // assert + assert.equal((skip.d.match(/C/g) || []).length, 2); + assert.match(skip.d, / L/); + assert.equal((direct.d.match(/C/g) || []).length, 1); + assert.deepEqual(overlaps(g.nodes), []); +}); + +test('an undirected flow is drawn with no head and still places its ends in different ranks', () => { + // arrange + const m = doc([node('p'), node('s', 'store'), node('t', 'store')], [flow('p', 's', { directed: false }), flow('t', 'p', { directed: false })]); + + // act + const g = lay(m); + + // assert + assert.deepEqual(g.edges.map((e) => [e.cls, e.heads]), [['und', 'none'], ['und', 'none']]); + assert.equal(new Set(g.nodes.map((b) => b.rank)).size > 1, true); + assert.deepEqual(overlaps(g.nodes), []); +}); + +test('sync and async flows between one pair share an edge drawn as mixed', () => { + // arrange + const m = doc([node('a'), node('b')], [flow('a', 'b', { label: 'x' }), flow('a', 'b', { mode: 'event', label: 'y' })]); + + // act + const g = lay(m); + + // assert + assert.equal(g.edges.length, 1); + assert.deepEqual([g.edges[0].mode, g.edges[0].label, g.edges[0].flows], ['mixed', 'x / y', [0, 1]]); +}); + +test('a contradicted flow outranks an unknown one on the same edge', () => { + // arrange + const ev = (existence) => ({ evidence: { existence } }); + const m = doc([node('a'), node('b'), node('c')], [flow('a', 'b', ev('unknown')), flow('a', 'b', ev('contradicted')), flow('b', 'c', ev('confirmed'))]); + + // act + const g = lay(m); + const svg = renderSvg(m, g); + + // assert + assert.deepEqual(g.edges.map((e) => e.existence), ['contradicted', 'confirmed']); + assert.match(svg, /class="e fwd sync bad"[^>]*>.*?marker-end="url\(#dfd-a-bad\)"/); +}); + +test('a model with every node layered has no unlayered lane, and one without layers has no band', () => { + // arrange + const layered = doc([node('a', 'process', { layer: 'raw' }), node('b', 'store', { layer: 'fine' })], [flow('a', 'b')]); + const plain = doc([node('a'), node('b', 'store')], [flow('a', 'b')]); + + // act + const g1 = lay(layered); + const g2 = lay(plain); + + // assert + assert.deepEqual(g1.bands.map((b) => b.name), ['raw', 'fine']); + assert.deepEqual(g1.nodes.map((b) => b.lane), ['raw', 'fine']); + assert.deepEqual(g2.bands, []); + assert.ok(g2.width < g1.width, 'the band label column is reserved only when there are bands'); +}); + +test('a single element with no flow still lays out', () => { + // arrange + const m = doc([node('only', 'external')], []); + + // act + const g = lay(m); + + // assert + assert.equal(g.nodes.length, 1); + assert.deepEqual(g.edges, []); + assert.ok(g.width > g.nodes[0].w && g.height > g.nodes[0].h); +}); + +test('the barycentre sweeps are a fixed number, and they untangle a crossed pair', () => { + // arrange: declared so that a->y and b->x cross unless the second rank is reordered + const m = doc([node('a'), node('b'), node('x', 'store'), node('y', 'store')], [flow('a', 'y'), flow('b', 'x')]); + + // act + const g = lay(m); + const box = Object.fromEntries(g.nodes.map((b) => [b.id, b])); + + // assert + assert.equal(SWEEPS, 4); + assert.ok(box.a.y < box.b.y); + assert.ok(box.y.y < box.x.y, 'y follows a to the top'); +}); + +// ── SVG and page ───────────────────────────────────────────────────────────────────────────────── + +test('each element kind is drawn in its Gane–Sarson shape', () => { + // arrange + const m = fixture(); + + // act + const svg = renderSvg(m, lay(m)); + + // assert + assert.match(svg, /P1 process: Order API — boundary platform<\/title><rect [^>]*rx="9"\/><line /); + assert.match(svg, /<g class="n store" data-id="events"><title>Q1 store \(topic\): order-events — boundary platform — Kafka topic<\/title><path d="M[\d.]+,[\d.]+ H[\d.]+ V[\d.]+ H[\d.]+"\/>/); + assert.match(svg, /<g class="n ext" data-id="customer"><title>E1 external \(user\): Customer — boundary internet<\/title><rect class="sh" /); + assert.match(svg, /<g class="n proc" data-id="ingest"><title>P2 process: Ingest — layer bronze<\/title>/); + assert.match(svg, /role="img" aria-labelledby="dfd-title dfd-desc"/); +}); + +test('an element marked unknown is drawn as an absence', () => { + // arrange + const m = doc([node('a', 'process', { unknown: true, note: 'not analysed' }), node('b', 'store')], [flow('a', 'b')]); + + // act + const svg = renderSvg(m, lay(m)); + + // assert + assert.match(svg, /<g class="n proc unk" data-id="a"><title>P1 process: a — not analysed<\/title>/); +}); + +test('edge markup carries its flows, its class and the right heads', () => { + // arrange + const m = fixture(); + + // act + const svg = renderSvg(m, lay(m)); + + // assert + assert.match(svg, /<g class="e bi sync" data-flows="0,1" data-from="customer" data-to="api">.*?marker-start="url\(#dfd-a-ink\)" marker-end="url\(#dfd-a-ink\)"/); + assert.match(svg, /<g class="e fb sync" data-flows="10" data-from="validate" data-to="transform"><title>feedback flow: validate → transform \(sync, existence declared\)<\/title>/); + assert.match(svg, /<g class="e fb async unk" data-flows="11"[^>]*>.*?marker-end="url\(#dfd-a-fb\)"/); + assert.match(svg, /<g class="e self sync" data-flows="5" data-from="ingest" data-to="ingest"><title>retry loop:/); + assert.match(svg, /<g class="e fwd async" data-flows="3" data-from="api" data-to="events">/); +}); + +test('an undirected, unknown flow is dotted, grey and headless', () => { + // arrange + const m = doc([node('p'), node('s', 'store')], [flow('p', 's', { directed: false, evidence: { witness: 'static', existence: 'unknown' } })]); + + // act + const svg = renderSvg(m, lay(m)); + const edge = /<g class="e und sync unk".*?<\/g>/.exec(svg)[0]; + + // assert + assert.match(edge, /direction not determined/); + assert.doesNotMatch(edge, /marker-/); +}); + +test('one model renders to the same bytes every time', () => { + // arrange + const a = fixture(); + const b = fixture(); + + // act + const first = renderHtml(a); + const second = renderHtml(b); + + // assert + assert.equal(first.html, second.html); + assert.equal(first.csp, second.csp); + assert.equal(sha(first.html), sha(renderHtml(fixture()).html)); +}); + +test('the page makes no request and carries no style attribute', () => { + // arrange + const m = fixture(); + + // act + const { html } = renderHtml(m); + const urls = [...html.matchAll(/url\(([^)]*)\)/g)].map((x) => x[1]); + const links = [...html.matchAll(/(?:href|src)="([^"]*)"/g)].map((x) => x[1]); + + // assert + assert.ok(urls.length > 0 && urls.every((u) => u.startsWith('#dfd-a-')), urls.join(' ')); + assert.ok(links.every((u) => u.startsWith('data:')), links.join(' ')); + assert.doesNotMatch(html, /\sstyle="/); + assert.doesNotMatch(html.replace('http://www.w3.org/2000/svg', ''), /https?:\/\//); + assert.equal((html.match(/<script/g) || []).length, 1); +}); + +test('the policy admits exactly the inlined style and the theme follower', () => { + // arrange + const m = fixture(); + + // act + const { html, csp } = renderHtml(m); + const style = /<style>([\s\S]*?)<\/style>/.exec(html)[1]; + const script = /<script>([\s\S]*?)<\/script>/.exec(html)[1]; + const meta = /<meta http-equiv="Content-Security-Policy" content="([^"]*)">/.exec(html)[1]; + + // assert + assert.equal(style, pageCss('none')); + assert.equal(script, FOLLOWER_JS); + assert.equal(csp, `default-src 'none'; style-src 'sha256-${sha(style)}'; script-src 'sha256-${sha(script)}'; img-src data:; font-src data:; base-uri 'none'; form-action 'none'; frame-ancestors 'self'`); + assert.equal(`${meta}; frame-ancestors 'self'`, csp); +}); + +test('served fonts point at the panel and inline fonts travel in the page', () => { + // arrange + const m = doc([node('a')], []); + + // act + const served = renderHtml(m, { fonts: 'served' }); + const inline = renderHtml(m, { fonts: 'inline' }); + + // assert + assert.match(served.csp, /font-src 'self'/); + assert.match(served.html, /url\("\/static\/fonts\//); + assert.match(inline.csp, /font-src data:/); + assert.match(inline.html, /url\("data:font\/woff2;base64,/); +}); + +test('labels, titles and evidence are escaped wherever they land', () => { + // arrange + const m = doc( + [node('a', 'process', { label: '<img src=x onerror=1>', group: '"g"', note: '<b>n</b>' }), node('b', 'store', { label: 'b&b' })], + [flow('a', 'b', { label: '</text><script>x</script>', evidence: { existence: 'confirmed', witness: 'static', refs: ['<r>', 'two'] } })], + { title: 'T <i>' }, + ); + + // act + const { html } = renderHtml(m); + + // assert + assert.doesNotMatch(html, /<img|<i>|<b>n|<r>|<script>x/); + assert.match(html, /<title>T <i> — data flow diagram<\/title>/); + assert.match(html, /confirmed · static — <r> \(\+1\)/); +}); + +test('the tables repeat every element and every flow of the diagram', () => { + // arrange + const m = fixture(); + + // act + const { html } = renderHtml(m); + const rows = (html.match(/<tr><td/g) || []).length; + + // assert + assert.equal(rows, m.nodes.length + m.flows.length); + assert.match(html, /<td class="m">Q1<\/td><td>store · topic<\/td><td class="m">order-events<\/td><td>platform<\/td><td><span class="absent">none<\/span><\/td><td class="tnum">2<\/td>/); + assert.match(html, /<td class="tnum">11<\/td><td class="m">P5 Validate quality<\/td><td>→<\/td><td class="m">P3 Clean and conform<\/td><td>sync<\/td><td>feedback<\/td><td>validation errors<\/td><td><span class="absent">declared, no evidence<\/span><\/td>/); + assert.match(html, /<td>async<\/td><td>feedback<\/td><td>replay request<\/td><td><span class="absent">existence unknown<\/span> · authored<\/td>/); + assert.match(html, /<td>retry loop<\/td><td>retry on timeout<\/td>/); + assert.match(html, /branch ×3/); + assert.match(html, /merge ×2, in a cycle/); +}); + +test('the findings name each topology in words', () => { + // arrange + const m = fixture(); + + // act + const { html } = renderHtml(m); + const fact = (k) => new RegExp(`<dt>${k}</dt><dd>([^\\n]*)</dd>`).exec(html)[1]; + + // assert + assert.equal(fact('Bidirectional pairs'), 'E1 Customer ⇄ P1 Order API'); + assert.equal(fact('Feedback flows'), 'P5 Validate quality → P3 Clean and conform; P5 Validate quality → D2 Raw orders'); + assert.equal(fact('Retry loops'), 'P2 Ingest'); + assert.equal(fact('Cycles'), '{E1, P1} two-way only; {D2, P3, P4, P5}'); + assert.equal(fact('Through a broker'), 'P1 → Q1 → P2'); + assert.match(fact('Source-to-sink paths'), /^4; 4 of those listed are multi-hop; longest listed: E1 → P1 → Q1 → P2 → D2 → P3 → P4 → P5 → D4 → P6 → D5 → P8 → E2$/); + assert.equal(fact('Direction not determined'), '<span class="absent">none</span>'); +}); + +test('an empty finding is written as an absence, and an uncountable one says so', () => { + // arrange + const specs = []; + for (let i = 0; i < 60; i++) specs.push([`p${i}`, `s${i}a`], [`p${i}`, `s${i}b`], [`s${i}a`, `p${i + 1}`], [`s${i}b`, `p${i + 1}`]); + const ids = [...new Set(specs.flat())]; + const big = doc(ids.map((id) => node(id, id[0] === 'p' ? 'process' : 'store', id === 's0a' ? { unknown: true } : {})), [...specs.map(([a, b]) => flow(a, b)), flow('p1', 'p0'), flow('p0', 's0a', { directed: false })]); + + // act + const { html } = renderHtml(big, { maxPaths: 5 }); + const fact = (k) => new RegExp(`<dt>${k.replace(/[()]/g, '\\$&')}</dt><dd>([^\\n]*)</dd>`).exec(html)[1]; + + // assert + assert.equal(fact('Retry loops'), '<span class="absent">none</span>'); + assert.match(fact('Source-to-sink paths'), /^<span class="absent">more than can be counted exactly<\/span>, first 5 listed; 5 of those listed are multi-hop/); + assert.match(fact('Cycles'), /uncertain: passes through an element marked unknown/); + assert.equal(fact('Direction not determined'), '1 flow(s), outside every figure above'); + assert.match(html, /<td>—<\/td>/); + assert.match(html, /<td>(?:[^<]*, )?unknown<\/td>/); +}); + +test('the banner says who produced the model, what the adapter left out and what the filter cut', () => { + // arrange + const bare = doc([node('a')], []); + const noted = doc([node('a')], [], { source: { producer: 'authored', note: 'by hand' } }); + const adapted = doc([node('a')], [], { source: { + producer: 'flow-graph', graph: { nodes: 9, edges: 4, generatedAt: '2026-11-11T00:00:00.000Z' }, omitted: { testModules: 3 }, + filter: { prefix: 'flow/', focus: null, hops: 1, seeds: 2, nodesKept: 1, nodesOmitted: 8, flowsOmitted: 4 }, + } }); + const focused = doc([node('a')], [], { source: { producer: 'flow-graph', graph: { nodes: 9, edges: 4, generatedAt: null }, + filter: { prefix: null, focus: 'a', hops: 2, seeds: 1, nodesKept: 1, nodesOmitted: 0, flowsOmitted: 0 } } }); + const banner = (m) => /<p class="banner">(.*?)<\/p>/.exec(renderHtml(m).html)[1]; + + // act + const texts = [bare, noted, adapted, focused].map(banner); + + // assert + assert.match(texts[0], /^Authored model\. No producer is recorded/); + assert.equal(texts[1], 'Producer: <code>authored</code>. by hand'); + assert.equal(texts[2], 'Producer: <code>flow-graph</code>. Flow graph of 9 nodes and 4 edges, generated 2026-11-11T00:00:00.000Z. Left out by the adapter: testModules 3. ' + + '<strong>This is a subset.</strong> Filter: prefix <code>flow/</code>, 1 hop(s) from 2 seed node(s); 8 nodes and 4 flows are not shown.'); + assert.match(texts[3], /Flow graph of 9 nodes and 4 edges\. <strong>This is a subset\.<\/strong> Filter: focus <code>a<\/code>, 2 hop\(s\)/); +}); diff --git a/lib/test/dfd.test.mjs b/lib/test/dfd.test.mjs new file mode 100644 index 000000000..064184e67 --- /dev/null +++ b/lib/test/dfd.test.mjs @@ -0,0 +1,554 @@ +// dfd@1: validation, the analysis and the flow-graph adapter (lib/dfd.mjs). The analysis is the +// part a reader trusts without re-deriving, so each topology is asserted on a model small enough to +// check by hand, and once more on the fixture that carries all of them together. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + validateDfd, analyse, fromFlowGraph, filterModel, stronglyConnected, backArcs, longestPathRanks, + isBroker, isDirected, schemaPath, NODE_KINDS, MODES, FLOW_CLASSES, +} from '../dfd.mjs'; +import { makeNode, makeEdge, nodeId } from '../../flow/graph.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const fixture = () => JSON.parse(readFileSync(join(HERE, 'fixtures', 'dfd-synthetic.json'), 'utf8')); + +/** 'a>b' sync, 'a~>b' async, 'a--b' undirected; node kind from the id's first letter (p, s, e). */ +function model(flowSpecs, nodeExtra = {}) { + const kinds = { p: 'process', s: 'store', e: 'external' }; + const ids = []; + const flows = flowSpecs.map((spec) => { + const m = /^(\w+)(~>|>|--)(\w+)$/.exec(spec); + for (const id of [m[1], m[3]]) if (!ids.includes(id)) ids.push(id); + return { from: m[1], to: m[3], mode: m[2] === '~>' ? 'async' : 'sync', ...(m[2] === '--' ? { directed: false } : {}) }; + }); + return { v: 1, kind: 'dfd', nodes: ids.map((id) => ({ id, kind: kinds[id[0]], label: id, ...(nodeExtra[id] || {}) })), flows }; +} + +// ── validation ─────────────────────────────────────────────────────────────────────────────────── + +test('the synthetic fixture is a valid dfd@1 document', () => { + // arrange + const doc = fixture(); + + // act + const r = validateDfd(doc); + + // assert + assert.deepEqual(r, { ok: true, errors: [] }); +}); + +test('the vocabulary is three element kinds, three modes and five flow classes', () => { + // arrange + const schema = JSON.parse(readFileSync(schemaPath(), 'utf8')); + + // act + const kinds = schema.properties.nodes.items.properties.kind.enum; + const modes = schema.properties.flows.items.properties.mode.enum; + + // assert + assert.deepEqual(kinds, [...NODE_KINDS]); + assert.deepEqual(modes, [...MODES]); + assert.equal(FLOW_CLASSES.length, 5); +}); + +test('a document whose first field is not v is refused', () => { + // arrange + const { v, ...rest } = fixture(); + const doc = { ...rest, v }; + + // act + const r = validateDfd(doc); + + // assert + assert.equal(r.ok, false); + assert.match(r.errors[0], /"v" must be the first field/); +}); + +test('a schema failure is reported alone, before any referential check runs', () => { + // arrange + const doc = fixture(); + doc.nodes[0].kind = 'broker'; + doc.flows[0].to = 'nowhere'; + + // act + const r = validateDfd(doc); + + // assert + assert.equal(r.ok, false); + assert.ok(r.errors.every((e) => e.startsWith('schema:')), r.errors.join('\n')); +}); + +test('a schema that cannot be opened validates nothing', () => { + // arrange + const doc = fixture(); + + // act + const r = validateDfd(doc, { path: join(HERE, 'fixtures', 'no-such.schema.json') }); + + // assert + assert.equal(r.ok, false); + assert.match(r.errors[0], /could not be read or parsed/); +}); + +test('duplicate ids, dangling flow ends and undeclared layers and groups are each named', () => { + // arrange + const doc = fixture(); + doc.nodes.push({ id: 'api', kind: 'process', label: 'again', layer: 'platinum', group: 'elsewhere' }); + doc.flows.push({ from: 'ghost', to: 'api', mode: 'sync' }); + doc.groups.push({ id: 'internet', boundary: 'trust' }); + + // act + const { ok, errors } = validateDfd(doc); + + // assert + assert.equal(ok, false); + assert.equal(errors.length, 5); + assert.match(errors.join('\n'), /node id "api" is declared more than once/); + assert.match(errors.join('\n'), /a group id is declared more than once/); + assert.match(errors.join('\n'), /names layer "platinum"/); + assert.match(errors.join('\n'), /names group "elsewhere"/); + assert.match(errors.join('\n'), /flows\[20\]\.from names "ghost"/); +}); + +test('without declared layers or groups any layer and group string is accepted', () => { + // arrange + const doc = model(['p1>s1'], { p1: { layer: 'raw', group: 'anything' } }); + + // act + const r = validateDfd(doc); + + // assert + assert.deepEqual(r.errors, []); +}); + +test('a broker is a store with a broker subtype, and a flow is directed unless it says otherwise', () => { + // arrange + const topic = { id: 't', kind: 'store', label: 't', subtype: 'topic' }; + const table = { id: 'd', kind: 'store', label: 'd', subtype: 'relational' }; + const proc = { id: 'p', kind: 'process', label: 'p', subtype: 'queue' }; + + // act + const brokers = [topic, table, proc, undefined].map(isBroker); + const directed = [{ from: 'a', to: 'b' }, { from: 'a', to: 'b', directed: true }, { from: 'a', to: 'b', directed: false }].map(isDirected); + + // assert + assert.deepEqual(brokers, [true, false, false, false]); + assert.deepEqual(directed, [true, true, false]); +}); + +// ── graph primitives ───────────────────────────────────────────────────────────────────────────── + +test('stronglyConnected numbers components by their first member', () => { + // arrange: 0 -> 1 -> 2 -> 1, 3 alone + const out = [[[1, 0]], [[2, 1]], [[1, 2]], []]; + + // act + const { comp, count } = stronglyConnected(out); + + // assert + assert.equal(count, 3); + assert.deepEqual(comp, [0, 1, 1, 2]); +}); + +test('backArcs walks a component from the node the outside flows into', () => { + // arrange: 2 -> 1 -> 2 is a cycle; node 0 enters it at 2, so 1 -> 2 is the arc that returns + const out = [[[2, 0]], [[2, 1]], [[1, 2]]]; + const { comp, count } = stronglyConnected(out); + + // act + const back = backArcs(out, comp, count); + + // assert + assert.deepEqual([...back], [1]); +}); + +test('backArcs starts at the first member when nothing outside enters the component', () => { + // arrange: 0 -> 1 -> 2 -> 0 with a chord 1 -> 0 + const out = [[[1, 0]], [[2, 1], [0, 3]], [[0, 2]]]; + const { comp, count } = stronglyConnected(out); + + // act + const back = backArcs(out, comp, count); + + // assert + assert.deepEqual([...back].sort(), [2, 3]); +}); + +test('longestPathRanks puts a node one past its deepest predecessor', () => { + // arrange: a diamond with a long side + const arcs = [[0, 1], [0, 2], [2, 3], [1, 4], [3, 4]]; + + // act + const { rank, order } = longestPathRanks(5, arcs); + + // assert + assert.deepEqual(rank, [0, 1, 1, 2, 3]); + assert.equal(order.length, 5); +}); + +test('longestPathRanks refuses a cyclic arc list', () => { + // arrange + const arcs = [[0, 1], [1, 0]]; + + // act + const run = () => longestPathRanks(2, arcs); + + // assert + assert.throws(run, /cyclic arc list/); +}); + +// ── analysis ───────────────────────────────────────────────────────────────────────────────────── + +test('a pair of opposite flows is bidirectional, not feedback', () => { + // arrange + const m = model(['e1>p1', 'p1>e1', 'p1>s1']); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.flowClass, ['bidirectional', 'bidirectional', 'forward']); + assert.deepEqual(a.bidirectional, [{ a: 'e1', b: 'p1', flows: [0, 1], modes: ['sync'] }]); + assert.deepEqual(a.feedback, []); + assert.deepEqual(a.cycles, [{ members: ['e1', 'p1'], feedback: [], bidirectionalOnly: true, uncertain: false }]); +}); + +test('parallel flows in both directions all join one bidirectional pair and report both modes', () => { + // arrange + const m = model(['p1>p2', 'p1~>p2', 'p2>p1']); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.bidirectional, [{ a: 'p1', b: 'p2', flows: [0, 1, 2], modes: ['sync', 'async'] }]); +}); + +test('a flow from a downstream process back to an upstream one is the feedback flow', () => { + // arrange + const m = model(['e1>p1', 'p1>p2', 'p2>p3', 'p3>p1', 'p3>s1']); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.feedback, [{ flow: 3, from: 'p3', to: 'p1' }]); + assert.deepEqual(a.cycles, [{ members: ['p1', 'p2', 'p3'], feedback: [3], bidirectionalOnly: false, uncertain: false }]); + assert.deepEqual(a.nodes.map((n) => n.rank), [0, 1, 2, 3, 4]); + assert.deepEqual(a.nodes.map((n) => n.sccRank), [0, 1, 1, 1, 2]); + assert.deepEqual(a.nodes.map((n) => n.cyclic), [false, true, true, true, false]); +}); + +test('a cycle through an element marked unknown is reported as uncertain', () => { + // arrange + const m = model(['p1>s1', 's1>p2', 'p2>p1'], { s1: { unknown: true } }); + + // act + const a = analyse(m); + + // assert + assert.equal(a.cycles.length, 1); + assert.equal(a.cycles[0].uncertain, true); +}); + +test('a retry on one process is a self-loop and adds nothing to its fan-in or fan-out', () => { + // arrange + const m = model(['p1>p1', 'p1>s1']); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.selfLoops, [{ node: 'p1', flow: 0 }]); + assert.deepEqual(a.nodes[0], { id: 'p1', rank: 0, sccRank: 0, cyclic: false, fanOut: 1, fanIn: 0 }); + assert.deepEqual(a.cycles, []); +}); + +test('an undirected flow takes no part in ranks, cycles, paths or fan counts', () => { + // arrange: the only thing closing the loop is a flow whose direction nobody determined + const m = model(['p1>s1', 's1--p1', 'p1--s2']); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.flowClass, ['forward', 'undirected', 'undirected']); + assert.deepEqual(a.undirected, [1, 2]); + assert.deepEqual(a.cycles, []); + assert.equal(a.paths.total, 1); + assert.equal(a.nodes[2].fanIn, 0); +}); + +test('branch and merge points are counted over distinct neighbours', () => { + // arrange: p1 fans out to two stores twice over, both merge into p2 + const m = model(['p1>s1', 'p1>s1', 'p1>s2', 's1>p2', 's2>p2']); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.branches, [{ node: 'p1', to: ['s1', 's2'] }]); + assert.deepEqual(a.merges, [{ node: 'p2', from: ['s1', 's2'] }]); + assert.deepEqual(a.sources, ['p1']); + assert.deepEqual(a.sinks, ['p2']); + assert.deepEqual(a.paths, { total: 2, listed: [['p1', 's1', 'p2'], ['p1', 's2', 'p2']], truncated: false, multiHop: 2 }); +}); + +test('a flow is an async hop by its mode, by a broker at either end, or by both', () => { + // arrange + const m = model(['p1~>s1', 's1>p2', 'p2~>s2', 'p2>p3'], { s1: { subtype: 'queue' }, s2: { subtype: 'log' } }); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.asyncHops, [ + { flow: 0, from: 'p1', to: 's1', reason: 'mode+broker' }, + { flow: 1, from: 's1', to: 'p2', reason: 'broker' }, + { flow: 2, from: 'p2', to: 's2', reason: 'mode' }, + ]); + assert.deepEqual(a.brokered, [{ from: 'p1', via: 's1', to: 'p2' }]); +}); + +test('paths are listed up to the cap and the cut is declared', () => { + // arrange + const m = model(['p1>s1', 'p1>s2', 'p1>s3']); + + // act + const a = analyse(m, { maxPaths: 2 }); + + // assert + assert.equal(a.paths.total, 3); + assert.equal(a.paths.listed.length, 2); + assert.equal(a.paths.truncated, true); + assert.equal(a.paths.multiHop, 0); +}); + +test('a path count past the safe integers is null, never a rounded number', () => { + // arrange: sixty diamonds in a row is 2^60 source-to-sink paths + const specs = []; + for (let i = 0; i < 60; i++) specs.push(`p${i}>s${i}a`, `p${i}>s${i}b`, `s${i}a>p${i + 1}`, `s${i}b>p${i + 1}`); + const m = model(specs); + + // act + const a = analyse(m, { maxPaths: 3 }); + + // assert + assert.equal(a.paths.total, null); + assert.equal(a.paths.truncated, true); + assert.equal(a.paths.listed.length, 3); + assert.equal(a.paths.listed[0].length, 121); +}); + +test('the fixture carries every topology at once', () => { + // arrange + const m = fixture(); + const at = (from, to) => m.flows.findIndex((f) => f.from === from && f.to === to); + + // act + const a = analyse(m); + + // assert + assert.deepEqual(a.bidirectional.map((b) => [b.a, b.b]), [['customer', 'api']]); + assert.deepEqual(a.selfLoops, [{ node: 'ingest', flow: at('ingest', 'ingest') }]); + assert.deepEqual(a.feedback.map((f) => `${f.from}>${f.to}`), ['validate>transform', 'validate>bronze']); + assert.deepEqual(a.cycles.map((c) => c.members), [['customer', 'api'], ['bronze', 'transform', 'enrich', 'validate']]); + assert.deepEqual(a.branches.map((b) => b.node), ['api', 'validate', 'silver']); + assert.deepEqual(a.merges.map((b) => b.node), ['bronze', 'transform', 'gold']); + assert.deepEqual(a.brokered, [{ from: 'api', via: 'events', to: 'ingest' }]); + assert.deepEqual(a.asyncHops.map((h) => h.reason), ['mode+broker', 'mode+broker', 'mode']); + assert.equal(a.paths.total, 4); + assert.equal(Math.max(...a.paths.listed.map((p) => p.length)), 13); + assert.equal(a.flowClass.filter((c) => c === 'forward').length, 15); +}); + +// ── the flow-graph adapter ─────────────────────────────────────────────────────────────────────── + +function flowGraph() { + const mod = (p, extra) => makeNode('module', p, extra); + const nodes = [ + mod('bin/tool.mjs', { analysis: 'analysed' }), + mod('bin/helper.mjs', { analysis: 'analysed' }), + mod('bin/broken.mjs', { analysis: 'unparseable' }), + mod('bin/test/tool.test.mjs', { analysis: 'analysed' }), + mod('alone.mjs', { analysis: 'analysed' }), + makeNode('store', 'reports/out.json'), + makeNode('artifact', 'rollup.json', { composed: true }), + makeNode('env', 'CW_HOME'), + makeNode('process', 'git'), + makeNode('process', 'CREATE TABLE t (a TEXT)'), + makeNode('process', 'COMMIT'), + makeNode('process', 'bin/helper.mjs'), + ]; + const m = (p) => nodeId('module', p); + const edges = [ + makeEdge(m('bin/tool.mjs'), nodeId('store', 'reports/out.json'), 'writes', { evidence: ['bin/tool.mjs:9 writeFileSync'] }), + makeEdge(m('bin/helper.mjs'), nodeId('store', 'reports/out.json'), 'reads', { witness: 'both', existence: 'confirmed', evidence: ['bin/helper.mjs:4 readFileSync'] }), + makeEdge(m('bin/tool.mjs'), nodeId('artifact', 'rollup.json'), 'touches', { evidence: ['bin/tool.mjs:12 literal'] }), + makeEdge(m('bin/tool.mjs'), nodeId('env', 'CW_HOME'), 'reads', { evidence: ['bin/tool.mjs:3 env'] }), + makeEdge(m('bin/tool.mjs'), m('bin/helper.mjs'), 'reads', { evidence: ['bin/tool.mjs static-import'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'git'), 'spawns', { evidence: ['bin/tool.mjs:20 execFileSync', 'bin/tool.mjs:31 spawnSync'] }), + makeEdge(m('bin/helper.mjs'), nodeId('process', 'git'), 'spawns', { evidence: ['bin/helper.mjs:7 spawn'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'CREATE TABLE t (a TEXT)'), 'spawns', { evidence: ['bin/tool.mjs:40 exec'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'COMMIT'), 'spawns', { evidence: ['bin/tool.mjs:41 exec'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'bin/helper.mjs'), 'spawns', { evidence: null }), + makeEdge(m('bin/test/tool.test.mjs'), nodeId('store', 'reports/out.json'), 'reads', { evidence: ['bin/test/tool.test.mjs:5 readFileSync'] }), + makeEdge(m('bin/broken.mjs'), nodeId('env', 'CW_HOME'), 'reads', { evidence: ['bin/broken.mjs:1 env'] }), + ]; + return { v: 1, generatedAt: '2026-11-11T00:00:00.000Z', nodes, edges }; +} + +test('fromFlowGraph maps modules, files, env keys and spawned binaries by the stated rules', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + const byLabel = Object.fromEntries(d.nodes.map((n) => [n.label, n])); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(Object.keys(d).slice(0, 2), ['v', 'kind']); + assert.deepEqual(byLabel['bin/tool.mjs'], { id: 'node:module:bin/tool.mjs', kind: 'process', label: 'bin/tool.mjs', group: 'bin' }); + assert.equal(byLabel['bin/broken.mjs'].unknown, true); + assert.match(byLabel['bin/broken.mjs'].note, /not analysed \(unparseable\)/); + assert.deepEqual(byLabel['reports/out.json'], { id: 'node:store:reports/out.json', kind: 'store', label: 'reports/out.json', subtype: 'file' }); + assert.equal(byLabel['rollup.json'].unknown, true); + assert.deepEqual(byLabel.CW_HOME, { id: 'node:env:CW_HOME', kind: 'external', label: 'CW_HOME', subtype: 'env' }); + assert.deepEqual(byLabel.git, { id: 'node:process:git', kind: 'external', label: 'git', subtype: 'binary' }); + assert.equal(d.nodes.some((n) => /CREATE TABLE|COMMIT|test/.test(n.label)), false); +}); + +test('fromFlowGraph turns a read around, keeps a write, and leaves a touch undirected', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + const flow = (label, from) => d.flows.find((f) => f.label === label && f.from === from); + + // assert + assert.deepEqual(flow('writes', 'node:module:bin/tool.mjs'), { + from: 'node:module:bin/tool.mjs', to: 'node:store:reports/out.json', label: 'writes', mode: 'sync', + evidence: { witness: 'static', existence: 'unknown', refs: ['bin/tool.mjs:9 writeFileSync'] }, + }); + assert.deepEqual(flow('reads', 'node:store:reports/out.json'), { + from: 'node:store:reports/out.json', to: 'node:module:bin/helper.mjs', label: 'reads', mode: 'sync', + evidence: { witness: 'both', existence: 'confirmed', refs: ['bin/helper.mjs:4 readFileSync'] }, + }); + assert.equal(flow('reads', 'node:env:CW_HOME').to, 'node:module:bin/tool.mjs'); + assert.equal(flow('touches', 'node:module:bin/tool.mjs').directed, false); +}); + +test('fromFlowGraph reads a spawn as sync only when every witness is a Sync call', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + const spawns = d.flows.filter((f) => f.label === 'spawns').map((f) => [f.from, f.to, f.mode]); + + // assert + assert.deepEqual(spawns, [ + ['node:module:bin/tool.mjs', 'node:process:git', 'sync'], + ['node:module:bin/helper.mjs', 'node:process:git', 'async'], + ['node:module:bin/tool.mjs', 'node:module:bin/helper.mjs', 'async'], + ]); +}); + +test('fromFlowGraph counts everything it leaves out', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual(d.source, { + producer: 'flow-graph', + graph: { v: 1, generatedAt: '2026-11-11T00:00:00.000Z', nodes: 12, edges: 12 }, + options: { tests: false, imports: false }, + omitted: { testModules: 1, importEdges: 1, spawnTargetsNotCommands: 1, ambiguousExecEdges: 1, edgesOfOmittedNodes: 2, isolatedNodes: 2 }, + }); + assert.equal(d.title, 'flow graph'); +}); + +test('fromFlowGraph keeps tests and imports when asked, and takes a title', () => { + // arrange + const graph = flowGraph(); + delete graph.v; + delete graph.generatedAt; + + // act + const d = fromFlowGraph(graph, { tests: true, imports: true, title: 'everything' }); + + // assert + assert.equal(d.title, 'everything'); + assert.equal(d.source.omitted.testModules, 0); + assert.equal(d.source.omitted.importEdges, 0); + assert.deepEqual(d.source.graph, { v: null, generatedAt: null, nodes: 12, edges: 12 }); + assert.ok(d.nodes.some((n) => n.label === 'bin/test/tool.test.mjs')); + assert.deepEqual(d.flows.find((f) => f.label === 'imports'), { + from: 'node:module:bin/helper.mjs', to: 'node:module:bin/tool.mjs', label: 'imports', mode: 'sync', + evidence: { witness: 'static', existence: 'unknown', refs: ['bin/tool.mjs static-import'] }, + }); +}); + +// ── filter ─────────────────────────────────────────────────────────────────────────────────────── + +test('filterModel with no filter returns the model itself', () => { + // arrange + const m = fixture(); + + // act + const out = filterModel(m); + + // assert + assert.equal(out, m); +}); + +test('filterModel keeps a focus node and its neighbours, in either direction, and counts the cut', () => { + // arrange + const m = fixture(); + + // act + const out = filterModel(m, { focus: 'Ingest' }); + + // assert + assert.deepEqual(out.nodes.map((n) => n.id), ['events', 'ingest', 'bronze']); + assert.equal(out.flows.length, 3); + assert.deepEqual(out.source.filter, { prefix: null, focus: 'Ingest', hops: 1, seeds: 1, nodesKept: 3, nodesOmitted: 13, flowsOmitted: 17 }); + assert.deepEqual(Object.keys(out).slice(0, 4), ['v', 'kind', 'title', 'source']); + assert.deepEqual(validateDfd(out).errors, []); +}); + +test('filterModel seeds from a prefix and widens by hops', () => { + // arrange + const m = model(['p1>s1', 's1>p2', 'p2>s2', 's2>p3']); + + // act + const none = filterModel(m, { prefix: 'p1', hops: 0 }); + const two = filterModel(m, { prefix: 'p1', hops: 2 }); + const byFocusId = filterModel(m, { focus: 's2', hops: 1 }); + + // assert + assert.deepEqual(none.nodes.map((n) => n.id), ['p1']); + assert.deepEqual(two.nodes.map((n) => n.id), ['p1', 's1', 'p2']); + assert.deepEqual(byFocusId.nodes.map((n) => n.id), ['p2', 's2', 'p3']); + assert.equal(two.source.producer, 'authored'); + assert.equal('title' in two, false); +}); + +test('filterModel returns null when nothing matches, never an empty diagram', () => { + // arrange + const m = fixture(); + + // act + const out = filterModel(m, { prefix: 'no/such/', focus: 'nobody' }); + + // assert + assert.equal(out, null); +}); diff --git a/lib/test/fixtures/dfd-synthetic.json b/lib/test/fixtures/dfd-synthetic.json new file mode 100644 index 000000000..e665753c9 --- /dev/null +++ b/lib/test/fixtures/dfd-synthetic.json @@ -0,0 +1,54 @@ +{ + "v": 1, + "kind": "dfd", + "title": "Order analytics (synthetic fixture)", + "source": { + "producer": "authored", + "note": "Hand-authored test fixture: every element kind and every flow topology the visualiser supports, in one model." + }, + "layers": ["bronze", "silver", "gold"], + "groups": [ + { "id": "internet", "label": "Internet", "boundary": "trust" }, + { "id": "platform", "label": "Order platform", "boundary": "system" } + ], + "nodes": [ + { "id": "customer", "kind": "external", "label": "Customer", "subtype": "user", "group": "internet" }, + { "id": "api", "kind": "process", "label": "Order API", "group": "platform" }, + { "id": "orders-db", "kind": "store", "label": "Orders DB", "subtype": "relational", "group": "platform" }, + { "id": "events", "kind": "store", "label": "order-events", "subtype": "topic", "group": "platform", "note": "Kafka topic" }, + { "id": "ingest", "kind": "process", "label": "Ingest", "layer": "bronze" }, + { "id": "bronze", "kind": "store", "label": "Raw orders", "subtype": "lake", "layer": "bronze" }, + { "id": "transform", "kind": "process", "label": "Clean and conform", "layer": "silver" }, + { "id": "enrich", "kind": "process", "label": "Enrich", "layer": "silver" }, + { "id": "validate", "kind": "process", "label": "Validate quality", "layer": "silver" }, + { "id": "quarantine", "kind": "store", "label": "Quarantine", "subtype": "log", "layer": "silver" }, + { "id": "silver", "kind": "store", "label": "Conformed orders", "subtype": "lake", "layer": "silver" }, + { "id": "aggregate", "kind": "process", "label": "Aggregate", "layer": "gold" }, + { "id": "features", "kind": "process", "label": "Build features", "layer": "gold" }, + { "id": "gold", "kind": "store", "label": "Gold marts", "subtype": "relational", "layer": "gold" }, + { "id": "serve", "kind": "process", "label": "Report API" }, + { "id": "analyst", "kind": "external", "label": "BI analyst", "subtype": "user" } + ], + "flows": [ + { "from": "customer", "to": "api", "label": "order request", "mode": "sync" }, + { "from": "api", "to": "customer", "label": "order status", "mode": "sync" }, + { "from": "api", "to": "orders-db", "label": "order row", "mode": "sync", "evidence": { "witness": "both", "existence": "confirmed", "refs": ["api/orders.mjs:41 insert"] } }, + { "from": "api", "to": "events", "label": "OrderPlaced", "mode": "async" }, + { "from": "events", "to": "ingest", "label": "OrderPlaced", "mode": "event" }, + { "from": "ingest", "to": "ingest", "label": "retry on timeout", "mode": "sync" }, + { "from": "ingest", "to": "bronze", "label": "raw batch", "mode": "sync" }, + { "from": "bronze", "to": "transform", "label": "raw batch", "mode": "sync" }, + { "from": "transform", "to": "enrich", "label": "clean rows", "mode": "sync" }, + { "from": "enrich", "to": "validate", "label": "candidate rows", "mode": "sync" }, + { "from": "validate", "to": "transform", "label": "validation errors", "mode": "sync" }, + { "from": "validate", "to": "bronze", "label": "replay request", "mode": "async", "evidence": { "witness": "authored", "existence": "unknown" } }, + { "from": "validate", "to": "quarantine", "label": "rejected rows", "mode": "sync" }, + { "from": "validate", "to": "silver", "label": "valid rows", "mode": "sync" }, + { "from": "silver", "to": "aggregate", "label": "orders", "mode": "sync" }, + { "from": "silver", "to": "features", "label": "orders", "mode": "sync" }, + { "from": "aggregate", "to": "gold", "label": "daily totals", "mode": "sync" }, + { "from": "features", "to": "gold", "label": "feature rows", "mode": "sync" }, + { "from": "gold", "to": "serve", "label": "marts", "mode": "sync" }, + { "from": "serve", "to": "analyst", "label": "report", "mode": "sync" } + ] +} diff --git a/schema/dfd.schema.json b/schema/dfd.schema.json new file mode 100644 index 000000000..20f4ca7f2 --- /dev/null +++ b/schema/dfd.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://commitwork.online/schema/dfd.schema.json", + "title": "dfd@1 — a source-independent data flow diagram model (lib/dfd.mjs)", + "description": "Three element kinds (process, store, external) and flows between them. A broker or queue is a store with subtype broker|queue|topic|stream. Referential integrity (unique ids, flow endpoints, declared layers and groups, v as the first field) is checked in code by validateDfd, not here.", + "type": "object", + "required": ["v", "kind", "nodes", "flows"], + "additionalProperties": false, + "properties": { + "$comment": { "type": "string" }, + "v": { "const": 1 }, + "kind": { "const": "dfd" }, + "title": { "type": "string", "minLength": 1, "maxLength": 200 }, + "source": { + "type": "object", + "required": ["producer"], + "additionalProperties": false, + "properties": { + "producer": { "type": "string", "minLength": 1 }, + "note": { "type": "string" }, + "graph": { "type": "object" }, + "options": { "type": "object" }, + "omitted": { "type": "object" }, + "filter": { "type": "object" } + } + }, + "layers": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, + "groups": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "boundary"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "label": { "type": "string", "minLength": 1 }, + "boundary": { "enum": ["trust", "system"] } + } + } + }, + "nodes": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "kind", "label"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "enum": ["process", "store", "external"] }, + "label": { "type": "string", "minLength": 1 }, + "subtype": { "type": "string", "minLength": 1 }, + "group": { "type": "string", "minLength": 1 }, + "layer": { "type": "string", "minLength": 1 }, + "unknown": { "type": "boolean" }, + "note": { "type": "string" } + } + } + }, + "flows": { + "type": "array", + "items": { + "type": "object", + "required": ["from", "to", "mode"], + "additionalProperties": false, + "properties": { + "from": { "type": "string", "minLength": 1 }, + "to": { "type": "string", "minLength": 1 }, + "label": { "type": "string" }, + "mode": { "enum": ["sync", "async", "event"] }, + "directed": { "type": "boolean" }, + "evidence": { + "type": "object", + "required": ["existence"], + "additionalProperties": false, + "properties": { + "witness": { "enum": ["static", "runtime", "both", "authored"] }, + "existence": { "enum": ["confirmed", "contradicted", "unknown"] }, + "refs": { "type": "array", "items": { "type": "string" } } + } + } + } + } + } + } +} From 7731ea35b70460257b4d3a5499669b1019e2ec25 Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:26:15 +1030 Subject: [PATCH 02/36] test(dfd): point the CLI tests at the fixture's new home --- flow/test/dfd-cli.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/test/dfd-cli.test.mjs b/flow/test/dfd-cli.test.mjs index 76fa06b23..bad846b59 100644 --- a/flow/test/dfd-cli.test.mjs +++ b/flow/test/dfd-cli.test.mjs @@ -12,7 +12,7 @@ import { validateDfd } from '../../lib/dfd.mjs'; import { makeNode, makeEdge, nodeId } from '../graph.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); -const FIXTURE = join(HERE, 'fixtures', 'dfd-synthetic.json'); +const FIXTURE = join(HERE, '..', '..', 'lib', 'test', 'fixtures', 'dfd-synthetic.json'); const tmp = () => mkdtempSync(join(tmpdir(), 'flow-dfd-')); function run(argv, env = {}) { From 1fcdbc64b1c519169478bc22654545369fe1744a Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:32:46 +1030 Subject: [PATCH 03/36] feat(flow): trace network connects and listens, and record open access --- flow/runtime.mjs | 65 ++++-- flow/test/unit-runtime-read-trace.test.mjs | 6 +- flow/trace-hook.cjs | 258 ++++++++++++++++++++- 3 files changed, 310 insertions(+), 19 deletions(-) diff --git a/flow/runtime.mjs b/flow/runtime.mjs index 16bd8e97d..36bd25423 100644 --- a/flow/runtime.mjs +++ b/flow/runtime.mjs @@ -69,45 +69,84 @@ export function readTrace(path) { const installed = rows.find((r) => r.t === 'installed'); if (!installed) return { state: 'unusable', rows, reason: 'no installed row — the preload never ran' }; if (!installed.selfWitness) return { state: 'unusable', rows, reason: 'hook could not witness its own read — patch is inert' }; - return { state: 'usable', rows, reason: null, malformed, node: installed.node, pid: installed.pid }; + return { state: 'usable', rows, reason: null, malformed, node: installed.node, pid: installed.pid, network: networkWitness(rows) }; } /** - * -> { state, observations, coverage, reason } + * -> { state: 'usable'|'unusable', reason, pids } + * + * The network has its own non-vacuity proof, per PROCESS: one trace file collects every process a + * run spawns (`node --test` forks one per file), and a process whose probe was not observed may be + * one whose patches never took. Its network rows are refused, not merged into a clean-looking set. + */ +export function networkWitness(rows) { + const installed = rows.filter((r) => r.t === 'installed'); + const pids = installed.filter((r) => r.netWitness === true).map((r) => r.pid); + if (pids.length === 0) { + const why = installed.map((r) => r.netReason).find(Boolean) || 'no installed row carries a network self-witness'; + return { state: 'unusable', reason: `hook could not witness its own network probe — ${why}`, pids }; + } + return { state: 'usable', reason: null, pids }; +} + +/** + * -> { state, observations, coverage, reason, unattributed, network } * * `coverage` is the set of repo modules this run actually entered, derived from module loads. An * observation whose actor is null is kept but produces no edge — attribution failed, which is * unknown, not absent. + * + * Kinds: reads | writes | spawns from fs and child_process; connects | listens from the network, + * with `to` in C1's endpoint key form (`<scheme>://<host>[:<port>]`, `listen:<port>`). An actual + * open carries `access` (read | write | append | readwrite, accumulated); a stat does not. + * `network.state` says whether absence of a network observation means anything. */ export function summarise(traceRead) { if (traceRead.state !== 'usable') { - return { v: SCHEMA_VERSION, state: traceRead.state, reason: traceRead.reason, observations: [], coverage: [], unattributed: 0 }; + return { v: SCHEMA_VERSION, state: traceRead.state, reason: traceRead.reason, observations: [], coverage: [], unattributed: 0, + network: { state: 'unusable', reason: traceRead.reason, observed: 0, refused: 0 } }; } const coverage = new Set(); const observations = []; let unattributed = 0; + let refused = 0; + const network = traceRead.network || networkWitness(traceRead.rows); + const netPids = new Set(network.pids); for (const r of traceRead.rows) { - if (r.t === 'installed') continue; + if (r.t === 'installed' || r.probe) continue; // A module load: the ESM/CJS loader reads the source through the public fs, and no repo frame is // on the stack. That is coverage, not a dataflow edge. const isModuleLoad = r.actor === null && r.t === 'read' && /\.[cm]?js$/.test(r.path || ''); if (isModuleLoad) { coverage.add(r.path); continue; } - if (!r.actor) { unattributed += 1; continue; } + const isNet = r.t === 'connect' || r.t === 'listen'; + // A network row from a process that never proved it can see the network is refused outright: + // keeping it would be harmless, but it would let a half-blind run look like a sighted one. + if (isNet && !netPids.has(r.pid)) { refused += 1; continue; } + if (!r.actor || (isNet && !r.key)) { unattributed += 1; continue; } coverage.add(r.actor); - observations.push({ from: r.actor, to: r.path, kind: r.t === 'spawn' ? 'spawns' : (r.t === 'write' ? 'writes' : 'reads'), via: r.via }); + observations.push({ from: r.actor, to: isNet ? r.key : r.path, kind: KIND[r.t] || 'reads', via: r.via, + ...(r.access ? { access: r.access } : {}), ...(r.ephemeral ? { ephemeral: true } : {}) }); } - const seen = new Set(); - const deduped = observations.filter((o) => { + // One observation per from|to|kind. `access` accumulates — a module that reads a file and later + // rewrites it opened it both ways, and that union is what gives a `touches` edge `both`. + const by = new Map(); + for (const o of observations) { const k = JSON.stringify([o.from, o.to, o.kind]); - if (seen.has(k)) return false; - seen.add(k); - return true; - }); - return { v: SCHEMA_VERSION, state: 'usable', reason: null, observations: deduped, coverage: [...coverage].sort(), unattributed }; + const prev = by.get(k); + const { access, ...rest } = o; + if (!prev) { by.set(k, { ...rest, ...(access ? { access: [access] } : {}) }); continue; } + if (access && !(prev.access || []).includes(access)) prev.access = [...(prev.access || []), access].sort(); + } + const deduped = [...by.values()]; + const netObserved = deduped.filter((o) => o.kind === 'connects' || o.kind === 'listens').length; + return { v: SCHEMA_VERSION, state: 'usable', reason: null, observations: deduped, coverage: [...coverage].sort(), unattributed, + network: { state: network.state, reason: network.reason, observed: netObserved, refused } }; } +const KIND = { read: 'reads', write: 'writes', spawn: 'spawns', connect: 'connects', listen: 'listens' }; + /** Convenience: run, read, summarise. Cleans the trace unless `keep` is set. */ export function observe(argv, opts = {}) { const run = record(argv, opts); diff --git a/flow/test/unit-runtime-read-trace.test.mjs b/flow/test/unit-runtime-read-trace.test.mjs index 8afc5e768..35bf49258 100644 --- a/flow/test/unit-runtime-read-trace.test.mjs +++ b/flow/test/unit-runtime-read-trace.test.mjs @@ -58,7 +58,7 @@ test('returns usable state with node and pid when installed row has selfWitness' const p = join(dir, 'trace.json'); writeFileSync(p, JSON.stringify({ t: 'installed', selfWitness: true, node: 'v22.0.0', pid: 42 }) + '\n'); const r = readTrace(p); - assert.deepEqual(r, { state: "usable", rows: [{ t: "installed", selfWitness: true, node: "v22.0.0", pid: 42 }], reason: null, malformed: 0, node: "v22.0.0", pid: 42 }); + assert.deepEqual(r, { state: "usable", rows: [{ t: "installed", selfWitness: true, node: "v22.0.0", pid: 42 }], reason: null, malformed: 0, node: "v22.0.0", pid: 42, network: { state: "unusable", reason: "hook could not witness its own network probe \u2014 no installed row carries a network self-witness", pids: [] } }); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -70,7 +70,7 @@ test('counts malformed lines when a line is not valid JSON', () => { const p = join(dir, 'trace.json'); writeFileSync(p, JSON.stringify({ t: 'installed', selfWitness: true, node: 'v22', pid: 7 }) + '\nnot-json\n'); const r = readTrace(p); - assert.deepEqual(r, { state: "usable", rows: [{ t: "installed", selfWitness: true, node: "v22", pid: 7 }], reason: null, malformed: 1, node: "v22", pid: 7 }); + assert.deepEqual(r, { state: "usable", rows: [{ t: "installed", selfWitness: true, node: "v22", pid: 7 }], reason: null, malformed: 1, node: "v22", pid: 7, network: { state: "unusable", reason: "hook could not witness its own network probe \u2014 no installed row carries a network self-witness", pids: [] } }); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -83,7 +83,7 @@ test('skips empty lines and parses valid JSON lines correctly', () => { const lines = [JSON.stringify({ t: 'installed', selfWitness: true, node: 'v22', pid: 99 }), '', JSON.stringify({ t: 'other' })]; writeFileSync(p, lines.join('\n') + '\n'); const r = readTrace(p); - assert.deepEqual(r, { state: "usable", rows: [{ t: "installed", selfWitness: true, node: "v22", pid: 99 }, { t: "other" }], reason: null, malformed: 0, node: "v22", pid: 99 }); + assert.deepEqual(r, { state: "usable", rows: [{ t: "installed", selfWitness: true, node: "v22", pid: 99 }, { t: "other" }], reason: null, malformed: 0, node: "v22", pid: 99, network: { state: "unusable", reason: "hook could not witness its own network probe \u2014 no installed row carries a network self-witness", pids: [] } }); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/flow/trace-hook.cjs b/flow/trace-hook.cjs index 6f58c1a93..57d13b0ff 100644 --- a/flow/trace-hook.cjs +++ b/flow/trace-hook.cjs @@ -21,6 +21,11 @@ const fs = require('node:fs'); const fsp = require('node:fs/promises'); const cp = require('node:child_process'); const path = require('node:path'); +const os = require('node:os'); +const net = require('node:net'); +const tls = require('node:tls'); +const http = require('node:http'); +const https = require('node:https'); const OUT = process.env.CW_FLOW_TRACE; const ROOT = process.env.CW_FLOW_ROOT || process.cwd(); @@ -45,14 +50,125 @@ function underRoot(abs) { return null; } +// ── open flags ────────────────────────────────────────────────────────────────────────────────── +// A `touches` edge is a literal whose direction C1 could not determine. The runtime CAN determine +// it, but only from an actual open: a stat or an existsSync says nothing about which way data +// moves, so those rows carry no `access` and give no edge a direction. + +/** fs flags (string or O_* number) -> 'read' | 'write' | 'append' | 'readwrite'. */ +function flagsAccess(flags) { + if (flags === undefined || flags === null) return 'read'; + if (typeof flags === 'number') { + const C = fs.constants; + const mode = flags & 3; + if (mode === C.O_RDWR) return 'readwrite'; + if (mode === C.O_WRONLY) return (flags & C.O_APPEND) ? 'append' : 'write'; + return 'read'; + } + const f = String(flags); + if (f.includes('+')) return 'readwrite'; + if (f.includes('a')) return 'append'; + if (f.includes('w')) return 'write'; + return 'read'; +} + +const optFlag = (o, key) => (o && typeof o === 'object' ? o[key] : undefined); + +// via -> how to read the access off the call's arguments. Entries absent here are metadata. +const OPENS = { + readFileSync: (a) => flagsAccess(optFlag(a[1], 'flag') ?? 'r'), + readFile: (a) => flagsAccess(optFlag(a[1], 'flag') ?? 'r'), + createReadStream: (a) => flagsAccess(optFlag(a[1], 'flags') ?? 'r'), + writeFileSync: (a) => flagsAccess(optFlag(a[2], 'flag') ?? 'w'), + writeFile: (a) => flagsAccess(optFlag(a[2], 'flag') ?? 'w'), + appendFileSync: (a) => flagsAccess(optFlag(a[2], 'flag') ?? 'a'), + appendFile: (a) => flagsAccess(optFlag(a[2], 'flag') ?? 'a'), + createWriteStream: (a) => flagsAccess(optFlag(a[1], 'flags') ?? 'w'), + openSync: (a) => flagsAccess(typeof a[1] === 'function' ? 'r' : a[1]), + open: (a) => flagsAccess(typeof a[1] === 'function' ? 'r' : a[1]), +}; + +// ── network keys ──────────────────────────────────────────────────────────────────────────────── +// The key form is the contract with C1: `<scheme>://<host>[:<port>]` for an outbound connection, +// `listen:<port>` for a server. A scheme's default port is omitted, as a URL literal omits it. +const DEFAULT_PORT = { http: 80, https: 443, ws: 80, wss: 443 }; + +function endpointKey(scheme, host, port) { + const h = String(host || 'localhost').toLowerCase(); + const p = port === undefined || port === null || port === '' ? null : Number(port); + const bracketed = h.includes(':') && !h.startsWith('[') ? `[${h}]` : h; + if (p === null || Number.isNaN(p) || DEFAULT_PORT[scheme] === p) return `${scheme}://${bracketed}`; + return `${scheme}://${bracketed}:${p}`; +} + +function urlKey(u) { + const url = u instanceof URL ? u : new URL(String(u)); + const scheme = url.protocol.replace(/:$/, ''); + if (!/^(https?|wss?)$/.test(scheme)) return null; // data:, blob:, file: are not the network + return endpointKey(scheme, url.hostname.replace(/^\[|\]$/g, ''), url.port || null); +} + +/** http(s).request / .get arguments -> key. (url[, options][, cb]) or (options[, cb]). */ +function httpKey(args, defScheme) { + let base = null; + let opts = null; + if (typeof args[0] === 'string' || args[0] instanceof URL) { + base = new URL(String(args[0])); + if (args[1] && typeof args[1] === 'object') opts = args[1]; + } else if (args[0] && typeof args[0] === 'object') { + opts = args[0]; + } + const o = opts || {}; + if (o.socketPath) return `unix://${o.socketPath}`; + const scheme = String(o.protocol || (base && base.protocol) || `${defScheme}:`).replace(/:$/, ''); + const host = o.hostname || o.host || (base && base.hostname.replace(/^\[|\]$/g, '')) || 'localhost'; + const port = o.port ?? (base && base.port) ?? o.defaultPort ?? null; + return endpointKey(scheme, host, port || null); +} + +/** net.connect / tls.connect / Socket#connect arguments -> key. */ +function socketKey(args, scheme) { + const a = Array.isArray(args[0]) ? args[0] : args; // net's internal normalised-args form + const first = a[0]; + if (first && typeof first === 'object') { + if (first.path) return `unix://${first.path}`; + return endpointKey(scheme, first.host || first.servername || 'localhost', first.port ?? null); + } + if (typeof first === 'string' && !/^\d+$/.test(first)) return `unix://${first}`; + return endpointKey(scheme, typeof a[1] === 'string' ? a[1] : 'localhost', first ?? null); +} + +/** Server#listen arguments -> { key, ephemeral }. Port 0 / omitted is decided by the kernel. */ +function listenKey(args) { + const first = args[0]; + let port; + if (first && typeof first === 'object') { + if (first.path) return { key: `listen:unix:${first.path}`, ephemeral: false }; + if (first.port === undefined && (first.fd !== undefined || first._handle || first.handle)) return { key: 'listen:handle', ephemeral: false }; + port = first.port; + } else if (typeof first === 'string' && !/^\d+$/.test(first)) { + return { key: `listen:unix:${first}`, ephemeral: false }; + } else { + port = typeof first === 'function' ? undefined : first; + } + const n = port === undefined || port === null ? 0 : Number(port); + return { key: n ? `listen:${n}` : null, ephemeral: !n }; +} + +module.exports = { flagsAccess, endpointKey, urlKey, httpKey, socketKey, listenKey, OPENS }; + if (OUT) install(); function install() { const appendFileSync = fs.appendFileSync; // captured BEFORE patching, or the hook traces itself const writeFileSync = fs.writeFileSync; const mkdirSync = fs.mkdirSync; + const unlinkSync = fs.unlinkSync; let busy = false; let emitted = 0; + let probing = false; + let depth = 0; // network calls nest: http.request -> net -> Socket#connect + const wrappers = new WeakSet(); try { mkdirSync.call(fs, path.dirname(OUT), { recursive: true }); } catch { /* the append will report it */ } @@ -60,7 +176,8 @@ function install() { if (busy) return; busy = true; emitted += 1; - try { appendFileSync.call(fs, OUT, `${JSON.stringify(row)}\n`); } catch { /* tracing must never break the traced run */ } + const full = probing ? { ...row, pid: process.pid, probe: true } : { ...row, pid: process.pid }; + try { appendFileSync.call(fs, OUT, `${JSON.stringify(full)}\n`); } catch { /* tracing must never break the traced run */ } busy = false; }; @@ -91,13 +208,29 @@ function install() { return underRoot(s) ?? s; }; + const isPathArg = (p) => typeof p === 'string' || p instanceof URL || Buffer.isBuffer(p); + const wrap = (obj, name, t) => { const orig = obj[name]; if (typeof orig !== 'function') return; + const accessOf = OPENS[name]; obj[name] = function (...args) { - emit({ t, path: rel(args[0]), actor: actorOf(), via: name, at: Date.now() }); + if (!isPathArg(args[0])) return orig.apply(this, args); // an fd or a FileHandle is not a path + const actor = actorOf(); + const at = Date.now(); + if (name === 'copyFileSync' || name === 'copyFile' || name === 'cpSync') { + emit({ t: 'read', path: rel(args[0]), actor, via: name, access: 'read', at }); + if (isPathArg(args[1])) emit({ t: 'write', path: rel(args[1]), actor, via: name, access: 'write', at }); + } else if (accessOf) { + let access = 'read'; + try { access = accessOf(args); } catch { /* an odd argument shape is the callee's to reject */ } + emit({ t: access === 'read' ? 'read' : 'write', path: rel(args[0]), actor, via: name, access, at }); + } else { + emit({ t, path: rel(args[0]), actor, via: name, at }); + } return orig.apply(this, args); }; + wrappers.add(obj[name]); }; const READS = ['readFileSync', 'readFile', 'createReadStream', 'readdirSync', 'readdir', @@ -118,13 +251,132 @@ function install() { }; } + // ── network ────────────────────────────────────────────────────────────────────────────────── + // Same discipline as fs: the builtins are `require`d and patched before any ESM facade exists, so + // `import { request } from 'node:http'` binds the wrapper. Only the OUTERMOST call of a nest is + // recorded — http.request reaches net.createConnection and Socket#connect synchronously, and one + // request is one edge, at the resolution the caller wrote it. + const netWrap = (obj, name, via, keyOf) => { + const orig = obj[name]; + if (typeof orig !== 'function') return; + const w = function (...args) { + if (depth === 0) { + let key = null; + try { key = keyOf(args); } catch { /* unparseable target: the callee will reject it */ } + emit({ t: 'connect', key, actor: actorOf(), via, at: Date.now() }); + } + depth += 1; + try { return orig.apply(this, args); } finally { depth -= 1; } + }; + obj[name] = w; + wrappers.add(w); + }; + + netWrap(net, 'connect', 'net.connect', (a) => socketKey(a, 'tcp')); + netWrap(net, 'createConnection', 'net.createConnection', (a) => socketKey(a, 'tcp')); + netWrap(net.Socket.prototype, 'connect', 'net.Socket.connect', (a) => socketKey(a, 'tcp')); + netWrap(tls, 'connect', 'tls.connect', (a) => socketKey(a, 'tls')); + netWrap(http, 'request', 'http.request', (a) => httpKey(a, 'http')); + netWrap(http, 'get', 'http.get', (a) => httpKey(a, 'http')); + netWrap(https, 'request', 'https.request', (a) => httpKey(a, 'https')); + netWrap(https, 'get', 'https.get', (a) => httpKey(a, 'https')); + + const fetchDesc = Object.getOwnPropertyDescriptor(globalThis, 'fetch'); + if (fetchDesc && typeof fetchDesc.value === 'function') { + const orig = fetchDesc.value; + const fetch = function fetch(...args) { + let key = null; + try { + const input = args[0]; + key = urlKey(input && typeof input === 'object' && !(input instanceof URL) && 'url' in input ? input.url : input); + } catch { /* fetch rejects an unparseable URL itself */ } + emit({ t: 'connect', key, actor: actorOf(), via: 'fetch', at: Date.now() }); + return Reflect.apply(orig, this, args); + }; + Object.defineProperty(fetch, 'length', { value: orig.length }); + Object.defineProperty(globalThis, 'fetch', { ...fetchDesc, value: fetch }); + wrappers.add(fetch); + } + + const origListen = net.Server.prototype.listen; + net.Server.prototype.listen = function listen(...args) { + let k = { key: null, ephemeral: false }; + try { k = listenKey(args); } catch { /* the server rejects its own arguments */ } + const actor = actorOf(); + if (k.key) { + emit({ t: 'listen', key: k.key, actor, via: 'listen', at: Date.now() }); + } else if (k.ephemeral) { + // The kernel picks the port; it is known only once bound, and the actor only now. + const self = this; + this.once('listening', () => { + const addr = self.address(); + const key = addr && typeof addr === 'object' ? `listen:${addr.port}` : null; + emit({ t: 'listen', key, actor, via: 'listen', ephemeral: true, at: Date.now() }); + }); + } + return origListen.apply(this, args); + }; + wrappers.add(net.Server.prototype.listen); + // Non-vacuity, written by the hook itself: a trace with no `installed` row means the preload never // ran, which flow/runtime.mjs reports as UNUSABLE rather than as a clean, edgeless run. // The probe is written with the ORIGINAL writeFileSync (untraced) and read back through the // PATCHED readFileSync. selfWitness is true only if that read produced a row — i.e. the patch is // actually on the path, not merely assigned. + probing = true; try { writeFileSync.call(fs, `${OUT}.probe`, 'probe\n'); } catch { /* probe is best effort */ } const before = emitted; try { fs.readFileSync(`${OUT}.probe`, 'utf8'); } catch { /* counted either way */ } - emit({ t: 'installed', pid: process.pid, root: ROOT, node: process.version, selfWitness: emitted > before, at: Date.now() }); + const selfWitness = emitted > before; + + // The same proof for the network, against a local listener on a unix socket: a pipe binds + // synchronously, so the probe completes before the traced program's first line. Three calls go + // through the PATCHED entry points — Server#listen, net.connect, http.request — and each must + // produce a row. The remaining wrappers (tls, https, *.get, Socket#connect, fetch) must at least + // be the function the program will call. Without all of it, rows of this process are refused. + const net0 = probeNetwork(); + probing = false; + emit({ t: 'installed', pid: process.pid, root: ROOT, node: process.version, selfWitness, + netWitness: net0.ok, netReason: net0.reason, at: Date.now() }); + + function probeNetwork() { + const missing = [ + ['net.connect', net.connect], ['net.createConnection', net.createConnection], + ['net.Socket.connect', net.Socket.prototype.connect], ['tls.connect', tls.connect], + ['http.request', http.request], ['http.get', http.get], ['https.request', https.request], + ['https.get', https.get], ['net.Server.listen', net.Server.prototype.listen], + ['fetch', globalThis.fetch], + ].filter(([, f]) => typeof f === 'function' && !wrappers.has(f)).map(([n]) => n); + if (missing.length) return { ok: false, reason: `not patched: ${missing.join(', ')}` }; + const sock = process.platform === 'win32' + ? `\\\\?\\pipe\\cw-flow-${process.pid}-${Date.now()}` + : path.join(os.tmpdir(), `cw-flow-${process.pid}-${Date.now()}.sock`); + const seen = []; + let server = null; + try { + server = net.createServer((s) => s.destroy()); + server.on('error', () => {}); + let n = emitted; + server.listen(sock); + seen.push(['listen', emitted > n]); + server.unref(); + n = emitted; + const c = net.connect(sock); + seen.push(['net.connect', emitted > n]); + c.on('error', () => {}); + c.destroy(); + n = emitted; + const req = http.request({ socketPath: sock, path: '/' }); + seen.push(['http.request', emitted > n]); + req.on('error', () => {}); + req.destroy(); + } catch (e) { + return { ok: false, reason: `probe threw: ${e.code || e.message}` }; + } finally { + try { server && server.close(); } catch { /* closing a probe */ } + if (process.platform !== 'win32') { try { unlinkSync.call(fs, sock); } catch { /* close removes it */ } } + } + const blind = seen.filter(([, ok]) => !ok).map(([n]) => n); + return blind.length ? { ok: false, reason: `probe not observed: ${blind.join(', ')}` } : { ok: true, reason: null }; + } } From c6e7da1f4bb3ba62550430e6d5376433dd6f45fc Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:33:51 +1030 Subject: [PATCH 04/36] feat(flow): reconcile endpoints, give touches a direction, resolve bare names --- flow/reconcile.mjs | 205 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 203 insertions(+), 2 deletions(-) diff --git a/flow/reconcile.mjs b/flow/reconcile.mjs index a97975331..3a9f7950e 100644 --- a/flow/reconcile.mjs +++ b/flow/reconcile.mjs @@ -14,8 +14,59 @@ // null and `state` is 'unmeasured'. It is never 0. import { SCHEMA_VERSION, edgeKey } from './graph.mjs'; +import { readJson } from './store.mjs'; const MODULE_PREFIX = 'node:module:'; +const ENDPOINT_PREFIX = 'node:endpoint:'; +const NET_KINDS = new Set(['connects', 'listens']); +const DEFAULT_PORT = { http: 80, https: 443, ws: 80, wss: 443 }; + +/** + * An endpoint key -> the identity two witnesses can agree on: host and effective port, or the + * listening port. C1 reads `https://api.example.com` off a literal; C2 may see the same socket as + * `https://api.example.com` (https.request) or `tls://api.example.com:443` (a client that dials + * TLS itself). Those are one edge at two layers, and the scheme is not the claim — the peer is. + */ +export function endpointIdentity(key) { + const k = String(key); + if (k.startsWith('listen:')) return k; + const m = /^([a-z][a-z0-9+.-]*):\/\/(\[[^\]]+\]|[^/:?#]*)(?::(\d+))?/i.exec(k); + if (!m) return k; + const scheme = m[1].toLowerCase(); + if (scheme === 'unix') return k; + const port = m[3] ? Number(m[3]) : (DEFAULT_PORT[scheme] ?? null); + return `${m[2].toLowerCase()}:${port ?? ''}`; +} + +/** A store key C1 holds as a NAME rather than a location: `rollup.json`, `./x.json`, `../y/z.json`. */ +export function isNameKey(key) { + const k = String(key); + if (k.startsWith('/')) return false; + return !k.includes('/') || k.startsWith('./') || k.startsWith('../'); +} + +/** Does a concrete observed path answer to a name key? */ +export function nameMatches(key, observed) { + const k = String(key); + const t = String(observed); + if (!k.includes('/')) return t === k || t.endsWith(`/${k}`); + const rest = k.replace(/^(?:\.\.?\/)+/, ''); + return rest !== '' && (t === rest || t.endsWith(`/${rest}`)); +} + +/** access values (read | write | append | readwrite) -> 'reads' | 'writes' | 'both' | null */ +export function directionOf(accesses) { + let r = false; + let w = false; + for (const a of accesses) { + if (a === 'read' || a === 'readwrite') r = true; + if (a === 'write' || a === 'append' || a === 'readwrite') w = true; + } + if (r && w) return 'both'; + if (r) return 'reads'; + if (w) return 'writes'; + return null; +} function pathOf(id) { return id.replace(/^node:[a-z]+:/, ''); @@ -46,13 +97,41 @@ export function reconcile(graph, runtime) { unconfirmed: [], confirmed: [], outOfScope: [], - edges: (graph.edges || []).map((e) => ({ ...e, existence: 'unknown' })), + coverage: [], + observedPaths: {}, + network: runtime && runtime.network ? runtime.network : { state: 'unusable', reason: 'no runtime pass supplied' }, + edges: (graph.edges || []).map((e) => ({ ...e, witness: 'static', existence: 'unknown', runtimeCoverage: 'unvisited' })), }; } const coverage = new Set(runtime.coverage || []); const observed = runtimeKeys(runtime.observations || []); const staticKeys = new Set(); + // Absence of a network observation means something only when every traced process proved it can + // see the network. A runtime summary without a `network` block (an older one, or a union that + // dropped it) is treated as unwitnessed: a connect it saw still confirms, a connect it did not + // see refutes nothing. + const netWitnessed = Boolean(runtime.network && runtime.network.state === 'usable'); + const netObserved = new Set(); + const fsByActor = new Map(); + for (const o of runtime.observations || []) { + if (NET_KINDS.has(o.kind)) netObserved.add(JSON.stringify([o.from, endpointIdentity(o.to), o.kind])); + else if (o.kind === 'reads' || o.kind === 'writes') { + if (!fsByActor.has(o.from)) fsByActor.set(o.from, []); + fsByActor.get(o.from).push(o); + } + } + // Every access this actor's opens of `to` carried: exactly, or by name for a name key. + const accessesFor = (actor, to, byName) => { + const exact = []; + const named = []; + for (const o of fsByActor.get(actor) || []) { + if (!o.access) continue; + if (o.to === to) exact.push(...o.access); + else if (byName && nameMatches(to, o.to)) named.push(...o.access); + } + return { exact, named }; + }; const confirmed = []; // C1 claimed it, the run entered that module, the edge was not observed. @@ -68,7 +147,7 @@ export function reconcile(graph, runtime) { const edges = []; for (const e of graph.edges || []) { - if (!e.from.startsWith(MODULE_PREFIX)) { edges.push({ ...e, existence: 'unknown' }); continue; } + if (!e.from.startsWith(MODULE_PREFIX)) { edges.push({ ...e, witness: 'static', existence: 'unknown', runtimeCoverage: 'unvisited' }); continue; } const from = pathOf(e.from); const to = pathOf(e.to); const key = JSON.stringify([from, to, e.kind]); @@ -84,6 +163,34 @@ export function reconcile(graph, runtime) { edges.push({ ...e, witness: 'both', existence: 'confirmed', runtimeCoverage: 'visited' }); continue; } + if (NET_KINDS.has(e.kind) && e.to.startsWith(ENDPOINT_PREFIX)) { + if (netObserved.has(JSON.stringify([from, endpointIdentity(to), e.kind]))) { + confirmed.push({ from, to, kind: e.kind }); + edges.push({ ...e, witness: 'both', existence: 'confirmed', runtimeCoverage: 'visited' }); + continue; + } + if (!netWitnessed) { + edges.push({ ...e, witness: 'static', existence: 'unknown', runtimeCoverage: 'visited', unfalsifiable: true, networkUnwitnessed: true }); + continue; + } + } + // A `touches` edge gets a direction from the opens the runtime saw: read-only, for write, or + // both. An exact-path open also CONFIRMS the edge — the module did open that file. An open + // matched only by name gives the direction and confirms nothing, because the name is several + // files (the composed-node rule). + if (e.kind === 'touches') { + const { exact, named } = accessesFor(from, to, isNameKey(to)); + const direction = directionOf(exact.length ? exact : named); + if (exact.length) { + confirmed.push({ from, to, kind: e.kind, direction }); + edges.push({ ...e, witness: 'both', existence: 'confirmed', runtimeCoverage: 'visited', direction }); + continue; + } + if (direction) { + edges.push({ ...e, witness: 'static', existence: 'unknown', runtimeCoverage: 'visited', unfalsifiable: true, direction, directionBy: 'name' }); + continue; + } + } // A `touches` edge never claimed a direction, so nothing about it can be contradicted. Same for // a module edge the loader satisfies from cache, for a dynamic import that did not fire, and — // the one that matters most — for an ENV edge: C2 patches fs and child_process, not @@ -111,6 +218,7 @@ export function reconcile(graph, runtime) { const literalIndex = new Map( (graph.modules || []).filter((m) => m.pathLiterals).map((m) => [m.path, m.pathLiterals]), ); + const moduleIndex = new Map((graph.modules || []).map((m) => [m.path, m])); const couldHaveSeen = (actor, target) => { const lits = literalIndex.get(actor); if (!lits || !lits.length) return false; @@ -139,14 +247,52 @@ export function reconcile(graph, runtime) { const falseNegatives = []; const dynamicPaths = []; const matchedByName = []; + const staticNet = new Set(); + for (const e of graph.edges || []) { + if (e.from.startsWith(MODULE_PREFIX) && NET_KINDS.has(e.kind)) staticNet.add(JSON.stringify([pathOf(e.from), endpointIdentity(pathOf(e.to)), e.kind])); + } + // An endpoint is within C1's reach when the module carries a literal naming its host or port. + // An ephemeral listen (port 0) and a host taken from config are never literals. + const couldHaveSeenEndpoint = (actor, key, o) => { + if (o.ephemeral) return false; + const lits = [...(literalIndex.get(actor) || []), ...((moduleIndex.get(actor) || {}).urlLiterals || [])]; + const id = endpointIdentity(key); + const host = id.startsWith('listen:') ? null : id.slice(0, id.lastIndexOf(':')); + const port = id.slice(id.lastIndexOf(':') + 1); + return lits.some((l) => (host && host !== 'localhost' && String(l).toLowerCase().includes(host)) + || (id.startsWith('listen:') && String(l) === port)); + }; for (const o of runtime.observations || []) { if (!coverage.has(o.from)) continue; // an actor outside the analysed set + if (NET_KINDS.has(o.kind)) { + if (staticNet.has(JSON.stringify([o.from, endpointIdentity(o.to), o.kind]))) continue; + const row = { from: o.from, to: o.to, kind: o.kind, via: o.via }; + (couldHaveSeenEndpoint(o.from, o.to, o) ? falseNegatives : dynamicPaths).push(row); + continue; + } if (staticKeys.has(JSON.stringify([o.from, o.to, o.kind]))) continue; const row = { from: o.from, to: o.to, kind: o.kind, via: o.via }; if (matchesByName(o.from, o.to, o.kind)) { matchedByName.push(row); continue; } (couldHaveSeen(o.from, o.to) ? falseNegatives : dynamicPaths).push(row); } + // Where each NAME C1 holds was actually found. `rollup.json` is several files; the run says which + // ones this module opened. Only actors that C1 says use the name are asked, so a second module + // opening an unrelated `config.json` does not resolve this one's. + const observedPaths = {}; + for (const e of graph.edges || []) { + if (!e.from.startsWith(MODULE_PREFIX) || !/^node:(artifact|store):/.test(e.to)) continue; + const key = pathOf(e.to); + if (!isNameKey(key)) continue; + const actor = pathOf(e.from); + for (const o of fsByActor.get(actor) || []) { + if (!nameMatches(key, o.to)) continue; + const set = observedPaths[key] || (observedPaths[key] = []); + if (!set.includes(o.to)) set.push(o.to); + } + } + for (const k of Object.keys(observedPaths)) observedPaths[k].sort(); + const comparable = confirmed.length + unconfirmed.length + falseNegatives.length; return { ...base, @@ -155,6 +301,10 @@ export function reconcile(graph, runtime) { divergence: comparable === 0 ? null : falseNegatives.length + unconfirmed.length, comparable, coverageModules: coverage.size, + coverage: [...coverage].sort(), + network: { ...(runtime.network || { state: 'unusable', reason: 'runtime summary carries no network witness' }), witnessed: netWitnessed }, + directed: edges.filter((e) => e.direction).length, + observedPaths, falseNegatives, unconfirmed, dynamicPaths, // observed, and outside C1's reach BY CONSTRUCTION — not a finding @@ -177,7 +327,58 @@ export function formatReport(r) { ` dynamic paths (never a literal, NOT a finding) ${(r.dynamicPaths || []).length}`, ` matched by name only (bare literal vs resolved path) ${(r.matchedByName || []).length}`, ` runtime coverage ${r.coverageModules || 0} modules`, + ` touches given a direction by observed opens ${r.directed || 0}`, + ` bare-name stores resolved to observed paths ${Object.keys(r.observedPaths || {}).length}`, + ` network ${r.network ? r.network.state : 'unusable'}${r.network && r.network.reason ? ` (${r.network.reason})` : ''}`, ].join('\n'); } +const WITNESS = new Set(['static', 'runtime', 'both']); +const EXISTENCE = new Set(['confirmed', 'contradicted', 'unknown']); +const COVERAGE = new Set(['visited', 'unvisited']); +const DIRECTION = new Set(['reads', 'writes', 'both']); + +function invalid(path, why) { + const err = new Error(`flow-reconcile.json invalid ${path}: ${why}`); + err.code = 'CW_FLOW_RECONCILE_INVALID'; + return err; +} + +/** + * Read and validate a `flow-reconcile.json`. -> the report, with `edges`, `observedPaths` and + * `coverage` checked. Throws — it never returns a partial or defaulted report: a reader that + * fills a missing `edges` with `[]` has turned an unreadable reconciliation into an empty one. + * CW_FLOW_RECONCILE_ABSENT no file + * CW_FLOW_RECONCILE_INVALID wrong `v`, or a record of the wrong shape + * CW_FLOW_STORE_CORRUPT / _UNREADABLE from the store reader + */ +export function readReconcile(path) { + const { state, data } = readJson(path); + if (state === 'absent') { + const err = new Error(`flow-reconcile.json absent ${path}`); + err.code = 'CW_FLOW_RECONCILE_ABSENT'; + throw err; + } + if (!data || typeof data !== 'object' || Array.isArray(data)) throw invalid(path, 'not an object'); + if (data.v !== SCHEMA_VERSION) throw invalid(path, `v is ${JSON.stringify(data.v)}, expected ${SCHEMA_VERSION}`); + if (!Array.isArray(data.edges)) throw invalid(path, 'edges is not an array'); + data.edges.forEach((e, i) => { + const at = `edges[${i}]`; + if (!e || typeof e !== 'object') throw invalid(path, `${at} not an object`); + if (e.v !== SCHEMA_VERSION) throw invalid(path, `${at}.v is ${JSON.stringify(e.v)}`); + for (const f of ['from', 'to', 'kind']) if (typeof e[f] !== 'string' || !e[f]) throw invalid(path, `${at}.${f} missing`); + if (!WITNESS.has(e.witness)) throw invalid(path, `${at}.witness ${JSON.stringify(e.witness)}`); + if (!EXISTENCE.has(e.existence)) throw invalid(path, `${at}.existence ${JSON.stringify(e.existence)}`); + if (!COVERAGE.has(e.runtimeCoverage)) throw invalid(path, `${at}.runtimeCoverage ${JSON.stringify(e.runtimeCoverage)}`); + if (e.direction !== undefined && !DIRECTION.has(e.direction)) throw invalid(path, `${at}.direction ${JSON.stringify(e.direction)}`); + }); + const op = data.observedPaths; + if (!op || typeof op !== 'object' || Array.isArray(op)) throw invalid(path, 'observedPaths is not an object'); + for (const [k, v] of Object.entries(op)) { + if (!Array.isArray(v) || v.some((x) => typeof x !== 'string')) throw invalid(path, `observedPaths[${JSON.stringify(k)}] is not a list of paths`); + } + if (!Array.isArray(data.coverage) || data.coverage.some((x) => typeof x !== 'string')) throw invalid(path, 'coverage is not a list of modules'); + return data; +} + export { edgeKey }; From cd019e573739a27199a161327a2eaf0a404efd80 Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:35:39 +1030 Subject: [PATCH 05/36] test(flow): network observation, self-witness, and open access in C2 --- flow/runtime.mjs | 1 + flow/test/runtime-access.test.mjs | 106 +++++++++++++++++ flow/test/runtime-network.test.mjs | 145 ++++++++++++++++++++++++ flow/test/unit-trace-hook-keys.test.mjs | 100 ++++++++++++++++ 4 files changed, 352 insertions(+) create mode 100644 flow/test/runtime-access.test.mjs create mode 100644 flow/test/runtime-network.test.mjs create mode 100644 flow/test/unit-trace-hook-keys.test.mjs diff --git a/flow/runtime.mjs b/flow/runtime.mjs index 36bd25423..71389c49d 100644 --- a/flow/runtime.mjs +++ b/flow/runtime.mjs @@ -137,6 +137,7 @@ export function summarise(traceRead) { const prev = by.get(k); const { access, ...rest } = o; if (!prev) { by.set(k, { ...rest, ...(access ? { access: [access] } : {}) }); continue; } + if (prev.ephemeral && !rest.ephemeral) delete prev.ephemeral; // named once is named: a literal could match if (access && !(prev.access || []).includes(access)) prev.access = [...(prev.access || []), access].sort(); } const deduped = [...by.values()]; diff --git a/flow/test/runtime-access.test.mjs b/flow/test/runtime-access.test.mjs new file mode 100644 index 000000000..810e8bc08 --- /dev/null +++ b/flow/test/runtime-access.test.mjs @@ -0,0 +1,106 @@ +// C2 records how each file was OPENED — read, write, append, both — because that is what gives a +// direction to a `touches` edge C1 could not orient. Metadata calls carry no access. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { observe, summarise, networkWitness } from '../runtime.mjs'; + +function fixtureRepo() { + const d = mkdtempSync(join(tmpdir(), 'flow-acc-')); + mkdirSync(join(d, 'bin'), { recursive: true }); + mkdirSync(join(d, 'data'), { recursive: true }); + writeFileSync(join(d, 'data', 'in.txt'), 'x\n'); + return d; +} + +test('every open form records its access, named imports included', async () => { + // Arrange + const d = fixtureRepo(); + try { + writeFileSync(join(d, 'bin', 'io.mjs'), [ + 'import { openSync, closeSync, readFileSync, writeFileSync, appendFileSync, copyFileSync, existsSync, createWriteStream, createReadStream, readFile } from "node:fs";', + 'import { open, writeFile } from "node:fs/promises";', + 'closeSync(openSync("data/in.txt", "r"));', + 'closeSync(openSync("data/rw.txt", "w+"));', + 'readFileSync("data/both.txt", { flag: "a+" });', + 'writeFileSync("data/both.txt", "1");', + 'readFileSync("data/both.txt");', + 'appendFileSync("data/log.txt", "1");', + 'copyFileSync("data/in.txt", "data/copy.txt");', + 'existsSync("data/meta.txt");', + 'const fd = openSync("data/in.txt", "r"); readFileSync(fd); closeSync(fd);', + 'await (await open("data/p.txt", "a")).close();', + 'await writeFile("data/pw.txt", "1");', + 'await new Promise((ok) => createWriteStream("data/ws.txt").end("x", ok));', + 'await new Promise((ok) => createReadStream("data/in.txt").on("close", ok).resume());', + 'await new Promise((ok) => readFile("data/in.txt", ok));', + ].join('\n')); + // Act + const r = observe(['bin/io.mjs'], { root: d }); + // Assert + assert.equal(r.status, 0, r.stderr); + const acc = (to) => r.observations.filter((o) => o.from === 'bin/io.mjs' && o.to === to && o.access).flatMap((o) => o.access).sort(); + assert.deepEqual(acc('data/in.txt'), ['read']); + assert.deepEqual(acc('data/rw.txt'), ['readwrite']); + assert.deepEqual(acc('data/both.txt'), ['read', 'readwrite', 'write'], 'access accumulates over the run'); + assert.deepEqual(acc('data/log.txt'), ['append']); + assert.deepEqual(acc('data/copy.txt'), ['write'], 'copyFile opens its destination for write'); + assert.deepEqual(acc('data/p.txt'), ['append']); + assert.deepEqual(acc('data/pw.txt'), ['write']); + assert.deepEqual(acc('data/ws.txt'), ['write']); + assert.deepEqual(acc('data/meta.txt'), [], 'existsSync is metadata, not an open'); + assert.ok(r.observations.some((o) => o.to === 'data/meta.txt' && o.kind === 'reads'), 'but it is still observed'); + assert.ok(!r.observations.some((o) => /\/\d+$|^\d+$/.test(o.to)), 'an fd is not a path'); + } finally { rmSync(d, { recursive: true, force: true }); } +}); + +test('summarise refuses network rows from a process without a network witness, per pid', () => { + // Arrange + const rows = [ + { t: 'installed', pid: 1, selfWitness: true, netWitness: true }, + { t: 'installed', pid: 2, selfWitness: true, netWitness: false, netReason: 'probe not observed: listen' }, + { t: 'connect', key: 'tcp://a:1', actor: 'bin/a.mjs', via: 'net.connect', pid: 1 }, + { t: 'connect', key: 'tcp://b:1', actor: 'bin/b.mjs', via: 'net.connect', pid: 2 }, + { t: 'connect', key: null, actor: 'bin/a.mjs', via: 'fetch', pid: 1 }, + { t: 'listen', key: 'listen:1', actor: 'bin/a.mjs', via: 'listen', pid: 1, probe: true }, + ]; + // Act + const s = summarise({ state: 'usable', rows, network: networkWitness(rows) }); + // Assert + assert.equal(s.network.state, 'usable'); + assert.equal(s.network.refused, 1); + assert.equal(s.network.observed, 1); + assert.equal(s.unattributed, 1, 'an unkeyed connect is unattributed, not an edge'); + assert.deepEqual(s.observations, [{ from: 'bin/a.mjs', to: 'tcp://a:1', kind: 'connects', via: 'net.connect' }]); + assert.ok(!s.coverage.includes('bin/b.mjs'), 'a refused row is not evidence either'); +}); + +test('networkWitness says why when no process proved itself', () => { + // Arrange / Act + const none = networkWitness([]); + const failed = networkWitness([{ t: 'installed', pid: 3, netWitness: false, netReason: 'not patched: fetch' }]); + // Assert + assert.equal(none.state, 'unusable'); + assert.match(none.reason, /no installed row/); + assert.match(failed.reason, /not patched: fetch/); + assert.deepEqual(failed.pids, []); +}); + +test('summarise derives the network witness itself when readTrace did not supply one', () => { + // Arrange + const rows = [{ t: 'installed', pid: 9, netWitness: true }, { t: 'listen', key: 'listen:5', actor: 'bin/s.mjs', via: 'listen', pid: 9, ephemeral: true }]; + // Act + const s = summarise({ state: 'usable', rows }); + // Assert + assert.equal(s.network.state, 'usable'); + assert.deepEqual(s.observations, [{ from: 'bin/s.mjs', to: 'listen:5', kind: 'listens', via: 'listen', ephemeral: true }]); +}); + +test('an unusable trace summarises with an unusable network, not an empty one', () => { + // Arrange / Act + const s = summarise({ state: 'absent', rows: [], reason: 'no trace file' }); + // Assert + assert.deepEqual(s.network, { state: 'unusable', reason: 'no trace file', observed: 0, refused: 0 }); +}); diff --git a/flow/test/runtime-network.test.mjs b/flow/test/runtime-network.test.mjs new file mode 100644 index 000000000..0a8fe1578 --- /dev/null +++ b/flow/test/runtime-network.test.mjs @@ -0,0 +1,145 @@ +// C2 on the network: connects and listens are observed through every import form, keyed the way C1 +// keys an endpoint, and refused when the hook cannot prove it saw its own probe. No test here +// leaves 127.0.0.1 or a unix socket. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { hookPath, observe, readTrace, summarise } from '../runtime.mjs'; + +function fixtureRepo() { + const d = mkdtempSync(join(tmpdir(), 'flow-net-')); + mkdirSync(join(d, 'bin'), { recursive: true }); + return d; +} + +// A local server answering HTTP, then one call per client entry point. Each import form is the one +// that matters for that builtin: NAMED for http (the form an --import hook misses), namespace for +// net, default for https and tls. +const CLIENT = ` +import { request, get } from 'node:http'; +import * as net from 'node:net'; +import https from 'node:https'; +import tls from 'node:tls'; +const quiet = (x) => { x.on('error', () => {}); return x; }; +const s = net.createServer((c) => { c.on('error', () => {}); c.end('HTTP/1.1 200 OK\\r\\nContent-Length: 0\\r\\nConnection: close\\r\\n\\r\\n'); }); +s.listen(0, '127.0.0.1', async () => { + const port = s.address().port; + await new Promise((ok) => quiet(request({ host: '127.0.0.1', port, path: '/' }, (r) => { r.resume(); r.on('end', ok); })).end()); + await new Promise((ok) => quiet(get('http://127.0.0.1:' + port + '/x', (r) => { r.resume(); r.on('end', ok); }))); + await fetch('http://127.0.0.1:' + port + '/f').then((r) => r.text()); + quiet(https.request({ host: '127.0.0.1', port, path: '/' })).destroy(); + quiet(tls.connect({ host: '127.0.0.1', port, rejectUnauthorized: false })).destroy(); + quiet(net.connect(port, '127.0.0.1')).destroy(); + quiet(new net.Socket().connect(port, '127.0.0.1')).destroy(); + s.close(); + console.log(port); +}); +`; + +test('every client entry point is observed under its import form, keyed as C1 keys an endpoint', () => { + // Arrange + const d = fixtureRepo(); + try { + writeFileSync(join(d, 'bin', 'client.mjs'), CLIENT); + // Act + const r = observe(['bin/client.mjs'], { root: d }); + // Assert + assert.equal(r.status, 0, r.stderr); + const port = Number(r.stdout ? r.stdout.trim() : NaN) || Number(/listen:(\d+)/.exec(JSON.stringify(r.observations))[1]); + const seen = (to, via) => r.observations.some((o) => o.from === 'bin/client.mjs' && o.kind === 'connects' && o.to === to && o.via === via); + assert.equal(r.network.state, 'usable'); + assert.ok(seen(`http://127.0.0.1:${port}`, 'http.request'), 'named import { request }'); + assert.ok(r.observations.some((o) => o.to === `http://127.0.0.1:${port}` && o.kind === 'connects'), 'get and fetch share the key'); + assert.ok(seen(`https://127.0.0.1:${port}`, 'https.request')); + assert.ok(seen(`tls://127.0.0.1:${port}`, 'tls.connect')); + assert.ok(seen(`tcp://127.0.0.1:${port}`, 'net.connect')); + assert.ok(r.observations.some((o) => o.kind === 'listens' && o.to === `listen:${port}` && o.ephemeral === true), + 'a port-0 listen is recorded at the port the kernel chose, and marked ephemeral'); + } finally { rmSync(d, { recursive: true, force: true }); } +}); + +test('fetch and http.get are each observed, and a nested call is one edge, not three', () => { + // Arrange + const d = fixtureRepo(); + try { + writeFileSync(join(d, 'bin', 'client.mjs'), CLIENT); + // Act + const r = observe(['bin/client.mjs'], { root: d, keep: true }); + const rows = readTrace(r.tracePath).rows.filter((x) => !x.probe && x.actor === 'bin/client.mjs' && x.t === 'connect'); + rmSync(r.tracePath, { force: true }); + // Assert + const vias = rows.map((x) => x.via).sort(); + assert.deepEqual(vias, ['fetch', 'http.get', 'http.request', 'https.request', 'net.Socket.connect', 'net.connect', 'tls.connect'], + 'http.request reaches net.createConnection and Socket#connect synchronously; only the outer call is a row'); + } finally { rmSync(d, { recursive: true, force: true }); } +}); + +test('the hook witnesses its own network probe, and marks the probe rows so they are never edges', () => { + // Arrange + const d = fixtureRepo(); + try { + writeFileSync(join(d, 'bin', 'noop.mjs'), 'export const a = 1;\n'); + // Act + const r = observe(['bin/noop.mjs'], { root: d, keep: true }); + const t = readTrace(r.tracePath); + rmSync(r.tracePath, { force: true }); + // Assert + const installed = t.rows.find((x) => x.t === 'installed'); + assert.equal(installed.netWitness, true); + assert.equal(installed.netReason, null); + assert.deepEqual(t.rows.filter((x) => x.probe && x.t !== 'read').map((x) => x.via), ['listen', 'net.connect', 'http.request']); + assert.equal(r.observations.length, 0, 'probe rows are not observations'); + assert.equal(r.network.observed, 0); + } finally { rmSync(d, { recursive: true, force: true }); } +}); + +test('NEGATIVE CONTROL — a patch that cannot take leaves the network UNWITNESSED, and its rows are refused', () => { + // Arrange: a preload ahead of the hook freezes net.connect, so the hook's assignment is a silent + // no-op — the inert-patch shape, produced for real rather than hand-written into a trace. + const d = fixtureRepo(); + try { + writeFileSync(join(d, 'freeze.cjs'), + "const net = require('node:net'); Object.defineProperty(net, 'connect', { value: net.connect, writable: false, configurable: false });\n"); + writeFileSync(join(d, 'bin', 'c.mjs'), + "import * as net from 'node:net';\nconst s = net.createServer().listen(0, '127.0.0.1', () => { const c = net.createConnection(s.address().port, '127.0.0.1'); c.on('error', () => {}); c.destroy(); s.close(); });\n"); + const trace = join(d, 'trace.jsonl'); + // Act + spawnSync(process.execPath, ['-r', join(d, 'freeze.cjs'), '-r', hookPath(), 'bin/c.mjs'], + { cwd: d, encoding: 'utf8', env: { ...process.env, CW_FLOW_TRACE: trace, CW_FLOW_ROOT: d } }); + const read = readTrace(trace); + const s = summarise(read); + // Assert + assert.equal(read.state, 'usable', 'fs is still witnessed'); + assert.equal(read.rows.find((x) => x.t === 'installed').netWitness, false); + assert.match(read.network.reason, /not patched: net\.connect/); + assert.equal(s.network.state, 'unusable'); + assert.ok(s.network.refused >= 2, 'the listen and the connect the hook did see are refused, not reported'); + assert.ok(!s.observations.some((o) => o.kind === 'connects' || o.kind === 'listens')); + } finally { rmSync(d, { recursive: true, force: true }); } +}); + +test('an explicit listen port is recorded at once, as listen:<port>, not ephemeral', () => { + // Arrange: a port the kernel just freed — the only way to name a free port without guessing. + const d = fixtureRepo(); + try { + writeFileSync(join(d, 'bin', 'srv.mjs'), + "import http from 'node:http';\nimport net from 'node:net';\n" + + "const p = net.createServer().listen(0, '127.0.0.1', function () { const port = this.address().port; this.close(() => {\n" + + " const s = http.createServer(); s.on('error', () => {}); s.listen({ port, host: '127.0.0.1' }, () => s.close()); console.log(port); }); });\n"); + // Act + const r = observe(['bin/srv.mjs'], { root: d, keep: true }); + const rows = readTrace(r.tracePath).rows.filter((x) => !x.probe && x.t === 'listen'); + rmSync(r.tracePath, { force: true }); + // Assert + assert.equal(rows.length, 2); + assert.equal(rows[0].ephemeral, true, 'the kernel chose the first port'); + assert.equal(rows[1].ephemeral, undefined, 'the second was named'); + assert.equal(rows[0].key, rows[1].key); + const listens = r.observations.filter((o) => o.kind === 'listens'); + assert.equal(listens.length, 1, 'one from|to|kind is one observation'); + assert.equal(listens[0].ephemeral, undefined, 'named once is named: a literal could have matched it'); + } finally { rmSync(d, { recursive: true, force: true }); } +}); diff --git a/flow/test/unit-trace-hook-keys.test.mjs b/flow/test/unit-trace-hook-keys.test.mjs new file mode 100644 index 000000000..8115eaa7c --- /dev/null +++ b/flow/test/unit-trace-hook-keys.test.mjs @@ -0,0 +1,100 @@ +// The hook's pure parts: how an open's flags become an access, and how a call's arguments become +// C1's endpoint key. Loaded with CW_FLOW_TRACE unset, so nothing is patched in this process. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { constants as C } from 'node:fs'; +import { hookPath } from '../runtime.mjs'; + +const saved = process.env.CW_FLOW_TRACE; +delete process.env.CW_FLOW_TRACE; +const hook = createRequire(import.meta.url)(hookPath()); +if (saved !== undefined) process.env.CW_FLOW_TRACE = saved; + +test('flagsAccess reads string flags', () => { + // Arrange + const cases = { r: 'read', rs: 'read', 'r+': 'readwrite', w: 'write', wx: 'write', 'w+': 'readwrite', a: 'append', ax: 'append', 'a+': 'readwrite' }; + // Act / Assert + for (const [f, want] of Object.entries(cases)) assert.equal(hook.flagsAccess(f), want, f); +}); + +test('flagsAccess reads O_* numbers and defaults to read', () => { + // Arrange / Act / Assert + assert.equal(hook.flagsAccess(C.O_RDONLY), 'read'); + assert.equal(hook.flagsAccess(C.O_WRONLY | C.O_CREAT), 'write'); + assert.equal(hook.flagsAccess(C.O_WRONLY | C.O_APPEND), 'append'); + assert.equal(hook.flagsAccess(C.O_RDWR), 'readwrite'); + assert.equal(hook.flagsAccess(undefined), 'read'); + assert.equal(hook.flagsAccess(null), 'read'); +}); + +test('OPENS takes each call\'s own default and its flag option', () => { + // Arrange + const O = hook.OPENS; + // Act / Assert + assert.equal(O.readFileSync(['p']), 'read'); + assert.equal(O.readFile(['p', { flag: 'r+' }]), 'readwrite'); + assert.equal(O.createReadStream(['p']), 'read'); + assert.equal(O.writeFileSync(['p', 'x']), 'write'); + assert.equal(O.writeFile(['p', 'x', { flag: 'a' }]), 'append'); + assert.equal(O.appendFileSync(['p', 'x']), 'append'); + assert.equal(O.appendFile(['p', 'x', 'utf8']), 'append', 'a string option is an encoding, not a flag'); + assert.equal(O.createWriteStream(['p', { flags: 'a' }]), 'append'); + assert.equal(O.openSync(['p', 'w']), 'write'); + assert.equal(O.open(['p', () => {}]), 'read', 'open(path, cb) defaults to r'); +}); + +test('endpointKey omits a scheme\'s default port, keeps any other, brackets IPv6', () => { + // Arrange / Act / Assert + assert.equal(hook.endpointKey('https', 'API.example.com', 443), 'https://api.example.com'); + assert.equal(hook.endpointKey('http', 'h', 8080), 'http://h:8080'); + assert.equal(hook.endpointKey('tcp', 'db', '5432'), 'tcp://db:5432'); + assert.equal(hook.endpointKey('tcp', undefined, null), 'tcp://localhost'); + assert.equal(hook.endpointKey('http', '::1', 81), 'http://[::1]:81'); + assert.equal(hook.endpointKey('http', 'h', 'nan'), 'http://h'); +}); + +test('urlKey takes http(s) and ws(s) and declines everything else', () => { + // Arrange / Act / Assert + assert.equal(hook.urlKey('https://x.test/a?b'), 'https://x.test'); + assert.equal(hook.urlKey(new URL('http://x.test:81/')), 'http://x.test:81'); + assert.equal(hook.urlKey('wss://x.test'), 'wss://x.test'); + assert.equal(hook.urlKey('http://[::1]:9/'), 'http://[::1]:9'); + assert.equal(hook.urlKey('data:text/plain,hi'), null); + assert.throws(() => hook.urlKey('/relative')); +}); + +test('httpKey merges a URL with an options object, and options alone', () => { + // Arrange / Act / Assert + assert.equal(hook.httpKey(['http://a.test/x'], 'http'), 'http://a.test'); + assert.equal(hook.httpKey([new URL('https://a.test:8443/')], 'https'), 'https://a.test:8443'); + assert.equal(hook.httpKey(['http://a.test/x', { port: 9 }], 'http'), 'http://a.test:9'); + assert.equal(hook.httpKey([{ hostname: 'b.test', port: 443 }], 'https'), 'https://b.test'); + assert.equal(hook.httpKey([{ host: 'b.test', protocol: 'http:', defaultPort: 8000 }], 'https'), 'http://b.test:8000'); + assert.equal(hook.httpKey([{ socketPath: '/s.sock' }], 'http'), 'unix:///s.sock'); + assert.equal(hook.httpKey([() => {}], 'http'), 'http://localhost'); +}); + +test('socketKey reads (port, host), (options), (path), and net\'s normalised array', () => { + // Arrange / Act / Assert + assert.equal(hook.socketKey([5432, 'db'], 'tcp'), 'tcp://db:5432'); + assert.equal(hook.socketKey(['5432'], 'tcp'), 'tcp://localhost:5432'); + assert.equal(hook.socketKey([{ host: 'h', port: 1 }], 'tls'), 'tls://h:1'); + assert.equal(hook.socketKey([{ servername: 's', port: 2 }], 'tls'), 'tls://s:2'); + assert.equal(hook.socketKey([{ path: '/p.sock' }], 'tcp'), 'unix:///p.sock'); + assert.equal(hook.socketKey(['/p.sock'], 'tcp'), 'unix:///p.sock'); + assert.equal(hook.socketKey([[{ host: 'h', port: 3 }, () => {}]], 'tcp'), 'tcp://h:3'); +}); + +test('listenKey names a fixed port, a pipe, a handle, and marks port 0 ephemeral', () => { + // Arrange / Act / Assert + assert.deepEqual(hook.listenKey([8080]), { key: 'listen:8080', ephemeral: false }); + assert.deepEqual(hook.listenKey(['8080', '0.0.0.0']), { key: 'listen:8080', ephemeral: false }); + assert.deepEqual(hook.listenKey([{ port: 9 }]), { key: 'listen:9', ephemeral: false }); + assert.deepEqual(hook.listenKey([0]), { key: null, ephemeral: true }); + assert.deepEqual(hook.listenKey([() => {}]), { key: null, ephemeral: true }); + assert.deepEqual(hook.listenKey([{}]), { key: null, ephemeral: true }); + assert.deepEqual(hook.listenKey(['/s.sock']), { key: 'listen:unix:/s.sock', ephemeral: false }); + assert.deepEqual(hook.listenKey([{ path: '/s.sock' }]), { key: 'listen:unix:/s.sock', ephemeral: false }); + assert.deepEqual(hook.listenKey([{ fd: 3 }]), { key: 'listen:handle', ephemeral: false }); +}); From 35ef0f37b2e53846f6c45e956521a8adeb0bb193 Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:36:46 +1030 Subject: [PATCH 06/36] test(flow): reconcile endpoints, directions, observed paths and readReconcile --- .../test/reconcile-network-direction.test.mjs | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 flow/test/reconcile-network-direction.test.mjs diff --git a/flow/test/reconcile-network-direction.test.mjs b/flow/test/reconcile-network-direction.test.mjs new file mode 100644 index 000000000..c696db745 --- /dev/null +++ b/flow/test/reconcile-network-direction.test.mjs @@ -0,0 +1,257 @@ +// C3 with the network and directions: endpoints agree on the peer, not the scheme; an unwitnessed +// network refutes nothing; a `touches` edge takes the direction of the opens the run saw; a bare +// name is resolved to the paths actually opened; and the report file is read back validated. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { reconcile, readReconcile, formatReport, endpointIdentity, isNameKey, nameMatches, directionOf } from '../reconcile.mjs'; +import { makeEdge, nodeId } from '../graph.mjs'; + +const M = 'bin/a.mjs'; +const ep = (key, kind) => ({ v: 1, from: nodeId('module', M), to: `node:endpoint:${key}`, kind, witness: 'static', existence: 'unknown', evidence: [`${M}:1 literal`] }); +const touch = (to) => makeEdge(nodeId('module', M), nodeId('artifact', to), 'touches'); +const g = (edges, mod = {}) => ({ generatedAt: null, nodes: [], modules: [{ path: M, state: 'analysed', pathLiterals: [], ...mod }], edges }); +const NET_OK = { state: 'usable', reason: null, observed: 1, refused: 0 }; +const rt = (observations, network = NET_OK) => ({ state: 'usable', coverage: [M], observations, network }); +const conn = (to, extra = {}) => ({ from: M, to, kind: 'connects', via: 'x', ...extra }); + +test('endpointIdentity is host and effective port, whatever the scheme', () => { + // Arrange / Act / Assert + assert.equal(endpointIdentity('https://API.x.test'), 'api.x.test:443'); + assert.equal(endpointIdentity('tls://api.x.test:443'), 'api.x.test:443'); + assert.equal(endpointIdentity('http://h'), 'h:80'); + assert.equal(endpointIdentity('tcp://db:5432'), 'db:5432'); + assert.equal(endpointIdentity('postgres://db'), 'db:'); + assert.equal(endpointIdentity('http://[::1]:9'), '[::1]:9'); + assert.equal(endpointIdentity('listen:8080'), 'listen:8080'); + assert.equal(endpointIdentity('unix:///s.sock'), 'unix:///s.sock'); + assert.equal(endpointIdentity('garbage'), 'garbage'); +}); + +test('isNameKey / nameMatches / directionOf', () => { + // Arrange / Act / Assert + assert.equal(isNameKey('rollup.json'), true); + assert.equal(isNameKey('./x.json'), true); + assert.equal(isNameKey('../y/z.json'), true); + assert.equal(isNameKey('reports/x.json'), false); + assert.equal(isNameKey('/abs/x.json'), false); + assert.equal(nameMatches('rollup.json', 'reports/a/rollup.json'), true); + assert.equal(nameMatches('rollup.json', 'rollup.json'), true); + assert.equal(nameMatches('rollup.json', 'reports/a/xrollup.json'), false); + assert.equal(nameMatches('../y/z.json', 'lib/y/z.json'), true); + assert.equal(nameMatches('./', 'anything'), false); + assert.equal(directionOf(['read']), 'reads'); + assert.equal(directionOf(['write', 'append']), 'writes'); + assert.equal(directionOf(['readwrite']), 'both'); + assert.equal(directionOf(['read', 'append']), 'both'); + assert.equal(directionOf([]), null); +}); + +test('a static endpoint and the socket the run opened are ONE edge at two layers', () => { + // Arrange + const graph = g([ep('https://api.x.test', 'connects'), ep('listen:8080', 'listens')]); + // Act + const r = reconcile(graph, rt([conn('tls://api.x.test:443'), { from: M, to: 'listen:8080', kind: 'listens', via: 'listen' }])); + // Assert + assert.equal(r.confirmed.length, 2); + assert.ok(r.edges.every((e) => e.witness === 'both' && e.existence === 'confirmed')); + assert.equal(r.divergence, 0); + assert.equal(r.falseNegatives.length + r.dynamicPaths.length, 0, 'the matched observation is not also a miss'); +}); + +test('a witnessed network that did not see a static connect leaves it UNCONFIRMED', () => { + // Arrange / Act + const r = reconcile(g([ep('https://api.x.test', 'connects')]), rt([])); + // Assert + assert.equal(r.unconfirmed.length, 1); + assert.equal(r.edges[0].unconfirmed, true); + assert.equal(r.edges[0].existence, 'unknown'); +}); + +test('AN UNWITNESSED NETWORK REFUTES NOTHING — a connect it did not see is unknown, not unconfirmed', () => { + // Arrange + const graph = g([ep('https://api.x.test', 'connects'), ep('https://b.test', 'connects')]); + // Act: no network block at all (an older summary, or a union that dropped it) + const r = reconcile(graph, { state: 'usable', coverage: [M], observations: [conn('https://b.test')] }); + // Assert + assert.equal(r.unconfirmed.length, 0); + assert.equal(r.edges[0].networkUnwitnessed, true); + assert.equal(r.edges[0].unfalsifiable, true); + assert.equal(r.edges[1].existence, 'confirmed', 'an observation it DID make still confirms'); + assert.equal(r.network.witnessed, false); + assert.match(r.network.reason, /no network witness/); +}); + +test('a runtime-only endpoint is a false negative only when the module carries a literal naming it', () => { + // Arrange + const graph = g([], { pathLiterals: ['https://named.test/v1'], urlLiterals: ['9000'] }); + // Act + const r = reconcile(graph, rt([ + conn('https://named.test'), + conn('tcp://from-config.test:5432'), + { from: M, to: 'listen:9000', kind: 'listens', via: 'listen' }, + { from: M, to: 'listen:41234', kind: 'listens', via: 'listen', ephemeral: true }, + conn('tcp://localhost:1'), + ])); + // Assert + assert.deepEqual(r.falseNegatives.map((x) => x.to).sort(), ['https://named.test', 'listen:9000']); + assert.deepEqual(r.dynamicPaths.map((x) => x.to).sort(), ['listen:41234', 'tcp://from-config.test:5432', 'tcp://localhost:1']); +}); + +test('a runtime endpoint in a module outside coverage is ignored', () => { + // Arrange / Act + const r = reconcile(g([]), { ...rt([{ from: 'bin/other.mjs', to: 'tcp://h:1', kind: 'connects' }]), coverage: [M] }); + // Assert + assert.equal(r.dynamicPaths.length + r.falseNegatives.length, 0); +}); + +test('A TOUCHES EDGE TAKES THE DIRECTION OF THE OPENS — and an exact open confirms it', () => { + // Arrange + const graph = g([touch('data/r.json'), touch('data/w.json'), touch('data/b.json'), touch('data/meta.json')]); + const obs = [ + { from: M, to: 'data/r.json', kind: 'reads', access: ['read'] }, + { from: M, to: 'data/w.json', kind: 'writes', access: ['append'] }, + { from: M, to: 'data/b.json', kind: 'reads', access: ['read'] }, + { from: M, to: 'data/b.json', kind: 'writes', access: ['write'] }, + { from: M, to: 'data/meta.json', kind: 'reads' }, + ]; + // Act + const r = reconcile(graph, rt(obs)); + // Assert + const by = Object.fromEntries(r.edges.map((e) => [e.to.replace(/^node:artifact:/, ''), e])); + assert.equal(by['data/r.json'].direction, 'reads'); + assert.equal(by['data/w.json'].direction, 'writes'); + assert.equal(by['data/b.json'].direction, 'both'); + assert.equal(by['data/r.json'].existence, 'confirmed'); + assert.equal(by['data/r.json'].witness, 'both'); + assert.equal(by['data/meta.json'].direction, undefined, 'a stat opens nothing and orients nothing'); + assert.equal(by['data/meta.json'].unfalsifiable, true); + assert.equal(r.directed, 3); + assert.equal(r.confirmed.filter((c) => c.kind === 'touches').length, 3); +}); + +test('a direction found by NAME is given, and confirms nothing', () => { + // Arrange + const graph = g([touch('rollup.json')]); + // Act + const r = reconcile(graph, rt([{ from: M, to: 'reports/a/rollup.json', kind: 'writes', access: ['write'] }])); + // Assert + assert.equal(r.edges[0].direction, 'writes'); + assert.equal(r.edges[0].directionBy, 'name'); + assert.equal(r.edges[0].existence, 'unknown', 'the name is several files'); + assert.equal(r.confirmed.length, 0); +}); + +test('observedPaths maps each bare or relative store name to what THIS module actually opened', () => { + // Arrange + const other = 'bin/b.mjs'; + const graph = { + nodes: [], + modules: [{ path: M, pathLiterals: [] }, { path: other, pathLiterals: [] }], + edges: [ + makeEdge(nodeId('module', M), nodeId('artifact', 'rollup.json'), 'reads'), + makeEdge(nodeId('module', M), nodeId('store', '../cfg/c.json'), 'touches'), + makeEdge(nodeId('module', M), nodeId('artifact', 'reports/fixed.json'), 'reads'), + makeEdge(nodeId('module', M), nodeId('env', 'CW_X'), 'reads'), + makeEdge(nodeId('module', other), nodeId('artifact', 'unrun.json'), 'reads'), + ], + }; + const obs = [ + { from: M, to: 'reports/b/rollup.json', kind: 'reads', access: ['read'] }, + { from: M, to: 'reports/a/rollup.json', kind: 'reads' }, + { from: M, to: 'reports/a/rollup.json', kind: 'writes', access: ['write'] }, + { from: M, to: 'lib/cfg/c.json', kind: 'reads', access: ['read'] }, + { from: M, to: 'reports/fixed.json', kind: 'reads' }, + { from: other, to: 'x/rollup.json', kind: 'reads' }, + ]; + // Act + const r = reconcile(graph, { state: 'usable', coverage: [M, other], observations: obs, network: NET_OK }); + // Assert + assert.deepEqual(r.observedPaths, { + 'rollup.json': ['reports/a/rollup.json', 'reports/b/rollup.json'], + '../cfg/c.json': ['lib/cfg/c.json'], + }); +}); + +test('every reconciled edge carries the contract fields, measured or not', () => { + // Arrange + const stray = makeEdge('node:process:git', nodeId('artifact', 'x'), 'reads'); + const graph = g([ep('https://h.test', 'connects'), touch('t.json'), stray]); + // Act + const measured = reconcile(graph, rt([conn('https://h.test')])); + const unmeasured = reconcile(graph, null); + // Assert + for (const r of [measured, unmeasured]) { + for (const e of r.edges) { + assert.equal(e.v, 1); + for (const f of ['from', 'to', 'kind', 'witness', 'existence', 'runtimeCoverage']) assert.ok(e[f], `${f} on ${e.to}`); + } + assert.ok(Array.isArray(r.coverage)); + assert.equal(typeof r.observedPaths, 'object'); + } + assert.deepEqual(measured.coverage, [M]); + assert.equal(unmeasured.network.state, 'unusable'); + assert.deepEqual(reconcile(graph, { state: 'unusable', reason: 'r', network: NET_OK }).network, NET_OK); +}); + +test('formatReport names directions, resolved names and the network state', () => { + // Arrange + const r = reconcile(g([touch('rollup.json')]), rt([{ from: M, to: 'a/rollup.json', kind: 'reads', access: ['read'] }])); + // Act + const text = formatReport(r); + const bare = formatReport({ state: 'unmeasured', reason: 'x', confirmed: [], outOfScope: [] }); + // Assert + assert.match(text, /touches given a direction by observed opens\s+1/); + assert.match(text, /bare-name stores resolved to observed paths\s+1/); + assert.match(text, /network\s+usable/); + assert.match(bare, /network\s+unusable/); + assert.match(formatReport({ ...r, network: { state: 'unusable', reason: 'why' } }), /unusable \(why\)/); +}); + +function withFile(content, fn) { + const d = mkdtempSync(join(tmpdir(), 'flow-rec-')); + try { + const p = join(d, 'flow-reconcile.json'); + if (content !== undefined) writeFileSync(p, typeof content === 'string' ? content : JSON.stringify(content)); + return fn(p); + } finally { rmSync(d, { recursive: true, force: true }); } +} + +test('readReconcile round-trips what reconcile produces', () => { + // Arrange + const rec = reconcile(g([touch('rollup.json'), ep('https://h.test', 'connects')]), + rt([{ from: M, to: 'a/rollup.json', kind: 'reads', access: ['read'] }, conn('https://h.test')])); + // Act + const back = withFile(rec, (p) => readReconcile(p)); + // Assert + assert.deepEqual(back, JSON.parse(JSON.stringify(rec))); +}); + +test('readReconcile refuses an absent, old-shape or malformed report — it never defaults one', () => { + // Arrange + const good = reconcile(g([touch('x.json')]), rt([])); + const edge = good.edges[0]; + const bad = [ + [[], /not an object/], + [{ ...good, v: 2 }, /v is 2/], + [{ ...good, edges: null }, /edges is not an array/], + [{ ...good, edges: [null] }, /edges\[0\] not an object/], + [{ ...good, edges: [{ ...edge, v: 0 }] }, /edges\[0\]\.v/], + [{ ...good, edges: [{ ...edge, to: '' }] }, /edges\[0\]\.to missing/], + [{ ...good, edges: [{ ...edge, witness: 'maybe' }] }, /witness/], + [{ ...good, edges: [{ ...edge, existence: 'sure' }] }, /existence/], + [{ ...good, edges: [{ ...edge, runtimeCoverage: 'some' }] }, /runtimeCoverage/], + [{ ...good, edges: [{ ...edge, direction: 'sideways' }] }, /direction/], + [{ ...good, observedPaths: [] }, /observedPaths is not an object/], + [{ ...good, observedPaths: { a: [1] } }, /observedPaths\["a"\]/], + [{ ...good, coverage: 'bin/a.mjs' }, /coverage is not a list/], + ]; + // Act / Assert + for (const [content, why] of bad) { + withFile(content, (p) => assert.throws(() => readReconcile(p), (e) => e.code === 'CW_FLOW_RECONCILE_INVALID' && why.test(e.message), String(why))); + } + withFile(undefined, (p) => assert.throws(() => readReconcile(p), { code: 'CW_FLOW_RECONCILE_ABSENT' })); + withFile('{not json', (p) => assert.throws(() => readReconcile(p), { code: 'CW_FLOW_STORE_CORRUPT' })); +}); From 85a52ebe28a7c36c9b8b5be5a0a88f2d97a7f3ad Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:39:47 +1030 Subject: [PATCH 07/36] feat(dfd): closed subtypes, flow roles, verdict and direction; role-aware feedback; direction-aware filter --- lib/dfd.mjs | 197 +++++++---------- lib/test/dfd.test.mjs | 304 +++++++++++++++------------ lib/test/fixtures/dfd-synthetic.json | 38 ++-- schema/dfd.schema.json | 19 +- 4 files changed, 289 insertions(+), 269 deletions(-) diff --git a/lib/dfd.mjs b/lib/dfd.mjs index 1faebe837..85fedc6d6 100644 --- a/lib/dfd.mjs +++ b/lib/dfd.mjs @@ -1,5 +1,7 @@ -// lib/dfd.mjs — dfd@1: a source-independent data flow diagram model, its validation, its analysis -// and the adapter from the flow graph (flow/graph.mjs). Rendering is lib/dfd-render.mjs. +// lib/dfd.mjs — dfd@1: a source-independent data flow diagram model, its validation and its +// analysis. Rendering is lib/dfd-render.mjs. The adapters that build a model from the flow graph or +// from CodeQL code-flow records are flow/dfd-adapter.mjs: lib/ ships and flow/ does not, so nothing +// here imports from flow/. // // Three element kinds and no fourth. A message broker or queue is a STORE with subtype // broker|queue|topic|stream: data rests in it between a producer and a consumer, the classic @@ -16,18 +18,30 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { validateAgainstSchema } from './json-schema.mjs'; -import { isTestModule } from '../flow/static.mjs'; export const DFD_VERSION = 1; export const NODE_KINDS = Object.freeze(['process', 'store', 'external']); export const MODES = Object.freeze(['sync', 'async', 'event']); -export const BROKER_SUBTYPES = Object.freeze(['broker', 'queue', 'topic', 'stream']); +// `log` is a broker: an append-only file one process appends to and another tails holds data +// between them exactly as a topic does, and the reader is decoupled from the writer in time. +export const BROKER_SUBTYPES = Object.freeze(['broker', 'queue', 'topic', 'stream', 'log']); +export const STORE_SUBTYPES = Object.freeze([...BROKER_SUBTYPES, 'database', 'file']); +export const EXTERNAL_SUBTYPES = Object.freeze(['endpoint', 'scheduler', 'actor', 'system', 'env', 'binary']); +// A flow's role says what it carries back. Only `data` takes part in the two-way collapse: a +// feedback, error or retry flow is a return path even when its pair is adjacent. +export const ROLES = Object.freeze(['data', 'feedback', 'error', 'retry']); +// flow.direction: what the runtime saw a module do across a static `touches`, which never had one +export const FLOW_DIRECTIONS = Object.freeze(['reads', 'writes', 'both']); +// filterModel's walk: forward along flows, backward against them, or both +export const FILTER_DIRECTIONS = Object.freeze(['in', 'out', 'both']); export const FLOW_CLASSES = Object.freeze(['forward', 'bidirectional', 'feedback', 'self-loop', 'undirected']); export const schemaPath = () => join(dirname(fileURLToPath(import.meta.url)), '..', 'schema', 'dfd.schema.json'); export const isBroker = (node) => !!node && node.kind === 'store' && BROKER_SUBTYPES.includes(node.subtype); export const isDirected = (flow) => flow.directed !== false; +export const roleOf = (flow) => flow.role || 'data'; +const SUBTYPES_BY_KIND = { store: STORE_SUBTYPES, external: EXTERNAL_SUBTYPES }; /** -> { ok, errors }. Schema first; the referential checks run only on a document of the right shape. */ export function validateDfd(doc, { path = schemaPath() } = {}) { @@ -45,11 +59,19 @@ export function validateDfd(doc, { path = schemaPath() } = {}) { for (const n of doc.nodes) { if (layers && n.layer !== undefined && !layers.has(n.layer)) errors.push(`dfd: node ${JSON.stringify(n.id)} names layer ${JSON.stringify(n.layer)}, which "layers" does not declare`); if (groups && n.group !== undefined && !groups.has(n.group)) errors.push(`dfd: node ${JSON.stringify(n.id)} names group ${JSON.stringify(n.group)}, which "groups" does not declare`); + // the schema admits the union of the subtypes; which kind may carry which is checked here + if (n.subtype !== undefined) { + const allowed = SUBTYPES_BY_KIND[n.kind]; + if (!allowed) errors.push(`dfd: node ${JSON.stringify(n.id)} is a process and carries subtype ${JSON.stringify(n.subtype)}: only a store or an external has one`); + else if (!allowed.includes(n.subtype)) errors.push(`dfd: node ${JSON.stringify(n.id)} is a ${n.kind} and carries subtype ${JSON.stringify(n.subtype)}, which is a subtype of the other kind (${n.kind} takes ${allowed.join('|')})`); + } } doc.flows.forEach((f, i) => { for (const end of ['from', 'to']) { if (!ids.has(f[end])) errors.push(`dfd: flows[${i}].${end} names ${JSON.stringify(f[end])}, which is not a node`); } + // a resolved direction on a flow that still says it has none is two answers to one question + if (f.direction !== undefined && !isDirected(f)) errors.push(`dfd: flows[${i}] carries direction ${JSON.stringify(f.direction)} and directed:false — a resolved touch is directed`); }); return { ok: errors.length === 0, errors }; } @@ -151,9 +173,15 @@ export function longestPathRanks(n, arcs) { * Classification, in priority order — every flow takes exactly one class: * undirected directed:false; excluded from everything below * self-loop from === to (a retry on the same process) - * bidirectional the pair A->B and B->A both exist; collapses to one two-headed flow - * feedback a back arc inside a strongly connected component (see backArcs) + * feedback role feedback|error|retry: a declared return path, even when its pair is + * adjacent — A->B data with B->A error is a loop, not a synchronised pair + * bidirectional role data both ways: A->B and B->A; collapses to one two-headed flow + * feedback a back arc inside a strongly connected component of the data flows (backArcs) * forward everything else + * + * Ranks and paths run over the data flows less their back arcs; a declared return path is set + * aside as a back arc is. Cycles are the components of every directed flow, declared returns + * included, so a loop closed only by an error flow is still reported as a cycle. */ export function analyse(model, { maxPaths = 200 } = {}) { const ids = model.nodes.map((n) => n.id); @@ -162,19 +190,31 @@ export function analyse(model, { maxPaths = 200 } = {}) { const flows = model.flows; const flowClass = flows.map(() => 'forward'); const arcs = []; + const data = []; flows.forEach((f, k) => { if (!isDirected(f)) flowClass[k] = 'undirected'; else if (f.from === f.to) flowClass[k] = 'self-loop'; - else arcs.push({ a: ix.get(f.from), b: ix.get(f.to), k }); + else { + const arc = { a: ix.get(f.from), b: ix.get(f.to), k }; + arcs.push(arc); + if (roleOf(f) === 'data') data.push(arc); + else flowClass[k] = 'feedback'; + } }); - const out = Array.from({ length: n }, () => []); - for (const { a, b, k } of arcs) out[a].push([b, k]); - const { comp, count } = stronglyConnected(out); - const back = backArcs(out, comp, count); + const adjacency = (list) => { + const out = Array.from({ length: n }, () => []); + for (const { a, b, k } of list) out[a].push([b, k]); + return out; + }; + const dataOut = adjacency(data); + const dataScc = stronglyConnected(dataOut); + const back = backArcs(dataOut, dataScc.comp, dataScc.count); + for (const { k } of arcs) if (flowClass[k] === 'feedback') back.add(k); + const { comp, count } = stronglyConnected(adjacency(arcs)); const byPair = new Map(); - for (const { a, b, k } of arcs) { + for (const { a, b, k } of data) { const key = `${a}>${b}`; if (!byPair.has(key)) byPair.set(key, []); byPair.get(key).push(k); @@ -264,7 +304,7 @@ export function analyse(model, { maxPaths = 200 } = {}) { nodes: ids.map((id, i) => ({ id, rank: rank[i], sccRank: sccRank[comp[i]], cyclic: sizes[comp[i]] > 1, fanOut: succ[i].size, fanIn: pred[i].size })), bidirectional, selfLoops: flows.map((f, k) => ({ node: f.from, flow: k })).filter((_, k) => flowClass[k] === 'self-loop'), - feedback: flows.map((f, k) => ({ flow: k, from: f.from, to: f.to })).filter((_, k) => flowClass[k] === 'feedback'), + feedback: flows.map((f, k) => ({ flow: k, from: f.from, to: f.to, role: roleOf(f) })).filter((_, k) => flowClass[k] === 'feedback'), undirected: flowClass.map((c, k) => (c === 'undirected' ? k : -1)).filter((k) => k >= 0), cycles, branches: ids.map((id, i) => ({ node: id, to: [...succ[i]].map((j) => ids[j]) })).filter((b) => b.to.length > 1), @@ -283,127 +323,34 @@ export function analyse(model, { maxPaths = 200 } = {}) { }; } -// ── the flow-graph adapter ─────────────────────────────────────────────────────────────────────── - -// A spawn target the static pass captured is the first argument of exec/spawn AS WRITTEN. `exec` is -// also a database method, so SQL text arrives here as a "process". Only a single token that reads as -// a command name is drawn as an external entity; the rest are counted and left out. -const COMMAND_NAME = /^[A-Za-z0-9_@.+/-]+$/; -const SYNC_VERB = /Sync$/; -const verbOf = (ref) => String(ref).split(' ').pop(); - -/** - * Flow graph (flow/graph.mjs) -> dfd@1. The mapping: - * - * module -> process group = first path segment; unanalysable modules marked unknown - * artifact, store -> store subtype file; a bare-filename node is marked unknown - * env -> external subtype env: configuration supplied from outside the boundary - * process (spawn) -> external subtype binary, when the target reads as a command name; - * -> the module's own process when the target is a module path; - * otherwise omitted and counted - * reads M <- X -> flow X -> M writes M -> X -> flow M -> X - * spawns M -> P -> flow M -> P, async unless every witness is a *Sync call; omitted and - * counted when every witness is a bare `exec`, which a database handle - * also has - * touches M -- X -> flow M -- X, directed:false - * reads M <- M' -> a static or dynamic import: a code dependency, not a data flow. Omitted and - * counted unless { imports: true }. - * - * File reads and writes are `sync`: the mode says whether the two ends are coupled in time, and the - * static pass has no evidence that any of them is not. Every flow carries the edge's own witness and - * existence, unchanged. Nodes left with no flow are omitted and counted. - */ -export function fromFlowGraph(graph, { tests = false, imports = false, title } = {}) { - const src = new Map(graph.nodes.map((n) => [n.id, n])); - const modulePaths = new Set(graph.nodes.filter((n) => n.kind === 'module').map((n) => n.path)); - const omitted = { testModules: 0, importEdges: 0, spawnTargetsNotCommands: 0, ambiguousExecEdges: 0, edgesOfOmittedNodes: 0, isolatedNodes: 0 }; - const mapped = new Map(); - - const mapNode = (n) => { - if (n.kind === 'module') { - if (!tests && isTestModule(n.path)) { omitted.testModules++; return null; } - const unknown = n.analysis !== undefined && n.analysis !== 'analysed'; - return { id: n.id, kind: 'process', label: n.path, group: n.path.includes('/') ? n.path.split('/')[0] : '(root)', - ...(unknown ? { unknown: true, note: `not analysed (${n.analysis}): its flows are unknown, not absent` } : {}) }; - } - if (n.kind === 'artifact' || n.kind === 'store') { - return { id: n.id, kind: 'store', label: n.path, subtype: 'file', - ...(n.composed ? { unknown: true, note: 'bare filename: a name, not a location — several files may share this node' } : {}) }; - } - if (n.kind === 'env') return { id: n.id, kind: 'external', label: n.path, subtype: 'env' }; - if (modulePaths.has(n.path)) return mapped.get(`node:module:${n.path}`); - if (COMMAND_NAME.test(n.path)) return { id: n.id, kind: 'external', label: n.path, subtype: 'binary' }; - omitted.spawnTargetsNotCommands++; - return null; - }; - // modules first: a spawn target that names a module resolves to that module's process - for (const n of graph.nodes) if (n.kind === 'module') mapped.set(n.id, mapNode(n)); - for (const n of graph.nodes) if (n.kind !== 'module') mapped.set(n.id, mapNode(n)); - - const flows = []; - for (const e of graph.edges) { - const from = mapped.get(e.from); - const to = mapped.get(e.to); - if (!from || !to) { omitted.edgesOfOmittedNodes++; continue; } - const refs = [].concat(e.evidence || []); - const evidence = { witness: e.witness, existence: e.existence, refs }; - if (e.kind === 'reads') { - if (src.get(e.to).kind === 'module') { - if (!imports) { omitted.importEdges++; continue; } - flows.push({ from: to.id, to: from.id, label: 'imports', mode: 'sync', evidence }); - } else flows.push({ from: to.id, to: from.id, label: 'reads', mode: 'sync', evidence }); - } else if (e.kind === 'writes') flows.push({ from: from.id, to: to.id, label: 'writes', mode: 'sync', evidence }); - else if (e.kind === 'spawns') { - // a bare `exec` is child_process.exec or a database handle's exec: the witness cannot say which - if (refs.length > 0 && refs.every((r) => verbOf(r) === 'exec')) { omitted.ambiguousExecEdges++; continue; } - const sync = refs.length > 0 && refs.every((r) => SYNC_VERB.test(verbOf(r))); - flows.push({ from: from.id, to: to.id, label: 'spawns', mode: sync ? 'sync' : 'async', evidence }); - } else flows.push({ from: from.id, to: to.id, label: 'touches', mode: 'sync', directed: false, evidence }); - } - - const used = new Set(flows.flatMap((f) => [f.from, f.to])); - const nodes = []; - const seen = new Set(); - for (const m of mapped.values()) { - if (!m || seen.has(m.id)) continue; - seen.add(m.id); - if (used.has(m.id)) nodes.push(m); - else omitted.isolatedNodes++; - } - return { - v: DFD_VERSION, - kind: 'dfd', - title: title || 'flow graph', - source: { - producer: 'flow-graph', - graph: { v: graph.v ?? null, generatedAt: graph.generatedAt ?? null, nodes: graph.nodes.length, edges: graph.edges.length }, - options: { tests, imports }, - omitted, - }, - nodes, - flows, - }; -} - // ── filter ─────────────────────────────────────────────────────────────────────────────────────── /** - * Keep the seeds and everything within `hops` flows of them, in either direction. Seeds are the - * nodes whose label or id starts with `prefix`, plus the node whose id or label is `focus`. With - * neither, the model is returned as it is; when nothing matches, null — never an empty diagram. - * What was cut is counted in source.filter: a filtered diagram that does not say so reads as the - * whole system. + * Keep the seeds and everything within `hops` flows of them. Seeds are the nodes whose label or id + * starts with `prefix`, plus the node whose id or label is `around` (`focus` is its older name and + * is still read). `direction` chooses the walk: 'out' follows flows forward from the seeds (what + * they feed), 'in' follows them backward (what feeds them), 'both' (the default) ignores direction. + * An undirected flow is walked in every direction: which way it runs was never determined, so + * neither walk may drop it. With no seed criterion the model is returned as it is; when nothing + * matches, null — never an empty diagram. What was cut is counted in source.filter: a filtered + * diagram that does not say so reads as the whole system. */ -export function filterModel(model, { prefix, focus, hops = 1 } = {}) { - if (prefix === undefined && focus === undefined) return model; +export function filterModel(model, { prefix, around, focus, hops = 1, direction = 'both' } = {}) { + const centre = around ?? focus; + if (!FILTER_DIRECTIONS.includes(direction)) throw new Error(`dfd: filterModel direction must be one of ${FILTER_DIRECTIONS.join(', ')}, got ${JSON.stringify(direction)}`); + if (prefix === undefined && centre === undefined) return model; const seeds = new Set(); for (const n of model.nodes) { if (prefix !== undefined && (n.label.startsWith(prefix) || n.id.startsWith(prefix))) seeds.add(n.id); - if (focus !== undefined && (n.id === focus || n.label === focus)) seeds.add(n.id); + if (centre !== undefined && (n.id === centre || n.label === centre)) seeds.add(n.id); } if (seeds.size === 0) return null; const near = new Map(model.nodes.map((n) => [n.id, []])); - for (const f of model.flows) { near.get(f.from).push(f.to); near.get(f.to).push(f.from); } + for (const f of model.flows) { + const both = direction === 'both' || !isDirected(f); + if (both || direction === 'out') near.get(f.from).push(f.to); + if (both || direction === 'in') near.get(f.to).push(f.from); + } const keep = new Set(seeds); let edge = [...seeds]; for (let h = 0; h < hops; h++) { @@ -418,7 +365,7 @@ export function filterModel(model, { prefix, focus, hops = 1 } = {}) { v, kind, ...(title === undefined ? {} : { title }), source: { ...(source || { producer: 'authored' }), - filter: { prefix: prefix ?? null, focus: focus ?? null, hops, seeds: seeds.size, + filter: { prefix: prefix ?? null, focus: centre ?? null, hops, direction, seeds: seeds.size, nodesKept: nodes.length, nodesOmitted: model.nodes.length - nodes.length, flowsOmitted: model.flows.length - flows.length }, }, ...rest, diff --git a/lib/test/dfd.test.mjs b/lib/test/dfd.test.mjs index 064184e67..61938c4e8 100644 --- a/lib/test/dfd.test.mjs +++ b/lib/test/dfd.test.mjs @@ -1,4 +1,5 @@ -// dfd@1: validation, the analysis and the flow-graph adapter (lib/dfd.mjs). The analysis is the +// dfd@1: validation, the analysis and the filter (lib/dfd.mjs); the adapters are tested in +// flow/test/dfd-adapter.test.mjs. The analysis is the // part a reader trusts without re-deriving, so each topology is asserted on a model small enough to // check by hand, and once more on the fixture that carries all of them together. import { test } from 'node:test'; @@ -7,10 +8,10 @@ import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { - validateDfd, analyse, fromFlowGraph, filterModel, stronglyConnected, backArcs, longestPathRanks, - isBroker, isDirected, schemaPath, NODE_KINDS, MODES, FLOW_CLASSES, + validateDfd, analyse, filterModel, stronglyConnected, backArcs, longestPathRanks, + isBroker, isDirected, roleOf, schemaPath, NODE_KINDS, MODES, FLOW_CLASSES, + STORE_SUBTYPES, EXTERNAL_SUBTYPES, BROKER_SUBTYPES, ROLES, FLOW_DIRECTIONS, FILTER_DIRECTIONS, } from '../dfd.mjs'; -import { makeNode, makeEdge, nodeId } from '../../flow/graph.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const fixture = () => JSON.parse(readFileSync(join(HERE, 'fixtures', 'dfd-synthetic.json'), 'utf8')); @@ -110,7 +111,7 @@ test('duplicate ids, dangling flow ends and undeclared layers and groups are eac assert.match(errors.join('\n'), /a group id is declared more than once/); assert.match(errors.join('\n'), /names layer "platinum"/); assert.match(errors.join('\n'), /names group "elsewhere"/); - assert.match(errors.join('\n'), /flows\[20\]\.from names "ghost"/); + assert.match(errors.join('\n'), /flows\[26\]\.from names "ghost"/); }); test('without declared layers or groups any layer and group string is accepted', () => { @@ -127,18 +128,91 @@ test('without declared layers or groups any layer and group string is accepted', test('a broker is a store with a broker subtype, and a flow is directed unless it says otherwise', () => { // arrange const topic = { id: 't', kind: 'store', label: 't', subtype: 'topic' }; - const table = { id: 'd', kind: 'store', label: 'd', subtype: 'relational' }; + const table = { id: 'd', kind: 'store', label: 'd', subtype: 'database' }; + const log = { id: 'l', kind: 'store', label: 'l', subtype: 'log' }; const proc = { id: 'p', kind: 'process', label: 'p', subtype: 'queue' }; // act - const brokers = [topic, table, proc, undefined].map(isBroker); + const brokers = [topic, table, log, proc, undefined].map(isBroker); const directed = [{ from: 'a', to: 'b' }, { from: 'a', to: 'b', directed: true }, { from: 'a', to: 'b', directed: false }].map(isDirected); // assert - assert.deepEqual(brokers, [true, false, false, false]); + assert.deepEqual(brokers, [true, false, true, false, false]); assert.deepEqual(directed, [true, true, false]); }); +test('the schema admits exactly the subtypes, roles, verdicts and directions the code names', () => { + // arrange + const schema = JSON.parse(readFileSync(schemaPath(), 'utf8')); + const flow = schema.properties.flows.items.properties; + + // act + const subtypes = schema.properties.nodes.items.properties.subtype.enum; + + // assert + assert.deepEqual(subtypes, [...STORE_SUBTYPES, ...EXTERNAL_SUBTYPES]); + assert.deepEqual(flow.role.enum, [...ROLES]); + assert.deepEqual(flow.existence.enum, ['confirmed', 'unknown']); + assert.deepEqual(flow.direction.enum, [...FLOW_DIRECTIONS]); + assert.deepEqual(FILTER_DIRECTIONS, ['in', 'out', 'both']); + assert.ok(BROKER_SUBTYPES.every((t) => STORE_SUBTYPES.includes(t))); + assert.equal(schema.properties.v.const, 1); +}); + +test('a document written before the additions is still valid: every new field is optional', () => { + // arrange + const doc = { v: 1, kind: 'dfd', nodes: [{ id: 'a', kind: 'process', label: 'a' }, { id: 'b', kind: 'store', label: 'b' }], flows: [{ from: 'a', to: 'b', mode: 'sync', directed: true }] }; + + // act + const r = validateDfd(doc); + + // assert + assert.deepEqual(r, { ok: true, errors: [] }); +}); + +test('a subtype must belong to its kind, a process takes none, and an unlisted one fails the schema', () => { + // arrange + const doc = (subtype, kind) => ({ v: 1, kind: 'dfd', nodes: [{ id: 'x', kind, label: 'x', subtype }], flows: [] }); + + // act + const storeAsEndpoint = validateDfd(doc('endpoint', 'store')).errors; + const externalAsLog = validateDfd(doc('log', 'external')).errors; + const processSubtype = validateDfd(doc('queue', 'process')).errors; + const unlisted = validateDfd(doc('lake', 'store')).errors; + const fine = [validateDfd(doc('scheduler', 'external')), validateDfd(doc('database', 'store'))].map((r) => r.ok); + + // assert + assert.match(storeAsEndpoint.join(), /node "x" is a store and carries subtype "endpoint".*store takes broker\|queue\|topic\|stream\|log\|database\|file/); + assert.match(externalAsLog.join(), /node "x" is a external and carries subtype "log"/); + assert.match(processSubtype.join(), /node "x" is a process and carries subtype "queue": only a store or an external has one/); + assert.equal(unlisted.length, 1); + assert.doesNotMatch(unlisted[0], /^dfd:/); + assert.deepEqual(fine, [true, true]); +}); + +test('a resolved direction on a flow that still says directed:false is refused', () => { + // arrange + const doc = { v: 1, kind: 'dfd', nodes: [{ id: 'a', kind: 'process', label: 'a' }, { id: 'b', kind: 'store', label: 'b' }], + flows: [{ from: 'a', to: 'b', mode: 'sync', directed: false, direction: 'writes' }, { from: 'a', to: 'b', mode: 'sync', direction: 'writes', role: 'data', existence: 'confirmed' }] }; + + // act + const r = validateDfd(doc); + + // assert + assert.deepEqual(r.errors, ['dfd: flows[0] carries direction "writes" and directed:false — a resolved touch is directed']); +}); + +test('a role outside the four, or a verdict of contradicted, fails the schema', () => { + // arrange + const doc = (extra) => ({ v: 1, kind: 'dfd', nodes: [{ id: 'a', kind: 'process', label: 'a' }], flows: [{ from: 'a', to: 'a', mode: 'sync', ...extra }] }); + + // act + const results = [{ role: 'control' }, { existence: 'contradicted' }, { direction: 'sideways' }].map((x) => validateDfd(doc(x)).ok); + + // assert + assert.deepEqual(results, [false, false, false]); +}); + // ── graph primitives ───────────────────────────────────────────────────────────────────────────── test('stronglyConnected numbers components by their first member', () => { @@ -235,7 +309,7 @@ test('a flow from a downstream process back to an upstream one is the feedback f const a = analyse(m); // assert - assert.deepEqual(a.feedback, [{ flow: 3, from: 'p3', to: 'p1' }]); + assert.deepEqual(a.feedback, [{ flow: 3, from: 'p3', to: 'p1', role: 'data' }]); assert.deepEqual(a.cycles, [{ members: ['p1', 'p2', 'p3'], feedback: [3], bidirectionalOnly: false, uncertain: false }]); assert.deepEqual(a.nodes.map((n) => n.rank), [0, 1, 2, 3, 4]); assert.deepEqual(a.nodes.map((n) => n.sccRank), [0, 1, 1, 1, 2]); @@ -299,7 +373,7 @@ test('branch and merge points are counted over distinct neighbours', () => { test('a flow is an async hop by its mode, by a broker at either end, or by both', () => { // arrange - const m = model(['p1~>s1', 's1>p2', 'p2~>s2', 'p2>p3'], { s1: { subtype: 'queue' }, s2: { subtype: 'log' } }); + const m = model(['p1~>s1', 's1>p2', 'p2~>s2', 'p2>p3'], { s1: { subtype: 'queue' }, s2: { subtype: 'database' } }); // act const a = analyse(m); @@ -343,158 +417,91 @@ test('a path count past the safe integers is null, never a rounded number', () = assert.equal(a.paths.listed[0].length, 121); }); -test('the fixture carries every topology at once', () => { +test('an error flow back along an adjacent pair is feedback, not a two-way pair', () => { // arrange - const m = fixture(); - const at = (from, to) => m.flows.findIndex((f) => f.from === from && f.to === to); + const m = model(['p1>p2', 'p2>p1', 'p2>s1']); + m.flows[1].role = 'error'; // act const a = analyse(m); // assert - assert.deepEqual(a.bidirectional.map((b) => [b.a, b.b]), [['customer', 'api']]); - assert.deepEqual(a.selfLoops, [{ node: 'ingest', flow: at('ingest', 'ingest') }]); - assert.deepEqual(a.feedback.map((f) => `${f.from}>${f.to}`), ['validate>transform', 'validate>bronze']); - assert.deepEqual(a.cycles.map((c) => c.members), [['customer', 'api'], ['bronze', 'transform', 'enrich', 'validate']]); - assert.deepEqual(a.branches.map((b) => b.node), ['api', 'validate', 'silver']); - assert.deepEqual(a.merges.map((b) => b.node), ['bronze', 'transform', 'gold']); - assert.deepEqual(a.brokered, [{ from: 'api', via: 'events', to: 'ingest' }]); - assert.deepEqual(a.asyncHops.map((h) => h.reason), ['mode+broker', 'mode+broker', 'mode']); - assert.equal(a.paths.total, 4); - assert.equal(Math.max(...a.paths.listed.map((p) => p.length)), 13); - assert.equal(a.flowClass.filter((c) => c === 'forward').length, 15); -}); - -// ── the flow-graph adapter ─────────────────────────────────────────────────────────────────────── - -function flowGraph() { - const mod = (p, extra) => makeNode('module', p, extra); - const nodes = [ - mod('bin/tool.mjs', { analysis: 'analysed' }), - mod('bin/helper.mjs', { analysis: 'analysed' }), - mod('bin/broken.mjs', { analysis: 'unparseable' }), - mod('bin/test/tool.test.mjs', { analysis: 'analysed' }), - mod('alone.mjs', { analysis: 'analysed' }), - makeNode('store', 'reports/out.json'), - makeNode('artifact', 'rollup.json', { composed: true }), - makeNode('env', 'CW_HOME'), - makeNode('process', 'git'), - makeNode('process', 'CREATE TABLE t (a TEXT)'), - makeNode('process', 'COMMIT'), - makeNode('process', 'bin/helper.mjs'), - ]; - const m = (p) => nodeId('module', p); - const edges = [ - makeEdge(m('bin/tool.mjs'), nodeId('store', 'reports/out.json'), 'writes', { evidence: ['bin/tool.mjs:9 writeFileSync'] }), - makeEdge(m('bin/helper.mjs'), nodeId('store', 'reports/out.json'), 'reads', { witness: 'both', existence: 'confirmed', evidence: ['bin/helper.mjs:4 readFileSync'] }), - makeEdge(m('bin/tool.mjs'), nodeId('artifact', 'rollup.json'), 'touches', { evidence: ['bin/tool.mjs:12 literal'] }), - makeEdge(m('bin/tool.mjs'), nodeId('env', 'CW_HOME'), 'reads', { evidence: ['bin/tool.mjs:3 env'] }), - makeEdge(m('bin/tool.mjs'), m('bin/helper.mjs'), 'reads', { evidence: ['bin/tool.mjs static-import'] }), - makeEdge(m('bin/tool.mjs'), nodeId('process', 'git'), 'spawns', { evidence: ['bin/tool.mjs:20 execFileSync', 'bin/tool.mjs:31 spawnSync'] }), - makeEdge(m('bin/helper.mjs'), nodeId('process', 'git'), 'spawns', { evidence: ['bin/helper.mjs:7 spawn'] }), - makeEdge(m('bin/tool.mjs'), nodeId('process', 'CREATE TABLE t (a TEXT)'), 'spawns', { evidence: ['bin/tool.mjs:40 exec'] }), - makeEdge(m('bin/tool.mjs'), nodeId('process', 'COMMIT'), 'spawns', { evidence: ['bin/tool.mjs:41 exec'] }), - makeEdge(m('bin/tool.mjs'), nodeId('process', 'bin/helper.mjs'), 'spawns', { evidence: null }), - makeEdge(m('bin/test/tool.test.mjs'), nodeId('store', 'reports/out.json'), 'reads', { evidence: ['bin/test/tool.test.mjs:5 readFileSync'] }), - makeEdge(m('bin/broken.mjs'), nodeId('env', 'CW_HOME'), 'reads', { evidence: ['bin/broken.mjs:1 env'] }), - ]; - return { v: 1, generatedAt: '2026-11-11T00:00:00.000Z', nodes, edges }; -} - -test('fromFlowGraph maps modules, files, env keys and spawned binaries by the stated rules', () => { - // arrange - const graph = flowGraph(); - - // act - const d = fromFlowGraph(graph); - const byLabel = Object.fromEntries(d.nodes.map((n) => [n.label, n])); - - // assert - assert.deepEqual(validateDfd(d).errors, []); - assert.deepEqual(Object.keys(d).slice(0, 2), ['v', 'kind']); - assert.deepEqual(byLabel['bin/tool.mjs'], { id: 'node:module:bin/tool.mjs', kind: 'process', label: 'bin/tool.mjs', group: 'bin' }); - assert.equal(byLabel['bin/broken.mjs'].unknown, true); - assert.match(byLabel['bin/broken.mjs'].note, /not analysed \(unparseable\)/); - assert.deepEqual(byLabel['reports/out.json'], { id: 'node:store:reports/out.json', kind: 'store', label: 'reports/out.json', subtype: 'file' }); - assert.equal(byLabel['rollup.json'].unknown, true); - assert.deepEqual(byLabel.CW_HOME, { id: 'node:env:CW_HOME', kind: 'external', label: 'CW_HOME', subtype: 'env' }); - assert.deepEqual(byLabel.git, { id: 'node:process:git', kind: 'external', label: 'git', subtype: 'binary' }); - assert.equal(d.nodes.some((n) => /CREATE TABLE|COMMIT|test/.test(n.label)), false); + assert.deepEqual(a.flowClass, ['forward', 'feedback', 'forward']); + assert.deepEqual(a.bidirectional, []); + assert.deepEqual(a.feedback, [{ flow: 1, from: 'p2', to: 'p1', role: 'error' }]); + assert.deepEqual(a.cycles, [{ members: ['p1', 'p2'], feedback: [1], bidirectionalOnly: false, uncertain: false }]); + assert.deepEqual(a.nodes.map((n) => n.rank), [0, 1, 2]); + assert.deepEqual(a.paths.listed, [['p1', 'p2', 's1']]); }); -test('fromFlowGraph turns a read around, keeps a write, and leaves a touch undirected', () => { +test('only data flows collapse: a data pair stays two-way beside a retry along the same pair', () => { // arrange - const graph = flowGraph(); + const m = model(['p1>p2', 'p2>p1', 'p2>p1']); + m.flows[0].role = 'data'; + m.flows[2].role = 'retry'; // act - const d = fromFlowGraph(graph); - const flow = (label, from) => d.flows.find((f) => f.label === label && f.from === from); + const a = analyse(m); // assert - assert.deepEqual(flow('writes', 'node:module:bin/tool.mjs'), { - from: 'node:module:bin/tool.mjs', to: 'node:store:reports/out.json', label: 'writes', mode: 'sync', - evidence: { witness: 'static', existence: 'unknown', refs: ['bin/tool.mjs:9 writeFileSync'] }, - }); - assert.deepEqual(flow('reads', 'node:store:reports/out.json'), { - from: 'node:store:reports/out.json', to: 'node:module:bin/helper.mjs', label: 'reads', mode: 'sync', - evidence: { witness: 'both', existence: 'confirmed', refs: ['bin/helper.mjs:4 readFileSync'] }, - }); - assert.equal(flow('reads', 'node:env:CW_HOME').to, 'node:module:bin/tool.mjs'); - assert.equal(flow('touches', 'node:module:bin/tool.mjs').directed, false); + assert.deepEqual(a.flowClass, ['bidirectional', 'bidirectional', 'feedback']); + assert.deepEqual(a.bidirectional, [{ a: 'p1', b: 'p2', flows: [0, 1], modes: ['sync'] }]); + assert.deepEqual(a.feedback.map((f) => [f.flow, f.role]), [[2, 'retry']]); + assert.equal(a.cycles[0].bidirectionalOnly, false); }); -test('fromFlowGraph reads a spawn as sync only when every witness is a Sync call', () => { +test('a declared feedback flow is a return path even with no cycle among the data flows', () => { // arrange - const graph = flowGraph(); + const m = model(['e1>p1', 'p1>s1', 's1>p1', 'p1>s2']); + m.flows[2].role = 'feedback'; // act - const d = fromFlowGraph(graph); - const spawns = d.flows.filter((f) => f.label === 'spawns').map((f) => [f.from, f.to, f.mode]); + const a = analyse(m); // assert - assert.deepEqual(spawns, [ - ['node:module:bin/tool.mjs', 'node:process:git', 'sync'], - ['node:module:bin/helper.mjs', 'node:process:git', 'async'], - ['node:module:bin/tool.mjs', 'node:module:bin/helper.mjs', 'async'], - ]); + assert.deepEqual(a.flowClass, ['forward', 'forward', 'feedback', 'forward']); + assert.deepEqual(a.nodes.map((n) => [n.id, n.rank]), [['e1', 0], ['p1', 1], ['s1', 2], ['s2', 2]]); + assert.deepEqual(a.sources, ['e1']); + assert.equal(a.paths.total, 2); }); -test('fromFlowGraph counts everything it leaves out', () => { +test('a retry on one process stays a self-loop whatever its role, and roleOf defaults to data', () => { // arrange - const graph = flowGraph(); + const m = model(['p1>p1', 'p1>s1']); + m.flows[0].role = 'retry'; // act - const d = fromFlowGraph(graph); + const a = analyse(m); // assert - assert.deepEqual(d.source, { - producer: 'flow-graph', - graph: { v: 1, generatedAt: '2026-11-11T00:00:00.000Z', nodes: 12, edges: 12 }, - options: { tests: false, imports: false }, - omitted: { testModules: 1, importEdges: 1, spawnTargetsNotCommands: 1, ambiguousExecEdges: 1, edgesOfOmittedNodes: 2, isolatedNodes: 2 }, - }); - assert.equal(d.title, 'flow graph'); + assert.deepEqual(a.flowClass, ['self-loop', 'forward']); + assert.deepEqual([roleOf({}), roleOf({ role: 'error' })], ['data', 'error']); }); -test('fromFlowGraph keeps tests and imports when asked, and takes a title', () => { +test('the fixture carries every topology at once', () => { // arrange - const graph = flowGraph(); - delete graph.v; - delete graph.generatedAt; + const m = fixture(); + const at = (from, to) => m.flows.findIndex((f) => f.from === from && f.to === to); // act - const d = fromFlowGraph(graph, { tests: true, imports: true, title: 'everything' }); + const a = analyse(m); // assert - assert.equal(d.title, 'everything'); - assert.equal(d.source.omitted.testModules, 0); - assert.equal(d.source.omitted.importEdges, 0); - assert.deepEqual(d.source.graph, { v: null, generatedAt: null, nodes: 12, edges: 12 }); - assert.ok(d.nodes.some((n) => n.label === 'bin/test/tool.test.mjs')); - assert.deepEqual(d.flows.find((f) => f.label === 'imports'), { - from: 'node:module:bin/helper.mjs', to: 'node:module:bin/tool.mjs', label: 'imports', mode: 'sync', - evidence: { witness: 'static', existence: 'unknown', refs: ['bin/tool.mjs static-import'] }, - }); + assert.deepEqual(a.bidirectional.map((b) => [b.a, b.b]), [['customer', 'api']]); + assert.deepEqual(a.selfLoops, [{ node: 'ingest', flow: at('ingest', 'ingest') }]); + assert.deepEqual(a.feedback.map((f) => `${f.from}>${f.to} ${f.role}`), ['validate>transform error', 'validate>bronze retry']); + assert.deepEqual(a.cycles.map((c) => c.members), [['customer', 'api'], ['bronze', 'transform', 'enrich', 'validate']]); + assert.deepEqual(a.branches.map((b) => b.node), ['api', 'validate', 'silver']); + assert.deepEqual(a.merges.map((b) => b.node), ['api', 'bronze', 'transform', 'aggregate', 'gold']); + assert.deepEqual(a.brokered, [{ from: 'api', via: 'events', to: 'ingest' }, { from: 'api', via: 'audit', to: 'alerting' }]); + assert.deepEqual(a.asyncHops.map((h) => `${h.from}>${h.to} ${h.reason}`), [ + 'api>events mode+broker', 'events>ingest mode+broker', 'validate>bronze mode', 'nightly>aggregate mode', 'api>audit mode+broker', 'audit>alerting mode+broker', + ]); + assert.equal(a.paths.total, 8); + assert.equal(Math.max(...a.paths.listed.map((p) => p.length)), 13); + assert.equal(a.flowClass.filter((c) => c === 'forward').length, 21); + assert.deepEqual([...new Set(m.nodes.map((n) => n.subtype).filter(Boolean))].sort(), ['actor', 'database', 'endpoint', 'env', 'file', 'log', 'scheduler', 'system', 'topic']); + assert.deepEqual([...new Set(m.flows.map((f) => f.role).filter(Boolean))].sort(), ['data', 'error', 'retry']); }); // ── filter ─────────────────────────────────────────────────────────────────────────────────────── @@ -520,7 +527,7 @@ test('filterModel keeps a focus node and its neighbours, in either direction, an // assert assert.deepEqual(out.nodes.map((n) => n.id), ['events', 'ingest', 'bronze']); assert.equal(out.flows.length, 3); - assert.deepEqual(out.source.filter, { prefix: null, focus: 'Ingest', hops: 1, seeds: 1, nodesKept: 3, nodesOmitted: 13, flowsOmitted: 17 }); + assert.deepEqual(out.source.filter, { prefix: null, focus: 'Ingest', hops: 1, direction: 'both', seeds: 1, nodesKept: 3, nodesOmitted: 19, flowsOmitted: 23 }); assert.deepEqual(Object.keys(out).slice(0, 4), ['v', 'kind', 'title', 'source']); assert.deepEqual(validateDfd(out).errors, []); }); @@ -552,3 +559,42 @@ test('filterModel returns null when nothing matches, never an empty diagram', () // assert assert.equal(out, null); }); + +test('filterModel walks forward with out, backward with in, and both ways by default', () => { + // arrange + const m = model(['p1>s1', 's1>p2', 'p2>s2', 'e1>p2']); + + // act + const out = filterModel(m, { around: 'p2', hops: 2, direction: 'out' }); + const back = filterModel(m, { around: 'p2', hops: 2, direction: 'in' }); + const both = filterModel(m, { around: 'p2', hops: 1 }); + + // assert + assert.deepEqual(out.nodes.map((n) => n.id), ['p2', 's2']); + assert.deepEqual(back.nodes.map((n) => n.id), ['p1', 's1', 'p2', 'e1']); + assert.deepEqual(both.nodes.map((n) => n.id), ['s1', 'p2', 's2', 'e1']); + assert.deepEqual([out.source.filter.direction, back.source.filter.direction, both.source.filter.direction], ['out', 'in', 'both']); + assert.equal(out.source.filter.focus, 'p2'); +}); + +test('a directional walk still crosses an undirected flow, whose direction nobody determined', () => { + // arrange + const m = model(['p1>s1', 'p2--s1']); + + // act + const out = filterModel(m, { around: 's1', hops: 1, direction: 'out' }); + + // assert + assert.deepEqual(out.nodes.map((n) => n.id), ['s1', 'p2']); +}); + +test('filterModel refuses a direction it does not know', () => { + // arrange + const m = model(['p1>s1']); + + // act + const call = () => filterModel(m, { around: 'p1', direction: 'sideways' }); + + // assert + assert.throws(call, /direction must be one of in, out, both, got "sideways"/); +}); diff --git a/lib/test/fixtures/dfd-synthetic.json b/lib/test/fixtures/dfd-synthetic.json index e665753c9..87a839273 100644 --- a/lib/test/fixtures/dfd-synthetic.json +++ b/lib/test/fixtures/dfd-synthetic.json @@ -4,7 +4,7 @@ "title": "Order analytics (synthetic fixture)", "source": { "producer": "authored", - "note": "Hand-authored test fixture: every element kind and every flow topology the visualiser supports, in one model." + "note": "Hand-authored test fixture: every element kind, every subtype family, every flow role and every flow topology the visualiser supports, in one model. Elements after the analyst were added with the subtypes and roles; none before it was removed." }, "layers": ["bronze", "silver", "gold"], "groups": [ @@ -12,22 +12,28 @@ { "id": "platform", "label": "Order platform", "boundary": "system" } ], "nodes": [ - { "id": "customer", "kind": "external", "label": "Customer", "subtype": "user", "group": "internet" }, + { "id": "customer", "kind": "external", "label": "Customer", "subtype": "actor", "group": "internet" }, { "id": "api", "kind": "process", "label": "Order API", "group": "platform" }, - { "id": "orders-db", "kind": "store", "label": "Orders DB", "subtype": "relational", "group": "platform" }, + { "id": "orders-db", "kind": "store", "label": "Orders DB", "subtype": "database", "group": "platform" }, { "id": "events", "kind": "store", "label": "order-events", "subtype": "topic", "group": "platform", "note": "Kafka topic" }, { "id": "ingest", "kind": "process", "label": "Ingest", "layer": "bronze" }, - { "id": "bronze", "kind": "store", "label": "Raw orders", "subtype": "lake", "layer": "bronze" }, + { "id": "bronze", "kind": "store", "label": "Raw orders", "subtype": "file", "layer": "bronze" }, { "id": "transform", "kind": "process", "label": "Clean and conform", "layer": "silver" }, { "id": "enrich", "kind": "process", "label": "Enrich", "layer": "silver" }, { "id": "validate", "kind": "process", "label": "Validate quality", "layer": "silver" }, - { "id": "quarantine", "kind": "store", "label": "Quarantine", "subtype": "log", "layer": "silver" }, - { "id": "silver", "kind": "store", "label": "Conformed orders", "subtype": "lake", "layer": "silver" }, + { "id": "quarantine", "kind": "store", "label": "Quarantine", "subtype": "file", "layer": "silver" }, + { "id": "silver", "kind": "store", "label": "Conformed orders", "subtype": "file", "layer": "silver" }, { "id": "aggregate", "kind": "process", "label": "Aggregate", "layer": "gold" }, { "id": "features", "kind": "process", "label": "Build features", "layer": "gold" }, - { "id": "gold", "kind": "store", "label": "Gold marts", "subtype": "relational", "layer": "gold" }, + { "id": "gold", "kind": "store", "label": "Gold marts", "subtype": "database", "layer": "gold" }, { "id": "serve", "kind": "process", "label": "Report API" }, - { "id": "analyst", "kind": "external", "label": "BI analyst", "subtype": "user" } + { "id": "analyst", "kind": "external", "label": "BI analyst", "subtype": "actor" }, + { "id": "payments", "kind": "external", "label": "https://payments.example:443", "subtype": "endpoint" }, + { "id": "nightly", "kind": "external", "label": "cron: nightly rebuild", "subtype": "scheduler" }, + { "id": "db-url", "kind": "external", "label": "ORDERS_DB_URL", "subtype": "env" }, + { "id": "audit", "kind": "store", "label": "audit.log", "subtype": "log", "note": "appended by the API, tailed by alerting" }, + { "id": "alerting", "kind": "process", "label": "Alerting" }, + { "id": "pager", "kind": "external", "label": "Paging service", "subtype": "system" } ], "flows": [ { "from": "customer", "to": "api", "label": "order request", "mode": "sync" }, @@ -35,20 +41,26 @@ { "from": "api", "to": "orders-db", "label": "order row", "mode": "sync", "evidence": { "witness": "both", "existence": "confirmed", "refs": ["api/orders.mjs:41 insert"] } }, { "from": "api", "to": "events", "label": "OrderPlaced", "mode": "async" }, { "from": "events", "to": "ingest", "label": "OrderPlaced", "mode": "event" }, - { "from": "ingest", "to": "ingest", "label": "retry on timeout", "mode": "sync" }, + { "from": "ingest", "to": "ingest", "label": "retry on timeout", "mode": "sync", "role": "retry" }, { "from": "ingest", "to": "bronze", "label": "raw batch", "mode": "sync" }, { "from": "bronze", "to": "transform", "label": "raw batch", "mode": "sync" }, { "from": "transform", "to": "enrich", "label": "clean rows", "mode": "sync" }, { "from": "enrich", "to": "validate", "label": "candidate rows", "mode": "sync" }, - { "from": "validate", "to": "transform", "label": "validation errors", "mode": "sync" }, - { "from": "validate", "to": "bronze", "label": "replay request", "mode": "async", "evidence": { "witness": "authored", "existence": "unknown" } }, + { "from": "validate", "to": "transform", "label": "validation errors", "mode": "sync", "role": "error" }, + { "from": "validate", "to": "bronze", "label": "replay request", "mode": "async", "role": "retry", "evidence": { "witness": "authored", "existence": "unknown" } }, { "from": "validate", "to": "quarantine", "label": "rejected rows", "mode": "sync" }, { "from": "validate", "to": "silver", "label": "valid rows", "mode": "sync" }, { "from": "silver", "to": "aggregate", "label": "orders", "mode": "sync" }, { "from": "silver", "to": "features", "label": "orders", "mode": "sync" }, - { "from": "aggregate", "to": "gold", "label": "daily totals", "mode": "sync" }, + { "from": "aggregate", "to": "gold", "label": "daily totals", "mode": "sync", "role": "data", "direction": "writes", "existence": "confirmed", "evidence": { "witness": "both", "existence": "confirmed", "refs": ["aggregate.mjs:12 touch, resolved by the runtime"] } }, { "from": "features", "to": "gold", "label": "feature rows", "mode": "sync" }, { "from": "gold", "to": "serve", "label": "marts", "mode": "sync" }, - { "from": "serve", "to": "analyst", "label": "report", "mode": "sync" } + { "from": "serve", "to": "analyst", "label": "report", "mode": "sync" }, + { "from": "api", "to": "payments", "label": "charge", "mode": "sync", "existence": "unknown" }, + { "from": "nightly", "to": "aggregate", "label": "triggers", "mode": "event" }, + { "from": "db-url", "to": "api", "label": "connection string", "mode": "sync" }, + { "from": "api", "to": "audit", "label": "appends", "mode": "async" }, + { "from": "audit", "to": "alerting", "label": "tails", "mode": "async" }, + { "from": "alerting", "to": "pager", "label": "alert", "mode": "sync" } ] } diff --git a/schema/dfd.schema.json b/schema/dfd.schema.json index 20f4ca7f2..278118937 100644 --- a/schema/dfd.schema.json +++ b/schema/dfd.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://commitwork.online/schema/dfd.schema.json", "title": "dfd@1 — a source-independent data flow diagram model (lib/dfd.mjs)", - "description": "Three element kinds (process, store, external) and flows between them. A broker or queue is a store with subtype broker|queue|topic|stream. Referential integrity (unique ids, flow endpoints, declared layers and groups, v as the first field) is checked in code by validateDfd, not here.", + "description": "Three element kinds (process, store, external) and flows between them. A broker or queue is a store with subtype broker|queue|topic|stream|log. Referential integrity (unique ids, flow endpoints, declared layers and groups, which kind takes which subtype, v as the first field) is checked in code by validateDfd, not here. Every field added after the first release is optional, so a document written before it stays valid.", "type": "object", "required": ["v", "kind", "nodes", "flows"], "additionalProperties": false, @@ -49,7 +49,10 @@ "id": { "type": "string", "minLength": 1 }, "kind": { "enum": ["process", "store", "external"] }, "label": { "type": "string", "minLength": 1 }, - "subtype": { "type": "string", "minLength": 1 }, + "subtype": { + "description": "store: broker|queue|topic|stream|log (a broker: data waits in it between writer and reader) or database|file. external: endpoint (a network peer or a listening port), scheduler (launchd, a workflow schedule, cron, a timer, a watcher), actor (a person or an attacker), system (another program), env (configuration from outside), binary (an executable spawned by name). A process takes none. Which kind takes which is checked in validateDfd.", + "enum": ["broker", "queue", "topic", "stream", "log", "database", "file", "endpoint", "scheduler", "actor", "system", "env", "binary"] + }, "group": { "type": "string", "minLength": 1 }, "layer": { "type": "string", "minLength": 1 }, "unknown": { "type": "boolean" }, @@ -69,6 +72,18 @@ "label": { "type": "string" }, "mode": { "enum": ["sync", "async", "event"] }, "directed": { "type": "boolean" }, + "role": { + "description": "What the flow carries. Absent is data. feedback, error and retry are return paths: analysis classes them as feedback even when the opposite flow is adjacent, and only data flows collapse into a two-way pair.", + "enum": ["data", "feedback", "error", "retry"] + }, + "existence": { + "description": "The producer's verdict on whether the flow is real: confirmed only when a second witness (the runtime, for the flow-graph adapter) agreed. Distinct from evidence.existence, which is the source edge's own value carried unchanged.", + "enum": ["confirmed", "unknown"] + }, + "direction": { + "description": "On a flow made from an undirected touch, what the runtime saw the module do: reads (the flow runs into the module), writes (out of it) or both (two flows, one each way). Present only when the runtime resolved it; such a flow is directed.", + "enum": ["reads", "writes", "both"] + }, "evidence": { "type": "object", "required": ["existence"], From 2df882b2a4374f9b902bd166d89fdea6a9357a04 Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:39:48 +1030 Subject: [PATCH 08/36] refactor(dfd): move fromFlowGraph to flow/dfd-adapter.mjs so lib/ imports nothing from flow/ --- flow/dfd-adapter.mjs | 355 +++++++++++++++++++++++++++++++++++++++++++ flow/dfd-cli.mjs | 3 +- 2 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 flow/dfd-adapter.mjs diff --git a/flow/dfd-adapter.mjs b/flow/dfd-adapter.mjs new file mode 100644 index 000000000..b1b4d5e74 --- /dev/null +++ b/flow/dfd-adapter.mjs @@ -0,0 +1,355 @@ +// flow/dfd-adapter.mjs — the adapters that build a dfd@1 model (lib/dfd.mjs) from what this +// repository measures: the flow graph (flow/graph.mjs, with flow-reconcile.json beside it) and the +// CodeQL code-flow records of a scanned repository (cra/dataflow-evidence.mjs). +// +// They live in flow/ and not in lib/: lib/ ships and must stand alone, and the flow-graph adapter +// needs flow/static.mjs's idea of a test module. fromCodeFlows needs nothing from flow/; it is here +// beside its sibling because it consumes cra/dataflow-evidence.mjs's record shape and nothing else, +// so moving it to cra/ the day an installed command must draw a scanned repository is a file move. +// +// Pure functions. Nothing reads the clock, the environment or the disk. + +import { DFD_VERSION } from '../lib/dfd.mjs'; +import { isTestModule } from './static.mjs'; +import { edgeKey } from './graph.mjs'; + +// ── layer maps ─────────────────────────────────────────────────────────────────────────────────── + +/** + * A layer map assigns a maturity layer to a path: { layers: [names, in maturity order], rules: + * [{ prefix | glob, layer }] }. Rules are tried in order and the first match wins; a path no rule + * matches has no layer. A glob is `*` (within one segment), `**` (any depth, `**\/` also none) and + * `?` (one character). -> { ok, errors }, fail-closed: an unknown field is an error, not a skip. + */ +export function validateLayerMap(map) { + const errors = []; + if (!map || typeof map !== 'object' || Array.isArray(map)) return { ok: false, errors: ['layer map: not an object'] }; + for (const k of Object.keys(map)) if (!['$comment', 'v', 'kind', 'layers', 'rules'].includes(k)) errors.push(`layer map: unknown field ${JSON.stringify(k)}`); + if (map.v !== undefined && map.v !== 1) errors.push('layer map: v must be 1'); + if (map.kind !== undefined && map.kind !== 'dfd-layers') errors.push('layer map: kind must be "dfd-layers"'); + const layers = Array.isArray(map.layers) ? map.layers : null; + if (!layers || layers.length === 0 || !layers.every((l) => typeof l === 'string' && l.length > 0)) errors.push('layer map: layers must be a non-empty array of names'); + else if (new Set(layers).size !== layers.length) errors.push('layer map: a layer is declared more than once'); + if (!Array.isArray(map.rules) || map.rules.length === 0) errors.push('layer map: rules must be a non-empty array'); + else { + map.rules.forEach((r, i) => { + const keys = r && typeof r === 'object' ? Object.keys(r) : []; + const extra = keys.filter((k) => !['prefix', 'glob', 'layer', 'why'].includes(k)); + if (extra.length) errors.push(`layer map: rules[${i}] has unknown field(s) ${extra.join(', ')}`); + const has = ['prefix', 'glob'].filter((k) => typeof r?.[k] === 'string' && r[k].length > 0); + if (has.length !== 1) errors.push(`layer map: rules[${i}] needs exactly one of prefix and glob`); + if (layers && !layers.includes(r?.layer)) errors.push(`layer map: rules[${i}] names layer ${JSON.stringify(r?.layer)}, which "layers" does not declare`); + }); + } + return { ok: errors.length === 0, errors }; +} + +export function globToRegExp(glob) { + let re = ''; + for (let i = 0; i < glob.length; i++) { + const c = glob[i]; + if (c === '*' && glob[i + 1] === '*') { + if (glob[i + 2] === '/') { re += '(?:.*/)?'; i += 2; } else { re += '.*'; i += 1; } + } else if (c === '*') re += '[^/]*'; + else if (c === '?') re += '[^/]'; + else re += c.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + } + return new RegExp(`^${re}$`); +} + +const compiled = new WeakMap(); +const regexOf = (rule) => { if (!compiled.has(rule)) compiled.set(rule, globToRegExp(rule.glob)); return compiled.get(rule); }; + +/** -> the layer of `path` under `map`, or undefined. `map` may be a whole layer map or its rules. */ +export function layerOf(path, map) { + const rules = Array.isArray(map) ? map : map.rules; + const hit = rules.find((r) => (r.prefix !== undefined ? path.startsWith(r.prefix) : regexOf(r).test(path))); + return hit ? hit.layer : undefined; +} + +// ── the flow-graph adapter ─────────────────────────────────────────────────────────────────────── + +// A spawn target the static pass captured is the first argument of exec/spawn AS WRITTEN. `exec` is +// also a database method, so SQL text arrives here as a "process". Only a single token that reads as +// a command name is drawn as an external entity; the rest are counted and left out. +const COMMAND_NAME = /^[A-Za-z0-9_@.+/-]+$/; +const SYNC_VERB = /Sync$/; +const verbOf = (ref) => String(ref).split(' ').pop(); +const SCHEDULERS = { launchd: 'launchd agent', workflow: 'workflow schedule', cron: 'crontab', timer: 'in-process timer', watch: 'file watcher' }; +const RESOLVED = { reads: 'reads', read: 'reads', writes: 'writes', write: 'writes', both: 'both' }; + +/** + * Flow graph (flow/graph.mjs) -> dfd@1. The mapping: + * + * module -> process group = first path segment; unanalysable modules marked unknown + * artifact, store -> store subtype file; subtype log when one module appends to it and + * another tails it; a bare-filename node is marked unknown unless + * the reconcile's observedPaths names where it lives. A path that + * is also a module is that module's process, not a second element + * database -> store subtype database; table names go into the flow label + * env -> external subtype env + * endpoint -> external subtype endpoint (`<scheme>://<host>[:<port>]` or `listen:<port>`) + * scheduler -> external subtype scheduler (launchd, workflow schedule, cron, timer, watch) + * process (spawn) -> external subtype binary, when the target reads as a command name; + * -> the module's own process when the target is a module path; + * otherwise omitted and counted + * reads M <- X -> flow X -> M (tail:true from a log: async, "tails") + * writes M -> X -> flow M -> X (append:true: "appends"; async into a log) + * spawns M -> P -> flow M -> P, async unless every witness is a *Sync call; to the script's + * process when the spawn names one (`node <script>`), not to `node`; omitted + * and counted when every witness is a bare `exec` + * connects M -> E -> flow M -> E listens M -- E -> flow E -> M: a listener is fed from outside + * triggers S -> M -> flow S -> M, mode event + * touches M -- X -> flow M -- X, directed:false; with a runtime direction, directed and + * carrying `direction` (both: one flow each way) + * reads M <- M' -> an import: a code dependency, not a data flow. Omitted and counted unless + * include.imports. + * + * `reconcile` is flow-reconcile.json: a reconcile edge for the same from|to|kind replaces the edge's + * witness and existence, and the flow's `existence` is the adapter's verdict, confirmed only when + * that existence is. Nodes left with no flow are omitted and counted. `layers` is a layer map (see + * validateLayerMap) applied to the label of every process and store. + */ +export function fromFlowGraph(graph, { reconcile = null, layers = null, include = {}, tests, imports, title } = {}) { + const keepTests = include.tests ?? tests ?? false; + const keepImports = include.imports ?? imports ?? false; + // a bare rule list declares its layers in the order the rules first name them + const layerMap = !layers ? null : Array.isArray(layers) ? { layers: [...new Set(layers.map((r) => r?.layer))], rules: layers } : layers; + if (layerMap) { + const v = validateLayerMap(layerMap); + if (!v.ok) throw new Error(`dfd-adapter: ${v.errors.join('; ')}`); + } + const withLayer = (node) => { + if (!layerMap || node.kind === 'external') return node; + const layer = layerOf(node.label, layerMap); + return layer === undefined ? node : { ...node, layer }; + }; + + const src = new Map(graph.nodes.map((n) => [n.id, n])); + const modulePaths = new Set(graph.nodes.filter((n) => n.kind === 'module').map((n) => n.path)); + const recEdges = new Map(((reconcile && reconcile.edges) || []).map((e) => [edgeKey(e.from, e.to, e.kind), e])); + const observed = (reconcile && reconcile.observedPaths) || {}; + const appended = new Set(graph.edges.filter((e) => e.kind === 'writes' && e.append === true).map((e) => e.to)); + const tailed = new Set(graph.edges.filter((e) => e.kind === 'reads' && e.tail === true).map((e) => e.to)); + const logs = new Set([...appended].filter((id) => tailed.has(id))); + + const omitted = { testModules: 0, importEdges: 0, spawnTargetsNotCommands: 0, ambiguousExecEdges: 0, edgesOfOmittedNodes: 0, isolatedNodes: 0, selfReferences: 0, unknownNodeKinds: 0, unknownEdgeKinds: 0 }; + const merged = { pathsThatAreModules: 0, storesRenamed: 0, storesSplit: 0, directionsResolved: 0, reconcileMatched: 0 }; + const mapped = new Map(); // graph node id -> [dfd node]: a split store maps to several + + const fileStore = (n) => { + const subtype = logs.has(n.id) ? 'log' : 'file'; + const base = { kind: 'store', subtype }; + const paths = n.composed ? [].concat(observed[n.path] ?? observed[n.id] ?? []) : []; + if (n.composed && paths.length === 1) { + merged.storesRenamed++; + return [{ id: n.id, ...base, label: paths[0], note: `bare filename ${n.path}: the runtime opened it at ${paths[0]}` }]; + } + if (n.composed && paths.length > 1) { + merged.storesSplit++; + return paths.map((p) => ({ id: `${n.id}@${p}`, ...base, label: p, + note: `bare filename ${n.path}: one of ${paths.length} paths the runtime opened under that name; which module used which path was not observed` })); + } + return [{ id: n.id, ...base, label: n.path, + ...(n.composed ? { unknown: true, note: 'bare filename: a name, not a location — several files may share this node' } : {}) }]; + }; + + const mapNode = (n) => { + switch (n.kind) { + case 'module': { + if (!keepTests && isTestModule(n.path)) { omitted.testModules++; return []; } + const unknown = n.analysis !== undefined && n.analysis !== 'analysed'; + return [{ id: n.id, kind: 'process', label: n.path, group: n.path.includes('/') ? n.path.split('/')[0] : '(root)', + ...(unknown ? { unknown: true, note: `not analysed (${n.analysis}): its flows are unknown, not absent` } : {}) }]; + } + case 'artifact': + case 'store': + if (modulePaths.has(n.path)) { merged.pathsThatAreModules++; return mapped.get(`node:module:${n.path}`); } + return fileStore(n); + case 'database': return [{ id: n.id, kind: 'store', label: n.path, subtype: 'database' }]; + case 'env': return [{ id: n.id, kind: 'external', label: n.path, subtype: 'env' }]; + case 'endpoint': return [{ id: n.id, kind: 'external', label: n.path, subtype: 'endpoint', ...(n.path.startsWith('listen:') ? { note: `a port this tree listens on (${n.path.slice(7)})` } : {}) }]; + case 'scheduler': { + const how = SCHEDULERS[n.path.split(':')[0]]; + return [{ id: n.id, kind: 'external', label: n.path, subtype: 'scheduler', ...(how ? { note: how } : {}) }]; + } + case 'process': + if (modulePaths.has(n.path)) return mapped.get(`node:module:${n.path}`); + if (COMMAND_NAME.test(n.path)) return [{ id: n.id, kind: 'external', label: n.path, subtype: 'binary' }]; + omitted.spawnTargetsNotCommands++; + return []; + default: + omitted.unknownNodeKinds++; + return []; + } + }; + // modules first: a spawn target or a path literal that names a module resolves to its process + for (const n of graph.nodes) if (n.kind === 'module') mapped.set(n.id, mapNode(n)); + for (const n of graph.nodes) if (n.kind !== 'module') mapped.set(n.id, mapNode(n)); + + const flows = []; + const add = (from, to, f) => { + for (const a of from) for (const b of to) { + if (a.id === b.id) { omitted.selfReferences++; continue; } + const split = from.length > 1 || to.length > 1; + flows.push({ from: a.id, to: b.id, ...f, ...(split ? { existence: 'unknown' } : {}) }); + } + }; + const tables = (e) => (Array.isArray(e.tables) && e.tables.length ? ` ${e.tables.join(', ')}` : ''); + + for (const e of graph.edges) { + let from = mapped.get(e.from); + let to = mapped.get(e.to); + if (e.kind === 'spawns' && typeof e.script === 'string' && modulePaths.has(e.script)) to = mapped.get(`node:module:${e.script}`); + if (!from || !to || from.length === 0 || to.length === 0) { omitted.edgesOfOmittedNodes++; continue; } + const rec = recEdges.get(edgeKey(e.from, e.to, e.kind)); + if (rec) merged.reconcileMatched++; + const refs = [].concat(e.evidence || []); + const evidence = { witness: rec?.witness ?? e.witness, existence: rec?.existence ?? e.existence, refs }; + const base = { evidence, existence: evidence.existence === 'confirmed' ? 'confirmed' : 'unknown' }; + const toKind = src.get(e.to)?.kind; + if (e.kind === 'reads') { + if (toKind === 'module') { + if (!keepImports) { omitted.importEdges++; continue; } + add(to, from, { label: 'imports', mode: 'sync', ...base }); + } else if (e.tail === true && logs.has(e.to)) add(to, from, { label: 'tails', mode: 'async', ...base }); + else add(to, from, { label: `reads${tables(e)}`, mode: 'sync', ...base }); + } else if (e.kind === 'writes') { + if (e.append === true) add(from, to, { label: 'appends', mode: logs.has(e.to) ? 'async' : 'sync', ...base }); + else add(from, to, { label: `writes${tables(e)}`, mode: 'sync', ...base }); + } else if (e.kind === 'spawns') { + // a bare `exec` is child_process.exec or a database handle's exec: the witness cannot say which + if (refs.length > 0 && refs.every((r) => verbOf(r) === 'exec')) { omitted.ambiguousExecEdges++; continue; } + const sync = refs.length > 0 && refs.every((r) => SYNC_VERB.test(verbOf(r))); + const label = typeof e.script === 'string' && !modulePaths.has(e.script) ? `spawns ${e.script}` : 'spawns'; + add(from, to, { label, mode: sync ? 'sync' : 'async', ...base }); + } else if (e.kind === 'connects') add(from, to, { label: 'connects', mode: 'sync', ...base }); + else if (e.kind === 'listens') { + const moduleFirst = src.get(e.from)?.kind === 'module'; + add(moduleFirst ? to : from, moduleFirst ? from : to, { label: 'listens', mode: 'sync', ...base }); + } else if (e.kind === 'triggers') { + const schedFirst = src.get(e.from)?.kind === 'scheduler'; + add(schedFirst ? from : to, schedFirst ? to : from, { label: 'triggers', mode: 'event', ...base }); + } else if (e.kind === 'touches') { + const direction = RESOLVED[rec?.direction]; + if (!direction) { add(from, to, { label: 'touches', mode: 'sync', directed: false, ...base }); continue; } + merged.directionsResolved++; + if (direction !== 'writes') add(to, from, { label: 'reads', mode: 'sync', direction, ...base }); + if (direction !== 'reads') add(from, to, { label: 'writes', mode: 'sync', direction, ...base }); + } else omitted.unknownEdgeKinds++; + } + + const used = new Set(flows.flatMap((f) => [f.from, f.to])); + const nodes = []; + const seen = new Set(); + for (const list of mapped.values()) { + for (const m of list || []) { + if (seen.has(m.id)) continue; + seen.add(m.id); + if (used.has(m.id)) nodes.push(withLayer(m)); + else omitted.isolatedNodes++; + } + } + return { + v: DFD_VERSION, + kind: 'dfd', + title: title || 'flow graph', + source: { + producer: 'flow-graph', + graph: { v: graph.v ?? null, generatedAt: graph.generatedAt ?? null, nodes: graph.nodes.length, edges: graph.edges.length }, + options: { + tests: keepTests, imports: keepImports, + reconcile: reconcile ? { edges: recEdges.size, generatedAt: reconcile.generatedAt ?? null } : null, + layers: layerMap ? layerMap.rules.length : null, + merged, + }, + omitted, + }, + ...(layerMap ? { layers: [...layerMap.layers] } : {}), + nodes, + flows, + }; +} + +// ── the code-flow adapter ──────────────────────────────────────────────────────────────────────── + +// What a sink IS, read from the CodeQL rule id. Ordered: the first match wins. A rule no pattern +// names gets an external `system` marked unknown — the rule id did not say what the sink is. +const SINKS = [ + [/sql|nosql/, { kind: 'store', subtype: 'database', label: 'database query' }], + [/log-injection/, { kind: 'store', subtype: 'log', label: 'log' }], + [/path-injection|tainted-path|zipslip|file-system|file-access|file-write/, { kind: 'store', subtype: 'file', label: 'file system' }], + [/command|shell|exec/, { kind: 'external', subtype: 'binary', label: 'operating-system command' }], + [/request-forgery|ssrf|redirect/, { kind: 'external', subtype: 'endpoint', label: 'outbound request' }], + [/xss|html|dom/, { kind: 'external', subtype: 'actor', label: 'browser that renders the response' }], + [/code-injection|eval|deserial|prototype/, { kind: 'external', subtype: 'system', label: 'interpreter' }], +]; +export const sinkKindOf = (rule) => { + const hit = SINKS.find(([re]) => re.test(String(rule || '').toLowerCase())); + return hit ? { ...hit[1], category: hit[1].label } : { kind: 'external', subtype: 'system', label: 'unclassified sink', category: 'unclassified sink', unknown: true }; +}; +const at = (end) => `${end.file}:${end.line}`; + +/** + * cra/dataflow-evidence.mjs records (dataflowFromResult rows: { repo, rule, file, reachability, path: + * { steps, flows, source: { file, line, expr }, sink: { file, line, expr } } }) -> dfd@1. + * + * source -> external actor, one per repo and source location: the untrusted input + * source file -> process (group: the repo) + * sink file -> process, when it is not the source file + * sink -> store or external by what the rule says the sink is (sinkKindOf), one per + * repo and kind + * flows -> input -> source file -> sink file -> sink; the source-to-sink hop carries + * the step count and the rule + * + * A record carries the trace's two ends and its length, not the steps between: those are dropped by + * pathOf. A step in a third file is therefore not drawn. A record without a path (`reachability_ + * unknown`: the query never looked) draws nothing and is counted. Every flow is a static trace, so + * its existence is unknown: CodeQL proves data CAN reach the sink, and no runtime saw it do so. + */ +export function fromCodeFlows(records, { title } = {}) { + const omitted = { noPath: 0, unreadable: 0 }; + const nodes = new Map(); + const groups = new Map(); + const flows = new Map(); + const node = (n) => { if (!nodes.has(n.id)) nodes.set(n.id, n); return n.id; }; + const flow = (from, to, label, ref) => { + const key = JSON.stringify([from, to, label]); + if (!flows.has(key)) flows.set(key, { from, to, label, mode: 'sync', existence: 'unknown', evidence: { witness: 'static', existence: 'unknown', refs: [] } }); + const refs = flows.get(key).evidence.refs; + if (!refs.includes(ref)) refs.push(ref); + }; + for (const r of records || []) { + const p = r?.path; + if (!r || typeof r.repo !== 'string' || !r.rule) { omitted.unreadable++; continue; } + if (r.reachability !== 'reachable' || !p?.source?.file || !p?.sink?.file) { omitted.noPath++; continue; } + const group = `repo:${r.repo}`; + if (!groups.has(group)) groups.set(group, { id: group, label: r.repo, boundary: 'system' }); + if (!groups.has('untrusted')) groups.set('untrusted', { id: 'untrusted', label: 'Untrusted input', boundary: 'trust' }); + const input = node({ id: `input:${r.repo}:${at(p.source)}`, kind: 'external', subtype: 'actor', label: p.source.expr ? `${p.source.expr} (${at(p.source)})` : at(p.source), group: 'untrusted', + note: `a source CodeQL treats as untrusted in ${r.repo}` }); + const code = (file) => node({ id: `code:${r.repo}:${file}`, kind: 'process', label: file, group }); + const s = sinkKindOf(r.rule); + const sink = node({ id: `sink:${r.repo}:${s.category}`, kind: s.kind, subtype: s.subtype, label: s.label, + ...(s.unknown ? { unknown: true, note: `the rule id ${r.rule} does not say what its sink is` } : { note: `sink of ${r.repo}'s ${s.category} rules` }) }); + const ref = `${r.rule} ${at(p.source)} -> ${at(p.sink)}`; + const first = code(p.source.file); + flow(input, first, p.source.expr || 'untrusted input', ref); + let last = first; + if (p.sink.file !== p.source.file) { last = code(p.sink.file); flow(first, last, `${p.steps} steps (${r.rule})`, ref); } + flow(last, sink, p.sink.expr ? `${p.sink.expr} (${r.rule})` : r.rule, ref); + } + return { + v: DFD_VERSION, + kind: 'dfd', + title: title || 'code flows', + source: { + producer: 'codeql-code-flows', + note: 'CodeQL taint traces (cra/dataflow-evidence.mjs): static proofs that data can reach a sink, not runtime observations. A record holds its two ends, so a step in a third file is not drawn.', + omitted, + }, + groups: [...groups.values()], + nodes: [...nodes.values()], + flows: [...flows.values()], + }; +} diff --git a/flow/dfd-cli.mjs b/flow/dfd-cli.mjs index e97267d76..bf5ba5e0f 100644 --- a/flow/dfd-cli.mjs +++ b/flow/dfd-cli.mjs @@ -23,7 +23,8 @@ // without a path reads flow.json there) import { readJson, storePath } from './store.mjs'; -import { validateDfd, analyse, fromFlowGraph, filterModel } from '../lib/dfd.mjs'; +import { validateDfd, analyse, filterModel } from '../lib/dfd.mjs'; +import { fromFlowGraph } from './dfd-adapter.mjs'; import { renderHtml } from '../lib/dfd-render.mjs'; import { writeAtomic } from '../monitor/lockfile.mjs'; import { isMainModule } from '../lib/is-main.mjs'; From 367e720a626632d5f606696cab12e1f1e95e0449 Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:41:14 +1030 Subject: [PATCH 09/36] Place dfd labels clear of elements, transpose crossings, add TB orientation, subtypes, roles and group collapse --- lib/dfd-render.mjs | 588 ++++++++++++++++++++++++++++------- lib/test/dfd-render.test.mjs | 449 +++++++++++++++++++++++++- 2 files changed, 915 insertions(+), 122 deletions(-) diff --git a/lib/dfd-render.mjs b/lib/dfd-render.mjs index c8213d537..a5cc102b3 100644 --- a/lib/dfd-render.mjs +++ b/lib/dfd-render.mjs @@ -2,15 +2,21 @@ // // Notation: Gane–Sarson. A process is a rounded box with its reference in a top strip, a data store // is a box open at the right with its reference in a left compartment (D for a store, Q for a -// broker or queue), an external entity is a square box with a shadow. Solid line = sync, dashed = -// async or event, a head at both ends = bidirectional, a dotted line with no head = direction never -// determined. A feedback flow leaves the diagram's left-to-right order: it is routed over the top on -// its own track, heavier and in the secondary accent. A retry on one process is a loop on its corner. +// broker or queue), an external entity is a square box with a shadow. A store or external subtype +// the notation recognises carries a small mark at its right end (database cylinder, queue chevrons, +// log lines, file page; endpoint globe, scheduler clock, actor figure, system screen, env braces, +// binary prompt). Solid line = sync, dashed = async or event, a head at both ends = bidirectional, a +// dotted line with no head = direction never determined. A feedback flow leaves the diagram's rank +// order: it is routed outside the body on its own track, heavier and in the secondary accent. A +// retry on one process is a loop on its corner. A flow's role (error, retry) and its existence +// (confirmed, declared, unknown, contradicted) each change the line, never only its colour. // // Layout is a small layered (Sugiyama-style) pass and is deterministic: ranks by longest path, one -// virtual point per rank a long flow crosses, a fixed number of barycentre sweeps, then coordinates. -// Nothing reads the clock, so one model renders to the same bytes every time. Maturity layers -// (bronze, silver, gold) are horizontal bands; hop rank runs left to right. +// virtual point per rank a long flow crosses, a fixed number of barycentre sweeps, then adjacent +// transposition while it removes crossings, then coordinates, then label placement. Nothing reads +// the clock, so one model renders to the same bytes every time. Maturity layers (bronze, silver, +// gold) are bands across the rank axis. Ranks run left to right (LR) or top to bottom (TB); `auto` +// takes TB when the LR drawing is wider than AUTO_TB_WIDTH and TB is narrower. // // The page makes no request with `fonts: 'none'` or `'inline'`; it carries no script but the theme // follower, no style attribute, and a Content-Security-Policy that admits exactly its own bytes. @@ -24,14 +30,24 @@ import { FOLLOWER_JS } from './theme-follower.mjs'; import { analyse, isBroker, isDirected, longestPathRanks } from './dfd.mjs'; export const SWEEPS = 4; +export const TRANSPOSE_PASSES = 8; +export const AUTO_TB_WIDTH = 1600; +export const ORIENTATIONS = Object.freeze(['LR', 'TB', 'auto']); const NODE_H = 44; -const SLOT = 84; const CHAR = 6.7; +const ECHAR = 6.1; +const LINE = 12; const NODE_MIN = 124; const NODE_MAX = 260; const LABEL_MAX = Math.floor((NODE_MAX - 46) / CHAR); +const EDGE_LINE = 26; +const GLYPH_W = 16; const TRACK = 16; const LANE_PAD = 14; +// a feedback riser sits this far outside its column; in TB the one above a column clears the +// self-loop labels, which stand above their box +const RISER_OUT = 16; +const RISER_IN = { LR: 16, TB: 42 }; const sha256b64 = (s) => createHash('sha256').update(s, 'utf8').digest('base64'); const num = (x) => String(Math.round(x * 10) / 10); @@ -45,6 +61,30 @@ export function clip(label, max = LABEL_MAX) { return `${s.slice(0, head)}…${s.slice(s.length - (max - 1 - head))}`; } +const BREAK_AFTER = '/-.·,:'; + +/** + * At most `lines` lines of at most `max` characters. A line breaks at the last space, or after the + * last separator, that leaves it at least a third full; else it is cut hard. Only the last line is + * ever shortened, and in the middle, so a path keeps its head and its tail. + */ +export function wrap(label, max = LABEL_MAX, lines = 2) { + let rest = String(label); + const out = []; + while (rest.length > max && out.length < lines - 1) { + let cut = max; + let skip = 0; + for (let b = max; b >= Math.ceil(max / 3); b--) { + if (rest[b] === ' ') { cut = b; skip = 1; break; } + if (BREAK_AFTER.includes(rest[b - 1])) { cut = b; break; } + } + out.push(rest.slice(0, cut).trimEnd()); + rest = rest.slice(cut + skip).trimStart(); + } + out.push(clip(rest, max)); + return out; +} + /** A polyline with rounded corners, as an SVG path. */ export function roundedPath(pts, r = 6) { let d = `M${num(pts[0][0])},${num(pts[0][1])}`; @@ -62,30 +102,195 @@ export function roundedPath(pts, r = 6) { return `${d} L${num(last[0])},${num(last[1])}`; } -const curve = (x1, y1, x2, y2) => { - const k = (x2 - x1) / 2; - return `C${num(x1 + k)},${num(y1)} ${num(x2 - k)},${num(y2)} ${num(x2)},${num(y2)}`; -}; - const modeStyle = (modes) => (modes.every((m) => m === 'sync') ? 'sync' : modes.every((m) => m !== 'sync') ? 'async' : 'mixed'); -// contradicted outranks unknown outranks confirmed; a flow with no evidence at all is `declared` -function existenceOf(flows) { - const states = flows.map((f) => (f.evidence ? f.evidence.existence : 'declared')); +/** A flow's existence: its own field, else its evidence's, else `declared` (no evidence at all). */ +export const flowExistence = (f) => f.existence || (f.evidence ? f.evidence.existence : 'declared'); + +// contradicted outranks unknown outranks declared outranks confirmed +const existenceOf = (states) => { for (const s of ['contradicted', 'unknown', 'declared']) if (states.includes(s)) return s; return 'confirmed'; +}; +const ROLE_ORDER = ['error', 'retry', 'feedback']; +const roleOf = (roles) => ROLE_ORDER.find((r) => roles.includes(r)) || 'data'; +const ROLE_MARK = { error: '✕ ', retry: '↻ ' }; + +/** Every group with at least one member -> its member count, in first-member order. */ +export function groupSizes(model) { + const out = new Map(); + for (const nd of model.nodes) if (nd.group !== undefined) out.set(nd.group, (out.get(nd.group) || 0) + 1); + return out; +} + +/** + * Each named group becomes one process. Flows that cross its boundary merge per (other end, + * direction, mode, role, directedness), their labels joined and their count written `×n`; flows + * between two members are dropped and counted. A name with no member throws. + */ +export function collapseGroups(model, ids) { + const want = [...new Set(ids)]; + if (!want.length) return model; + const sizes = groupSizes(model); + const absent = want.filter((g) => !sizes.has(g)); + if (absent.length) throw new Error(`no element is in group ${absent.map((g) => JSON.stringify(g)).join(', ')}`); + const declared = new Map((model.groups || []).map((g) => [g.id, g])); + const taken = new Set(model.nodes.map((nd) => nd.id)); + const into = new Map(); + const info = new Map(); + for (const g of want) { + let id = `group:${g}`; + while (taken.has(id)) id += '~'; + taken.add(id); + const members = model.nodes.filter((nd) => nd.group === g); + const layers = [...new Set(members.map((nd) => nd.layer))]; + const label = declared.get(g)?.label || g; + info.set(g, { id, label, members: members.length, internal: 0, merged: 0, layer: layers.length === 1 ? layers[0] : undefined }); + for (const nd of members) into.set(nd.id, g); + } + const nodes = []; + for (const nd of model.nodes) { + const g = into.get(nd.id); + if (g === undefined) { nodes.push(nd); continue; } + const c = info.get(g); + if (nodes.some((x) => x.id === c.id)) continue; + nodes.push({ id: c.id, kind: 'process', label: `${c.label} (${c.members} elements)`, + ...(c.layer === undefined ? {} : { layer: c.layer }), collapsed: { group: g, members: c.members } }); + } + const end = (id) => (into.has(id) ? info.get(into.get(id)).id : id); + const flows = []; + const merged = new Map(); + for (const f of model.flows) { + const from = end(f.from); + const to = end(f.to); + if (from === f.from && to === f.to) { flows.push(f); continue; } + if (from === to && into.get(f.from) === into.get(f.to)) { info.get(into.get(f.from)).internal++; continue; } + const role = f.role || 'data'; + const key = JSON.stringify([from, to, f.mode, role, isDirected(f)]); + if (!merged.has(key)) { + const m = { from, to, mode: f.mode, ...(role === 'data' ? {} : { role }), ...(isDirected(f) ? {} : { directed: false }), parts: [] }; + merged.set(key, m); + flows.push(m); + } + merged.get(key).parts.push(f); + for (const id of [f.from, f.to]) if (into.has(id)) info.get(into.get(id)).merged++; + } + const out = flows.map((f) => { + if (!f.parts) return f; + const { parts, ...rest } = f; + const labels = [...new Set(parts.map((p) => p.label).filter(Boolean))]; + const existence = existenceOf(parts.map(flowExistence)); + const refs = parts.flatMap((p) => (p.evidence && p.evidence.refs) || []); + return { ...rest, + ...(labels.length || parts.length > 1 ? { label: `${labels.join(' / ')}${parts.length > 1 ? ` ×${parts.length}` : ''}`.trim() } : {}), + ...(existence === 'declared' ? {} : { evidence: { existence, ...(refs.length ? { refs } : {}) } }), + merged: parts.length }; + }); + return { ...model, nodes, flows: out, collapsedGroups: want.map((g) => ({ group: g, ...info.get(g) })) }; +} + +// ── layout ───────────────────────────────────────────────────────────────────────────────────── + +/** Crossings between consecutive ranks, counted on the segments the ranks' items join. */ +export function countCrossings(cols) { + let c = 0; + for (const col of cols) { + const segs = []; + for (const it of col) for (const o of it.down) segs.push([it.pos, o.pos]); + for (let i = 0; i < segs.length; i++) { + for (let j = i + 1; j < segs.length; j++) if ((segs[i][0] - segs[j][0]) * (segs[i][1] - segs[j][1]) < 0) c++; + } + } + return c; +} + +const crossPair = (p, q) => { + let c = 0; + for (const side of ['up', 'down']) for (const x of p[side]) for (const y of q[side]) if (x.pos > y.pos) c++; + return c; +}; + +// adjacent transposition: swap two neighbours while that removes crossings, inside one lane, and +// never splitting a group that has more than one member in the rank +function transpose(cols) { + for (let pass = 0; pass < TRANSPOSE_PASSES; pass++) { + let improved = false; + for (const col of cols) { + const count = new Map(); + for (const it of col) count.set(it.group, (count.get(it.group) || 0) + 1); + for (let i = 0; i + 1 < col.length; i++) { + const p = col[i]; + const q = col[i + 1]; + const free = p.lane === q.lane && (p.group === q.group || (count.get(p.group) === 1 && count.get(q.group) === 1)); + if (free && crossPair(q, p) < crossPair(p, q)) { + col[i] = q; col[i + 1] = p; q.pos = i; p.pos = i + 1; + improved = true; + } + } + } + if (!improved) break; + } +} + +const hits = (a, b) => a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h; +const grow = (r, k) => ({ x: r.x - k, y: r.y - k, w: r.w + 2 * k, h: r.h + 2 * k }); + +// points along a sequence of path pieces; a cubic is sampled, a line is its two ends +function sample(pieces) { + const pts = [pieces[0].p]; + for (const s of pieces.slice(1)) { + if (s.c) { + const [p0] = [pts[pts.length - 1]]; + const [c1, c2, p3] = [s.c[0], s.c[1], s.p]; + for (let k = 1; k <= 8; k++) { + const t = k / 8; + const a = (1 - t) ** 3; const b = 3 * (1 - t) ** 2 * t; const c = 3 * (1 - t) * t * t; const d = t ** 3; + pts.push([a * p0[0] + b * c1[0] + c * c2[0] + d * p3[0], a * p0[1] + b * c1[1] + c * c2[1] + d * p3[1]]); + } + } else pts.push(s.p); + } + return pts; } +function along(pts, frac) { + const lens = [0]; + for (let i = 1; i < pts.length; i++) lens.push(lens[i - 1] + Math.hypot(pts[i][0] - pts[i - 1][0], pts[i][1] - pts[i - 1][1])); + const want = lens[lens.length - 1] * frac; + let i = 1; + while (i < pts.length - 1 && lens[i] < want) i++; + const seg = lens[i] - lens[i - 1] || 1; + const t = (want - lens[i - 1]) / seg; + return [pts[i - 1][0] + (pts[i][0] - pts[i - 1][0]) * t, pts[i - 1][1] + (pts[i][1] - pts[i - 1][1]) * t]; +} + +const FRACTIONS = [0.5, 0.4, 0.6, 0.3, 0.7, 0.2, 0.8, 0.12, 0.88]; +const NUDGES = [8, 16, 24, 32].flatMap((d) => [[0, -d], [0, d], [d, 0], [-d, 0]]); +const labelSize = (lines) => ({ w: Math.max(...lines.map((l) => l.length)) * ECHAR + 4, h: lines.length * LINE }); + /** * Model + analysis -> geometry. Every flow lands in exactly one drawn edge (`edges[].flows`), and - * flows between the same two nodes share one edge. + * flows between the same two nodes share one edge. `crossings` counts crossings between consecutive + * ranks; `collisions` counts edge labels that could not be placed clear of every element and label. */ -export function layout(model, analysis) { +export function layout(model, analysis, { orientation = 'auto', transpose: doTranspose = true } = {}) { + if (!ORIENTATIONS.includes(orientation)) throw new Error(`orientation is one of ${ORIENTATIONS.join(', ')}, got ${orientation}`); + if (orientation !== 'auto') return build(model, analysis, orientation, doTranspose); + const lr = build(model, analysis, 'LR', doTranspose); + if (lr.width <= AUTO_TB_WIDTH) return lr; + const tb = build(model, analysis, 'TB', doTranspose); + return tb.width < lr.width ? tb : lr; +} + +const glyphOf = (nd) => (nd.kind === 'store' ? STORE_GLYPH : nd.kind === 'external' ? EXT_GLYPH : {})[nd.subtype]; + +function build(model, analysis, orientation, doTranspose) { + const TB = orientation === 'TB'; const nodes = model.nodes; const n = nodes.length; const ix = new Map(nodes.map((nd, i) => [nd.id, i])); const flows = model.flows; const aRank = analysis.nodes.map((a) => a.rank); + const P = (u, v) => (TB ? [v, u] : [u, v]); const counters = { P: 0, D: 0, Q: 0, E: 0 }; const refs = nodes.map((nd) => { @@ -108,8 +313,11 @@ export function layout(model, analysis) { b.cls = b.self ? 'self' : classes.includes('bidirectional') ? 'bi' : classes.includes('feedback') ? 'fb' : classes.includes('forward') ? 'fwd' : 'und'; b.mode = modeStyle(b.flows.map((k) => flows[k].mode)); - b.existence = existenceOf(b.flows.map((k) => flows[k])); - b.label = clip([...new Set(b.flows.map((k) => flows[k].label).filter(Boolean))].join(' / '), 44); + b.existence = existenceOf(b.flows.map((k) => flowExistence(flows[k]))); + b.role = roleOf(b.flows.map((k) => flows[k].role || 'data')); + b.label = [...new Set(b.flows.map((k) => flows[k].label).filter(Boolean))].join(' / '); + const shown = b.label ? `${ROLE_MARK[b.role] || ''}${b.label}` : ''; + b.lines = shown ? wrap(shown, EDGE_LINE) : []; } // ── ranks: the analysis layering, then undirected pairs oriented along its topological order ──── @@ -181,100 +389,124 @@ export function layout(model, analysis) { settle(col); } } - - // ── coordinates ─────────────────────────────────────────────────────────────────────────────── - const widths = nodes.map((nd) => Math.max(NODE_MIN, Math.min(NODE_MAX, Math.round(clip(nd.label).length * CHAR + 46)))); - const colW = cols.map((col) => Math.max(24, ...col.filter((it) => it.node >= 0).map((it) => widths[it.node]))); - // a gutter holds the feedback risers and the label of every flow that starts on its left - const gutters = cols.map(() => Math.max(96, 52 + 12 * feedbacks.length)); + if (doTranspose) transpose(cols); + const crossings = countCrossings(cols); + + // ── coordinates, as (u along the ranks, v across them); P() turns them into (x, y) ─────────── + const lines = nodes.map((nd) => wrap(nd.label)); + const widths = nodes.map((nd, i) => Math.max(NODE_MIN, Math.min(NODE_MAX + GLYPH_W, + Math.round(Math.max(...lines[i].map((l) => l.length)) * CHAR + 46 + (glyphOf(nd) ? GLYPH_W : 0))))); + const du = (i) => (TB ? NODE_H : widths[i]); + const dv = (i) => (TB ? widths[i] : NODE_H); + const fb = feedbacks.length; + const riserIn = RISER_IN[orientation]; + const colSize = cols.map((col) => Math.max(24, ...col.filter((it) => it.node >= 0).map((it) => du(it.node)))); + // a gutter holds the feedback risers and the label of every flow that starts on its near side + const gutters = cols.map(() => (TB ? Math.max(76, fb ? RISER_OUT + riserIn + 12 * fb + 8 : 0) : Math.max(96, 52 + 12 * fb))); for (const b of bundles) { - if (!b.self && b.cls !== 'fb') gutters[rank[b.a]] = Math.max(gutters[rank[b.a]], Math.round(b.label.length * 6.1 + 30)); + if (b.self || b.cls === 'fb' || !b.lines.length) continue; + const s = labelSize(b.lines); + gutters[rank[b.a]] = Math.max(gutters[rank[b.a]], Math.round(TB ? s.h + 44 : s.w + 26)); } - const left = Math.max(40, 28 + 6 * feedbacks.length) + (layerNames.length ? 70 : 0); - const colX = []; - colW.forEach((w, r) => { colX.push(r === 0 ? left : colX[r - 1] + colW[r - 1] + gutters[r - 1]); }); - const width = colX[ranks - 1] + colW[ranks - 1] + Math.max(40, 28 + 6 * feedbacks.length); - const bodyTop = 22 + TRACK * feedbacks.length + 26; - + const left = TB ? Math.max(40, riserIn + 6 * fb + 8) + (layerNames.length ? 22 : 0) + : Math.max(40, 28 + 6 * fb) + (layerNames.length ? 70 : 0); + const colU = []; + colSize.forEach((w, r) => { colU.push(r === 0 ? left : colU[r - 1] + colSize[r - 1] + gutters[r - 1]); }); + const uEnd = colU[ranks - 1] + colSize[ranks - 1] + Math.max(40, 28 + 6 * fb); + const bodyTop = 22 + TRACK * fb + 26; + + const slot = laneNames.map((_, l) => { + const inLane = items.filter((it) => it.lane === l).map((it) => dv(it.node)); + return TB ? Math.max(NODE_MIN, ...inLane) + 36 : NODE_H + 40; + }); const rows = laneNames.map((_, l) => Math.max(1, ...cols.map((col) => col.filter((it) => it.lane === l).length))); const laneTop = []; - rows.forEach((_, l) => { laneTop.push(l === 0 ? bodyTop : laneTop[l - 1] + rows[l - 1] * SLOT + 2 * LANE_PAD); }); - const height = laneTop[rows.length - 1] + rows[rows.length - 1] * SLOT + 2 * LANE_PAD + 8; + rows.forEach((_, l) => { laneTop.push(l === 0 ? bodyTop : laneTop[l - 1] + rows[l - 1] * slot[l - 1] + 2 * LANE_PAD); }); + const vEnd = laneTop[rows.length - 1] + rows[rows.length - 1] * slot[rows.length - 1] + 2 * LANE_PAD + 8; for (const col of cols) { for (let l = 0; l < laneNames.length; l++) { const inLane = col.filter((it) => it.lane === l); - inLane.forEach((it, k) => { it.cy = laneTop[l] + LANE_PAD + ((rows[l] - inLane.length) / 2 + k + 0.5) * SLOT; }); + inLane.forEach((it, k) => { it.cv = laneTop[l] + LANE_PAD + ((rows[l] - inLane.length) / 2 + k + 0.5) * slot[l]; }); } } - const boxes = nodes.map((nd, i) => ({ - id: nd.id, ref: refs[i], kind: nd.kind, rank: rank[i], lane: laneNames[items[i].lane], - x: colX[rank[i]] + (colW[rank[i]] - widths[i]) / 2, y: items[i].cy - NODE_H / 2, w: widths[i], h: NODE_H, - })); + const [width, height] = P(uEnd, vEnd); + const uvBox = nodes.map((nd, i) => ({ u: colU[rank[i]] + (colSize[rank[i]] - du(i)) / 2, v: items[i].cv - dv(i) / 2, du: du(i), dv: dv(i) })); + const boxes = nodes.map((nd, i) => { + const [x, y] = P(uvBox[i].u, uvBox[i].v); + const [w, h] = P(uvBox[i].du, uvBox[i].dv); + return { id: nd.id, ref: refs[i], kind: nd.kind, rank: rank[i], lane: laneNames[items[i].lane], x, y, w, h, lines: lines[i] }; + }); - // ── ports: flows leave on the right and arrive on the left, spread so they do not share a point ─ + // ── ports: flows leave on the far side and arrive on the near side, spread apart ─────────────── const outs = nodes.map(() => []); const ins = nodes.map(() => []); bundles.forEach((b, bi) => { if (b.self) return; if (b.cls === 'fb') { outs[b.from].push({ bi, toward: -Infinity }); ins[b.to].push({ bi, toward: -Infinity }); return; } - outs[b.a].push({ bi, toward: b.chain[1].cy }); - ins[b.b].push({ bi, toward: b.chain[b.chain.length - 2].cy }); + outs[b.a].push({ bi, toward: b.chain[1].cv }); + ins[b.b].push({ bi, toward: b.chain[b.chain.length - 2].cv }); }); - const portY = new Map(); + const portV = new Map(); const spread = (list, i, side) => { list.sort((p, q) => (p.toward - q.toward) || (p.bi - q.bi)); - const step = list.length > 1 ? Math.min(9, (NODE_H - 14) / (list.length - 1)) : 0; - list.forEach((p, k) => portY.set(`${side}${p.bi}`, items[i].cy + (k - (list.length - 1) / 2) * step)); + const step = list.length > 1 ? (TB ? Math.min(16, (widths[i] - 24) / (list.length - 1)) : Math.min(9, (NODE_H - 14) / (list.length - 1))) : 0; + list.forEach((p, k) => portV.set(`${side}${p.bi}`, items[i].cv + (k - (list.length - 1) / 2) * step)); }; outs.forEach((list, i) => spread(list, i, 'o')); ins.forEach((list, i) => spread(list, i, 'i')); // ── edges ───────────────────────────────────────────────────────────────────────────────────── + const xy = (pt) => `${num(pt[0])},${num(pt[1])}`; const edges = bundles.map((b, bi) => { - const base = { flows: b.flows, cls: b.cls, mode: b.mode, existence: b.existence, label: b.label }; + const base = { flows: b.flows, cls: b.cls, mode: b.mode, existence: b.existence, role: b.role, label: b.label, lines: b.lines }; if (b.self) { const bx = boxes[b.lo]; const x1 = bx.x + bx.w - 36; const x2 = bx.x + bx.w - 10; - return { ...base, from: bx.id, to: bx.id, heads: 'end', lx: x2 + 6, ly: bx.y - 9, anchor: 'start', + // the label stands above its own box, ending at the loop: never in a gutter a riser uses + const shown = b.label ? wrap(`${ROLE_MARK[b.role] || ''}${b.label}`, Math.max(8, Math.floor((x1 - 4 - bx.x) / ECHAR))) : []; + return { ...base, lines: shown, from: bx.id, to: bx.id, heads: 'end', anchor: 'end', self: { x: x1 - 4, y: bx.y - 4 }, d: `M${num(x1)},${num(bx.y)} C${num(x1)},${num(bx.y - 26)} ${num(x2)},${num(bx.y - 26)} ${num(x2)},${num(bx.y)}` }; } if (b.cls === 'fb') { - const u = boxes[b.from]; - const v = boxes[b.to]; - const yt = 22 + TRACK * b.track; - const xr = colX[u.rank] + colW[u.rank] + 16 + 6 * b.track; - const xl = colX[v.rank] - 16 - 6 * b.track; - const y1 = portY.get(`o${bi}`); - const y2 = portY.get(`i${bi}`); - return { ...base, from: u.id, to: v.id, heads: 'end', lx: (xr + xl) / 2, ly: yt - 4, anchor: 'middle', - d: roundedPath([[u.x + u.w, y1], [xr, y1], [xr, yt], [xl, yt], [xl, y2], [v.x, y2]]) }; + const u = uvBox[b.from]; + const v = uvBox[b.to]; + const vt = 22 + TRACK * b.track; + const ur = colU[rank[b.from]] + colSize[rank[b.from]] + RISER_OUT + 6 * b.track; + const ul = colU[rank[b.to]] - riserIn - 6 * b.track; + const v1 = portV.get(`o${bi}`); + const v2 = portV.get(`i${bi}`); + const pts = [P(u.u + u.du, v1), P(ur, v1), P(ur, vt), P(ul, vt), P(ul, v2), P(v.u, v2)]; + return { ...base, from: nodes[b.from].id, to: nodes[b.to].id, heads: 'end', anchor: 'middle', track: TB ? pts[2][0] : pts[2][1], + risers: [[pts[1], pts[2]], [pts[3], pts[4]]], pts, d: roundedPath(pts) }; } - const a = boxes[b.a]; - const z = boxes[b.b]; - let x = a.x + a.w; - let y = portY.get(`o${bi}`); - let d = `M${num(x)},${num(y)}`; - let lx = null; - let ly = null; - const hop = (x2, y2) => { - if (lx === null) { lx = (x + x2) / 2; ly = (y + y2) / 2 - 5; } - d += ` ${curve(x, y, x2, y2)}`; - x = x2; y = y2; + const a = uvBox[b.a]; + const z = uvBox[b.b]; + let u = a.u + a.du; + let v = portV.get(`o${bi}`); + const pieces = [{ p: P(u, v) }]; + const hop = (u2, v2) => { + const k = (u2 - u) / 2; + pieces.push({ c: [P(u + k, v), P(u2 - k, v2)], p: P(u2, v2) }); + u = u2; v = v2; }; for (const dummy of b.chain.slice(1, -1)) { - hop(colX[dummy.rank], dummy.cy); - x = colX[dummy.rank] + colW[dummy.rank]; - d += ` L${num(x)},${num(y)}`; + hop(colU[dummy.rank], dummy.cv); + u = colU[dummy.rank] + colSize[dummy.rank]; + pieces.push({ p: P(u, v) }); } - hop(z.x, portY.get(`i${bi}`)); + hop(z.u, portV.get(`i${bi}`)); + const d = pieces.map((s, i) => (i === 0 ? `M${xy(s.p)}` : s.c ? `C${xy(s.c[0])} ${xy(s.c[1])} ${xy(s.p)}` : `L${xy(s.p)}`)).join(' '); // a forward flow always runs from the lower rank, so only a two-way pair takes a head at its start - return { ...base, from: a.id, to: z.id, heads: b.cls === 'bi' ? 'both' : b.cls === 'fwd' ? 'end' : 'none', lx, ly, anchor: 'middle', d }; + return { ...base, from: nodes[b.a].id, to: nodes[b.b].id, heads: b.cls === 'bi' ? 'both' : b.cls === 'fwd' ? 'end' : 'none', anchor: 'middle', pts: sample(pieces), d }; }); // ── bands and boundary hulls ────────────────────────────────────────────────────────────────── - const bands = laneNames.map((name, l) => ({ name, y: laneTop[l], h: rows[l] * SLOT + 2 * LANE_PAD })) - .filter((b) => b.name !== null); + const bands = laneNames.map((name, l) => { + const [x, y] = P(4, laneTop[l]); + const [w, h] = P(TB ? height - 8 : width - 8, rows[l] * slot[l] + 2 * LANE_PAD); + return { name, x, y, w, h }; + }).filter((b) => b.name !== null); const declared = new Map((model.groups || []).map((g) => [g.id, g])); const hulls = []; const loose = new Set(); @@ -285,60 +517,142 @@ export function layout(model, analysis) { x: Math.min(...inside.map((b) => b.x)) - 10, y: Math.min(...inside.map((b) => b.y)) - 20, x2: Math.max(...inside.map((b) => b.x + b.w)) + 10, y2: Math.max(...inside.map((b) => b.y + b.h)) + 10, }; - const hits = (r) => r.x < hull.x2 && r.x2 > hull.x && r.y < hull.y2 && r.y2 > hull.y; - const blocked = boxes.some((b, i) => nodes[i].group !== g && hits({ x: b.x, y: b.y, x2: b.x + b.w, y2: b.y + b.h })) || hulls.some(hits); + const meets = (r) => r.x < hull.x2 && r.x2 > hull.x && r.y < hull.y2 && r.y2 > hull.y; + const blocked = boxes.some((b, i) => nodes[i].group !== g && meets({ x: b.x, y: b.y, x2: b.x + b.w, y2: b.y + b.h })) || hulls.some(meets); // a boundary that cannot be drawn without enclosing a stranger is named under each member instead if (blocked) loose.add(g); else hulls.push(hull); } boxes.forEach((b, i) => { if (loose.has(nodes[i].group)) b.caption = declared.get(nodes[i].group)?.label || nodes[i].group; }); - return { width, height, nodes: boxes, edges, bands, hulls }; + // ── labels: self-loops, then feedback, then the rest; first clear spot along the edge ───────── + const obstacles = [ + ...boxes.map((b) => grow({ x: b.x, y: b.y, w: b.w, h: b.h }, 2)), + ...boxes.filter((b) => b.caption).map((b) => ({ x: b.x + b.w / 2 - 50, y: b.y + b.h + 2, w: 100, h: 12 })), + ...hulls.map((h) => ({ x: h.x + 4, y: h.y + 2, w: Math.min(h.x2 - h.x - 8, h.label.length * ECHAR + 4), h: 12 })), + // a riser is a line, but one a label would sit on for its whole width + ...edges.filter((e) => e.risers).flatMap((e) => e.risers.map(([[x1, y1], [x2, y2]]) => ({ x: Math.min(x1, x2) - 2, y: Math.min(y1, y2) - 2, w: Math.abs(x2 - x1) + 4, h: Math.abs(y2 - y1) + 4 }))), + ]; + const placed = []; + const inside = (r) => r.x >= 2 && r.y >= 2 && r.x + r.w <= width - 2 && r.y + r.h <= height - 2; + const clear = (r) => inside(r) && !obstacles.some((o) => hits(o, r)) && !placed.some((o) => hits(grow(o, 1), r)); + let collisions = 0; + const put = (e, candidates, nudge) => { + let box = candidates.find(clear); + if (!box && nudge) { + for (const r of candidates) { + box = NUDGES.map(([dx, dy]) => ({ ...r, x: r.x + dx, y: r.y + dy })).find(clear); + if (box) break; + } + } + if (!box) { box = candidates[0]; collisions++; } + placed.push(box); + e.lbox = box; + e.lx = e.anchor === 'end' ? box.x + box.w - 2 : box.x + box.w / 2; + e.ly = box.y + LINE - 3; + }; + const order = [...edges.keys()].sort((p, q) => (rankOf(edges[p]) - rankOf(edges[q])) || (p - q)); + function rankOf(e) { return e.cls === 'self' ? 0 : e.cls === 'fb' ? 1 : 2; } + for (const k of order) { + const e = edges[k]; + if (!e.lines.length) continue; + const s = labelSize(e.lines); + if (e.self) { + const at = { x: e.self.x - s.w, y: e.self.y - s.h, w: s.w, h: s.h }; + put(e, (TB ? [0, -6] : [0, -6, -12]).map((dy) => ({ ...at, y: at.y + dy })), false); + } else { + put(e, FRACTIONS.map((f) => { + const [px, py] = along(e.pts, f); + return { x: px - s.w / 2, y: py - s.h / 2, w: s.w, h: s.h }; + }), true); + } + } + for (const e of edges) { delete e.pts; delete e.self; } + + return { orientation, width, height, crossings, collisions, nodes: boxes, edges, bands, hulls, ranks: cols.map((col) => col.map((it) => it.node)) }; } // ── SVG ──────────────────────────────────────────────────────────────────────────────────────── -const MARKERS = ['ink', 'mut', 'fb', 'bad'].map((c) => `<marker id="dfd-a-${c}" viewBox="0 0 10 8" refX="9" refY="4" markerWidth="10" markerHeight="8" orient="auto-start-reverse" markerUnits="userSpaceOnUse"><path class="mk-${c}" d="M0,0 L10,4 L0,8 z"/></marker>`).join(''); - -const markerOf = (e) => (e.existence === 'contradicted' ? 'bad' : e.cls === 'fb' ? 'fb' : e.existence === 'unknown' ? 'mut' : 'ink'); +const ring = (cx, cy, r) => `M${cx - r},${cy} a${r},${r} 0 1,0 ${2 * r},0 a${r},${r} 0 1,0 ${-2 * r},0`; +// each mark fits a 12 x 14 box whose top-left corner is (0, 0) +const STORE_GLYPH = Object.freeze({ + database: 'M0,3 a6,2.5 0 0,0 12,0 a6,2.5 0 0,0 -12,0 v8 a6,2.5 0 0,0 12,0 v-8', + broker: 'M0,2 l4,5 l-4,5 M4,2 l4,5 l-4,5 M8,2 l4,5 l-4,5', + queue: 'M0,2 l4,5 l-4,5 M4,2 l4,5 l-4,5 M8,2 l4,5 l-4,5', + topic: 'M0,2 l4,5 l-4,5 M4,2 l4,5 l-4,5 M8,2 l4,5 l-4,5', + stream: 'M0,2 l4,5 l-4,5 M4,2 l4,5 l-4,5 M8,2 l4,5 l-4,5', + log: 'M0,3 h12 M0,7 h12 M0,11 h8', + file: 'M1,1 h7 l4,4 v8 h-11 z M8,1 v4 h4', +}); +const EXT_GLYPH = Object.freeze({ + endpoint: `${ring(6, 7, 6)} M6,1 a2.5,6 0 0,0 0,12 a2.5,6 0 0,0 0,-12 M0,7 h12`, + scheduler: `${ring(6, 7, 6)} M6,3 V7 H9`, + actor: `${ring(6, 3.5, 2.5)} M1,13 a5,5 0 0,1 10,0`, + system: 'M0,1 h12 v8 h-12 z M3,12.5 h6 M6,9 v3.5', + env: 'M4,1 q-2,0 -2,2 v2.5 l-2,1.5 l2,1.5 v2.5 q0,2 2,2 M8,1 q2,0 2,2 v2.5 l2,1.5 l-2,1.5 v2.5 q0,2 -2,2', + binary: 'M1,3 l4,4 l-4,4 M6,12 h6', +}); +const glyphSvg = (d, x, y) => `<path class="gl" transform="translate(${num(x)},${num(y)})" d="${d}"/>`; + +const MARKERS = ['ink', 'mut', 'fb', 'bad', 'hi'].map((c) => `<marker id="dfd-a-${c}" viewBox="0 0 10 8" refX="9" refY="4" markerWidth="10" markerHeight="8" orient="auto-start-reverse" markerUnits="userSpaceOnUse"><path class="mk-${c}" d="M0,0 L10,4 L0,8 z"/></marker>`).join(''); + +const markerOf = (e) => (e.existence === 'contradicted' ? 'bad' : e.role === 'error' ? 'hi' : e.cls === 'fb' ? 'fb' : e.existence === 'unknown' ? 'mut' : 'ink'); + +// one <text> per label, a <tspan> per line +const textLines = (cls, x, y, anchor, ls, step = LINE) => `<text${cls ? ` class="${cls}"` : ''} x="${num(x)}" y="${num(y)}" text-anchor="${anchor}">${ls.length === 1 ? esc(ls[0]) + : ls.map((l, i) => `<tspan x="${num(x)}"${i ? ` dy="${step}"` : ''}>${esc(l)}</tspan>`).join('')}</text>`; function nodeSvg(b, nd) { - const cx = b.x + b.w / 2; - const tip = `${b.ref} ${nd.kind}${nd.subtype ? ` (${nd.subtype})` : ''}: ${nd.label}${nd.group ? ` — boundary ${nd.group}` : ''}${nd.layer ? ` — layer ${nd.layer}` : ''}${nd.note ? ` — ${nd.note}` : ''}`; - const cls = `n ${nd.kind === 'process' ? 'proc' : nd.kind === 'store' ? 'store' : 'ext'}${nd.unknown ? ' unk' : ''}`; + const glyph = glyphOf(nd); + const tw = glyph ? b.w - GLYPH_W : b.w; + const cx = b.x + tw / 2; + const two = b.lines.length > 1; + const sub = nd.subtype ? ` (${nd.subtype})` : ''; + const tip = `${b.ref} ${nd.kind}${sub}: ${nd.label}${nd.group ? ` — boundary ${nd.group}` : ''}${nd.layer ? ` — layer ${nd.layer}` : ''}${nd.note ? ` — ${nd.note}` : ''}`; + const cls = `n ${nd.kind === 'process' ? 'proc' : nd.kind === 'store' ? 'store' : 'ext'}${nd.collapsed ? ' coll' : ''}${nd.unknown ? ' unk' : ''}`; let shape; if (nd.kind === 'process') { - shape = `<rect x="${num(b.x)}" y="${num(b.y)}" width="${b.w}" height="${b.h}" rx="9"/><line x1="${num(b.x)}" y1="${num(b.y + 15)}" x2="${num(b.x + b.w)}" y2="${num(b.y + 15)}"/>` - + `<text class="ref" x="${num(cx)}" y="${num(b.y + 11.5)}" text-anchor="middle">${b.ref}</text><text x="${num(cx)}" y="${num(b.y + 33.5)}" text-anchor="middle">${esc(clip(nd.label))}</text>`; + shape = `<rect x="${num(b.x)}" y="${num(b.y)}" width="${num(b.w)}" height="${num(b.h)}" rx="9"/>${nd.collapsed ? `<rect class="in" x="${num(b.x + 3)}" y="${num(b.y + 3)}" width="${num(b.w - 6)}" height="${num(b.h - 6)}" rx="7"/>` : ''}<line x1="${num(b.x)}" y1="${num(b.y + 15)}" x2="${num(b.x + b.w)}" y2="${num(b.y + 15)}"/>` + + `<text class="ref" x="${num(b.x + b.w / 2)}" y="${num(b.y + 11.5)}" text-anchor="middle">${b.ref}</text>${textLines('', b.x + b.w / 2, b.y + (two ? 27 : 33.5), 'middle', b.lines, 11.5)}`; } else if (nd.kind === 'store') { shape = `<path d="M${num(b.x + b.w)},${num(b.y)} H${num(b.x)} V${num(b.y + b.h)} H${num(b.x + b.w)}"/><line x1="${num(b.x + 30)}" y1="${num(b.y)}" x2="${num(b.x + 30)}" y2="${num(b.y + b.h)}"/>` - + `<text class="ref" x="${num(b.x + 15)}" y="${num(b.y + 26)}" text-anchor="middle">${b.ref}</text><text x="${num(b.x + 30 + (b.w - 30) / 2)}" y="${num(b.y + 26)}" text-anchor="middle">${esc(clip(nd.label))}</text>`; + + `<text class="ref" x="${num(b.x + 15)}" y="${num(b.y + 26)}" text-anchor="middle">${b.ref}</text>${textLines('', b.x + 30 + (tw - 30) / 2, b.y + (two ? 20 : 26), 'middle', b.lines, 12)}` + + `${glyph ? glyphSvg(glyph, b.x + b.w - GLYPH_W, b.y + 15) : ''}`; } else { - shape = `<rect class="sh" x="${num(b.x + 4)}" y="${num(b.y + 4)}" width="${b.w}" height="${b.h}"/><rect x="${num(b.x)}" y="${num(b.y)}" width="${b.w}" height="${b.h}"/>` - + `<text class="ref" x="${num(b.x + 6)}" y="${num(b.y + 12)}">${b.ref}</text><text x="${num(cx)}" y="${num(b.y + 30)}" text-anchor="middle">${esc(clip(nd.label))}</text>`; + shape = `<rect class="sh" x="${num(b.x + 4)}" y="${num(b.y + 4)}" width="${num(b.w)}" height="${num(b.h)}"/><rect x="${num(b.x)}" y="${num(b.y)}" width="${num(b.w)}" height="${num(b.h)}"/>` + + `<text class="ref" x="${num(b.x + 6)}" y="${num(b.y + 12)}">${b.ref}</text>${textLines('', cx, b.y + (two ? 24 : 30), 'middle', b.lines, 12)}` + + `${glyph ? glyphSvg(glyph, b.x + b.w - GLYPH_W, b.y + 4) : ''}`; } - const cap = b.caption ? `<text class="cap" x="${num(cx)}" y="${num(b.y + b.h + 12)}" text-anchor="middle">‹${esc(clip(b.caption, 30))}›</text>` : ''; + const cap = b.caption ? `<text class="cap" x="${num(b.x + b.w / 2)}" y="${num(b.y + b.h + 12)}" text-anchor="middle">‹${esc(clip(b.caption, 30))}›</text>` : ''; return `<g class="${cls}" data-id="${esc(nd.id)}"><title>${esc(tip)}${shape}${cap}`; } -function edgeSvg(e) { +const EXIST_CLASS = { unknown: 'unk', contradicted: 'bad', confirmed: 'ok' }; + +function edgeSvg(e, flows) { const m = markerOf(e); - const cls = ['e', e.cls, e.mode, ...(e.existence === 'unknown' ? ['unk'] : e.existence === 'contradicted' ? ['bad'] : [])].join(' '); + const cls = ['e', e.cls, e.mode, ...(EXIST_CLASS[e.existence] ? [EXIST_CLASS[e.existence]] : []), ...(e.role === 'data' ? [] : [`r-${e.role}`])].join(' '); const end = e.heads === 'none' ? '' : ` marker-end="url(#dfd-a-${m})"`; const start = e.heads === 'both' ? ` marker-start="url(#dfd-a-${m})"` : ''; const word = { fwd: 'flow', bi: 'two-way flow', fb: 'feedback flow', self: 'retry loop', und: 'flow, direction not determined' }[e.cls]; - const label = e.label ? `${esc(e.label)}` : ''; - return `${esc(`${word}: ${e.from} → ${e.to} (${e.mode}, existence ${e.existence})`)}${label}`; + const dirs = [...new Set(e.flows.map((k) => flows[k].direction).filter(Boolean))]; + const facts = [e.mode, `existence ${e.existence}`, ...(e.role === 'data' ? [] : [`role ${e.role}`]), ...(dirs.length ? [`direction ${dirs.join('/')}`] : [])]; + const tip = `${word}: ${e.from} → ${e.to} (${facts.join(', ')})${e.label ? ` — ${e.label}` : ''}`; + const label = e.lines.length ? textLines('el', e.lx, e.ly, e.anchor, e.lines) : ''; + return `${esc(tip)}${label}`; } export function renderSvg(model, lay) { const title = model.title || 'data flow diagram'; - const bands = lay.bands.map((b, i) => `${esc(clip(b.name, 12))}`).join(''); - const hulls = lay.hulls.map((h) => `${esc(clip(h.label, Math.floor((h.x2 - h.x - 12) / 6.1)))}`).join(''); + const TB = lay.orientation === 'TB'; + const bands = lay.bands.map((b, i) => `` + + `${esc(clip(b.name, 12))}`).join(''); + const hulls = lay.hulls.map((h) => `${esc(clip(h.label, Math.floor((h.x2 - h.x - 12) / ECHAR)))}`).join(''); return `` - + `${esc(title)}${lay.nodes.length} elements and ${model.flows.length} flows, drawn as ${lay.edges.length} edges. The tables after the diagram list every one.` + + `${esc(title)}${lay.nodes.length} elements and ${model.flows.length} flows, drawn as ${lay.edges.length} edges, ranks ${TB ? 'top to bottom' : 'left to right'}. The tables after the diagram list every one.` + `${MARKERS}${bands}${hulls}` - + `${lay.edges.map(edgeSvg).join('')}${lay.nodes.map((b, i) => nodeSvg(b, model.nodes[i])).join('')}`; + + `${lay.edges.map((e) => edgeSvg(e, model.flows)).join('')}${lay.nodes.map((b, i) => nodeSvg(b, model.nodes[i])).join('')}`; } // ── page ─────────────────────────────────────────────────────────────────────────────────────── @@ -360,20 +674,27 @@ svg.dfd{display:block} .dfd .n rect,.dfd .n path{fill:var(--panel2);stroke:var(--ink);stroke-width:1.4} .dfd .n line{stroke:var(--ink);stroke-width:1} .dfd .n.ext .sh{fill:var(--line2);stroke:none} +.dfd .n.coll .in{fill:none;stroke-width:1} .dfd .n.unk rect,.dfd .n.unk path{stroke-dasharray:4 3;stroke:var(--mut)} .dfd .n.unk text{font-style:italic;fill:var(--mut)} +.dfd .n path.gl{fill:none;stroke:var(--mut);stroke-width:1.1;stroke-dasharray:none} .dfd .e path{fill:none;stroke:var(--ink);stroke-width:1.5} .dfd .e.async path{stroke-dasharray:7 4} .dfd .e.mixed path{stroke-dasharray:10 3 2 3} .dfd .e.und path{stroke-dasharray:1 5;stroke-linecap:round;stroke-width:2} +.dfd .e.ok path{stroke-width:1.9} +.dfd .e.unk{opacity:.55} .dfd .e.unk path{stroke:var(--mut);stroke-width:1} .dfd .e.und.unk path{stroke-width:1.6} .dfd .e.fb path{stroke:var(--acc2);stroke-width:2.4} .dfd .e.fb.unk path{stroke-width:1.4} +.dfd .e.r-error path{stroke:var(--high);stroke-dasharray:5 2 1 2} +.dfd .e.r-retry path{stroke-dasharray:2 3;stroke-linecap:round} .dfd .e.bad path{stroke:var(--crit)} .dfd .el{fill:var(--mut);font-size:10px;paint-order:stroke;stroke:var(--panel);stroke-width:3px;stroke-linejoin:round} .dfd .e.fb .el{fill:var(--acc2);font-weight:600} -.dfd .mk-ink{fill:var(--ink)}.dfd .mk-mut{fill:var(--mut)}.dfd .mk-fb{fill:var(--acc2)}.dfd .mk-bad{fill:var(--crit)} +.dfd .e.r-error .el{fill:var(--high);font-weight:600} +.dfd .mk-ink{fill:var(--ink)}.dfd .mk-mut{fill:var(--mut)}.dfd .mk-fb{fill:var(--acc2)}.dfd .mk-bad{fill:var(--crit)}.dfd .mk-hi{fill:var(--high)} .dfd .band rect{fill:color-mix(in srgb,var(--acc) 5%,transparent);stroke:var(--line)} .dfd .band.alt rect{fill:color-mix(in srgb,var(--acc) 10%,transparent)} .dfd .band text{fill:var(--mut);font-weight:600;letter-spacing:.08em;text-transform:uppercase} @@ -392,18 +713,34 @@ td.m{font-family:var(--mono);font-size:.75rem;overflow-wrap:anywhere} const legendSvg = (inner) => ``; const legendLine = (cls, marker, heads = 'end') => legendSvg(``); +const legendGlyph = (kind, d) => legendSvg(`${glyphSvg(d, 23, 5)}`); const LEGEND = `
  • ${legendSvg('')} process
    • +
  • ${legendSvg('')} collapsed boundary (one node for a group)
  • ${legendSvg('')} data store (D), broker or queue (Q)
  • ${legendSvg('')} external entity
    • +
  • ${legendGlyph('store', STORE_GLYPH.database)} database
    • +
  • ${legendGlyph('store', STORE_GLYPH.queue)} broker, queue, topic or stream
    • +
  • ${legendGlyph('store', STORE_GLYPH.log)} log
    • +
  • ${legendGlyph('store', STORE_GLYPH.file)} file
    • +
  • ${legendGlyph('ext', EXT_GLYPH.endpoint)} endpoint
    • +
  • ${legendGlyph('ext', EXT_GLYPH.scheduler)} scheduler
    • +
  • ${legendGlyph('ext', EXT_GLYPH.actor)} actor
    • +
  • ${legendGlyph('ext', EXT_GLYPH.system)} system
    • +
  • ${legendGlyph('ext', EXT_GLYPH.env)} environment
    • +
  • ${legendGlyph('ext', EXT_GLYPH.binary)} binary
  • ${legendLine('fwd sync', 'ink')} sync flow
  • ${legendLine('fwd async', 'ink')} async or event flow
  • ${legendLine('fwd mixed', 'ink')} both modes on one pair
  • ${legendLine('bi sync', 'ink', 'both')} bidirectional
    • -
  • ${legendLine('fb sync', 'fb')} feedback (routed over the top)
    • +
  • ${legendLine('fb sync', 'fb')} feedback (routed outside the body)
    • +
  • ${legendLine('fwd sync r-error', 'hi')} error flow (label marked ✕)
    • +
  • ${legendLine('fwd sync r-retry', 'ink')} retry flow (label marked ↻)
  • ${legendSvg('')} retry loop
  • ${legendLine('und sync', 'ink', 'none')} direction not determined
    • -
  • ${legendLine('fwd sync unk', 'mut')} existence unknown
    • +
  • ${legendLine('fwd sync ok', 'ink')} existence confirmed
    • +
  • ${legendLine('fwd sync', 'ink')} declared, no evidence
    • +
  • ${legendLine('fwd sync unk', 'mut')} existence unknown (lighter)
  • ${legendLine('fwd sync bad', 'bad')} contradicted
  • ${legendSvg('')} system boundary
  • ${legendSvg('')} trust boundary
    • @@ -415,22 +752,34 @@ const list = (xs, empty = 'none') => (xs.length ? xs.map((x) => esc(x)).join('; function sourceBanner(model) { const s = model.source; - if (!s) return ''; - const parts = [`Producer: ${esc(s.producer)}.`]; - if (s.note) parts.push(esc(s.note)); - if (s.graph) parts.push(`Flow graph of ${esc(s.graph.nodes)} nodes and ${esc(s.graph.edges)} edges${s.graph.generatedAt ? `, generated ${esc(s.graph.generatedAt)}` : ''}.`); - if (s.omitted) parts.push(`Left out by the adapter: ${Object.entries(s.omitted).map(([k, v]) => `${esc(k)} ${esc(v)}`).join(', ')}.`); - if (s.filter) { - const f = s.filter; - parts.push(`This is a subset. Filter: ${f.prefix === null ? '' : `prefix ${esc(f.prefix)}, `}${f.focus === null ? '' : `focus ${esc(f.focus)}, `}${esc(f.hops)} hop(s) from ${esc(f.seeds)} seed node(s); ${esc(f.nodesOmitted)} nodes and ${esc(f.flowsOmitted)} flows are not shown.`); + const parts = []; + if (!s) parts.push('Authored model. No producer is recorded, so every flow is as declared and none is verified here.'); + else { + parts.push(`Producer: ${esc(s.producer)}.`); + if (s.note) parts.push(esc(s.note)); + if (s.graph) parts.push(`Flow graph of ${esc(s.graph.nodes)} nodes and ${esc(s.graph.edges)} edges${s.graph.generatedAt ? `, generated ${esc(s.graph.generatedAt)}` : ''}.`); + if (s.omitted) parts.push(`Left out by the adapter: ${Object.entries(s.omitted).map(([k, v]) => `${esc(k)} ${esc(v)}`).join(', ')}.`); + if (s.filter) { + const f = s.filter; + const around = f.around ?? f.focus ?? null; + const dir = f.direction && f.direction !== 'both' ? `, following flows ${f.direction === 'in' ? 'into' : 'out of'} the seeds only` : ''; + parts.push(`This is a subset. Filter: ${f.prefix == null ? '' : `prefix ${esc(f.prefix)}, `}${around === null ? '' : `focus ${esc(around)}, `}${esc(f.hops)} hop(s) from ${esc(f.seeds)} seed node(s)${dir}; ${esc(f.nodesOmitted)} nodes and ${esc(f.flowsOmitted)} flows are not shown.`); + } + } + if (model.collapsedGroups) { + parts.push(`Collapsed: ${model.collapsedGroups.map((c) => `${esc(c.label)} (${c.members} elements; ${c.internal} flow(s) inside it not drawn)`).join('; ')}. Flows across a collapsed boundary are merged per other end, mode and role, the count written ×n.`); } return ``; } -/** -> { html, csp }. `csp` is the header form; the page carries the same policy in a meta element. */ -export function renderHtml(model, { fonts = 'none', maxPaths = 200 } = {}) { +/** + * -> { html, csp, stats }. `csp` is the header form; the page carries the same policy in a meta + * element. `collapse` names groups to draw as one node each; `orientation` is LR, TB or auto. + */ +export function renderHtml(input, { fonts = 'none', maxPaths = 200, collapse = [], orientation = 'auto' } = {}) { + const model = collapseGroups(input, collapse); const analysis = analyse(model, { maxPaths }); - const lay = layout(model, analysis); + const lay = layout(model, analysis, { orientation }); const ref = new Map(lay.nodes.map((b) => [b.id, b.ref])); const labels = new Map(model.nodes.map((nd) => [nd.id, nd.label])); const name = (id) => `${ref.get(id)} ${labels.get(id)}`; @@ -440,15 +789,17 @@ export function renderHtml(model, { fonts = 'none', maxPaths = 200 } = {}) { const nodeRows = model.nodes.map((nd, i) => { const a = analysis.nodes[i]; const roles = [a.fanOut > 1 ? `branch ×${a.fanOut}` : '', a.fanIn > 1 ? `merge ×${a.fanIn}` : '', analysis.sources.includes(nd.id) ? 'source' : '', - analysis.sinks.includes(nd.id) ? 'sink' : '', a.cyclic ? 'in a cycle' : '', nd.unknown ? 'unknown' : ''].filter(Boolean); + analysis.sinks.includes(nd.id) ? 'sink' : '', a.cyclic ? 'in a cycle' : '', nd.unknown ? 'unknown' : '', nd.collapsed ? `collapsed group of ${nd.collapsed.members}` : ''].filter(Boolean); return `${lay.nodes[i].ref}${esc(nd.kind)}${nd.subtype ? ` · ${esc(nd.subtype)}` : ''}${esc(nd.label)}${nd.group ? esc(nd.group) : absent('none')}${nd.layer ? esc(nd.layer) : absent('none')}${a.rank}${roles.length ? esc(roles.join(', ')) : ''}${nd.note ? ` ${esc(nd.note)}` : ''}`; }).join('\n'); const flowRows = model.flows.map((f, k) => { const ev = f.evidence; const refs = ev && ev.refs && ev.refs.length ? ` — ${esc(ev.refs[0])}${ev.refs.length > 1 ? ` (+${ev.refs.length - 1})` : ''}` : ''; - const evidence = !ev ? absent('declared, no evidence') - : `${ev.existence === 'unknown' ? absent('existence unknown') : esc(ev.existence)}${ev.witness ? ` · ${esc(ev.witness)}` : ''}${refs}`; - return `${k + 1}${esc(name(f.from))}${isDirected(f) ? '→' : '—'}${esc(name(f.to))}${esc(f.mode)}${classWord[analysis.flowClass[k]]}${f.label ? esc(f.label) : ''}${evidence}`; + const existence = flowExistence(f); + const evidence = existence === 'declared' ? absent('declared, no evidence') + : `${existence === 'unknown' ? absent('existence unknown') : esc(existence)}${ev && ev.witness ? ` · ${esc(ev.witness)}` : ''}${refs}`; + const role = f.role && f.role !== 'data' ? ` · ${esc(f.role)}` : ''; + return `${k + 1}${esc(name(f.from))}${isDirected(f) ? '→' : '—'}${esc(name(f.to))}${esc(f.mode)}${classWord[analysis.flowClass[k]]}${role}${f.label ? esc(f.label) : ''}${evidence}`; }).join('\n'); const p = analysis.paths; @@ -477,7 +828,7 @@ export function renderHtml(model, { fonts = 'none', maxPaths = 200 } = {}) {

    ${esc(title)}

    -

    Data flow diagram, Gane–Sarson notation · dfd@${esc(model.v)} · ${model.nodes.length} elements, ${model.flows.length} flows

    +

    Data flow diagram, Gane–Sarson notation · dfd@${esc(model.v)} · ${model.nodes.length} elements, ${model.flows.length} flows · ranks ${lay.orientation === 'TB' ? 'top to bottom' : 'left to right'}

    ${sourceBanner(model)}
    ${renderSvg(model, lay)}
    ${LEGEND} @@ -495,5 +846,6 @@ ${flowRows}
    `; - return { html, csp: policy(["frame-ancestors 'self'"]) }; + const stats = { orientation: lay.orientation, width: lay.width, height: lay.height, crossings: lay.crossings, collisions: lay.collisions, elements: model.nodes.length, flows: model.flows.length }; + return { html, csp: policy(["frame-ancestors 'self'"]), stats }; } diff --git a/lib/test/dfd-render.test.mjs b/lib/test/dfd-render.test.mjs index 8fdf6fdfc..0ce6be576 100644 --- a/lib/test/dfd-render.test.mjs +++ b/lib/test/dfd-render.test.mjs @@ -8,13 +8,14 @@ import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { analyse } from '../dfd.mjs'; -import { layout, renderSvg, renderHtml, pageCss, clip, roundedPath, SWEEPS } from '../dfd-render.mjs'; +import { layout, renderSvg, renderHtml, pageCss, clip, wrap, roundedPath, collapseGroups, groupSizes, countCrossings, flowExistence, SWEEPS, AUTO_TB_WIDTH } from '../dfd-render.mjs'; import { FOLLOWER_JS } from '../theme-follower.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const fixture = () => JSON.parse(readFileSync(join(HERE, 'fixtures', 'dfd-synthetic.json'), 'utf8')); const sha = (s) => createHash('sha256').update(s, 'utf8').digest('base64'); -const lay = (m) => layout(m, analyse(m)); +// the LR-specific assertions below name their orientation; `auto` picks TB for the wide fixture +const lay = (m, opts = { orientation: 'LR' }) => layout(m, analyse(m), opts); const node = (id, kind = 'process', extra = {}) => ({ id, kind, label: id, ...extra }); const flow = (from, to, extra = {}) => ({ from, to, mode: 'sync', ...extra }); const doc = (nodes, flows, extra = {}) => ({ v: 1, kind: 'dfd', ...extra, nodes, flows }); @@ -108,7 +109,7 @@ test('a feedback edge runs over the top: its track is above every element', () = // act const g = lay(m); const top = Math.min(...g.nodes.map((b) => b.y)); - const tracks = g.edges.filter((e) => e.cls === 'fb').map((e) => e.ly + 4); + const tracks = g.edges.filter((e) => e.cls === 'fb').map((e) => e.track); // assert assert.equal(tracks.length, 2); @@ -324,7 +325,8 @@ test('edge markup carries its flows, its class and the right heads', () => { // assert assert.match(svg, /.*?marker-start="url\(#dfd-a-ink\)" marker-end="url\(#dfd-a-ink\)"/); - assert.match(svg, /feedback flow: validate → transform \(sync, existence declared\)<\/title>/); + assert.match(svg, /<g class="e fb sync" data-flows="10" data-from="validate" data-to="transform"><title>feedback flow: validate → transform \(sync, existence declared\) — validation errors<\/title>/); + assert.match(svg, /<g class="e fwd sync ok" data-flows="2" data-from="api" data-to="orders-db">/); assert.match(svg, /<g class="e fb async unk" data-flows="11"[^>]*>.*?marker-end="url\(#dfd-a-fb\)"/); assert.match(svg, /<g class="e self sync" data-flows="5" data-from="ingest" data-to="ingest"><title>retry loop:/); assert.match(svg, /<g class="e fwd async" data-flows="3" data-from="api" data-to="events">/); @@ -502,3 +504,442 @@ test('the banner says who produced the model, what the adapter left out and what + '<strong>This is a subset.</strong> Filter: prefix <code>flow/</code>, 1 hop(s) from 2 seed node(s); 8 nodes and 4 flows are not shown.'); assert.match(texts[3], /Flow graph of 9 nodes and 4 edges\. <strong>This is a subset\.<\/strong> Filter: focus <code>a<\/code>, 2 hop\(s\)/); }); + +// ── wrapping, orientation, crossings and label placement ───────────────────────────────────────── + +test('wrap keeps a short label, breaks at a space or after a separator, and clips only the last line', () => { + // arrange + const path = 'monitor/extractors/sast-lint/_deno-check-counts.mjs'; + + // act + const short = wrap('order row', 26); + const spaced = wrap('validation errors from the quality gate', 26); + const sliced = wrap(path, 31); + const solid = wrap('x'.repeat(40), 10); + const three = wrap('aa bb cc dd ee ff gg hh', 5, 3); + + // assert + assert.deepEqual(short, ['order row']); + assert.deepEqual(spaced, ['validation errors from the', 'quality gate']); + assert.deepEqual(sliced, ['monitor/extractors/sast-lint/', '_deno-check-counts.mjs']); + assert.deepEqual(solid, ['x'.repeat(10), 'xxxxx…xxxx']); + assert.equal(three.length, 3); + assert.ok(three.every((l) => l.length <= 5)); +}); + +test('a long element label is drawn on two lines and kept whole in the tooltip and the table', () => { + // arrange + const label = 'monitor/extractors/sast-lint/_deno-check-counts-and-more.mjs'; + const m = doc([node('a', 'process', { label }), node('b', 'store', { label: `${label} store` }), node('c', 'external', { label: `${label} ext` })], + [flow('a', 'b'), flow('c', 'a')]); + + // act + const { html } = renderHtml(m); + + // assert + assert.match(html, /<tspan x="[\d.]+">monitor\/extractors\/sast-lint\/<\/tspan><tspan x="[\d.]+" dy="11.5">_deno-check-counts-and-more\.mjs<\/tspan>/); + assert.match(html, new RegExp(`<title>P1 process: ${label.replace(/[.]/g, '\\.')}`)); + assert.match(html, new RegExp(`${label.replace(/[.]/g, '\\.')} ext`)); +}); + +test('auto orientation keeps a narrow model left to right and turns the wide fixture top to bottom', () => { + // arrange + const narrow = doc([node('a'), node('b', 'store')], [flow('a', 'b')]); + const m = fixture(); + + // act + const small = lay(narrow, {}); + const lr = lay(m, { orientation: 'LR' }); + const tb = lay(m, { orientation: 'TB' }); + const auto = lay(m, {}); + + // assert + assert.equal(small.orientation, 'LR'); + assert.ok(lr.width > AUTO_TB_WIDTH); + assert.equal(auto.orientation, 'TB'); + assert.deepEqual(auto, tb); + assert.ok(auto.width < 1600, `the sample is ${auto.width} wide`); +}); + +test('auto keeps LR when TB would be no narrower', () => { + // arrange: one rank of many wide elements is narrow LR but wide TB; a chain makes LR exceed the threshold + const nodes = []; + const flows = []; + for (let i = 0; i < 16; i++) nodes.push(node(`n${i}`, 'process', { label: `${'w'.repeat(30)}${i}` })); + for (let i = 1; i < 16; i++) flows.push(flow('n0', `n${i}`)); + for (let i = 0; i < 12; i++) { nodes.push(node(`c${i}`)); flows.push(flow(i ? `c${i - 1}` : 'n1', `c${i}`)); } + const m = doc(nodes, flows); + + // act + const lr = lay(m); + const tb = lay(m, { orientation: 'TB' }); + const auto = lay(m, {}); + + // assert + assert.ok(lr.width > AUTO_TB_WIDTH && tb.width >= lr.width, `${lr.width} / ${tb.width}`); + assert.equal(auto.orientation, 'LR'); +}); + +test('an orientation outside LR, TB and auto is refused', () => { + // arrange + const m = doc([node('a')], []); + + // act + const run = () => lay(m, { orientation: 'RL' }); + + // assert + assert.throws(run, /orientation is one of LR, TB, auto, got RL/); +}); + +test('top to bottom: ranks run downward, layers are columns, feedback runs down the left', () => { + // arrange + const m = fixture(); + + // act + const g = lay(m, { orientation: 'TB' }); + const box = Object.fromEntries(g.nodes.map((b) => [b.id, b])); + const left = Math.min(...g.nodes.map((b) => b.x)); + + // assert + assert.deepEqual(overlaps(g.nodes), []); + assert.ok(g.nodes.every((b) => b.x >= 0 && b.y >= 0 && b.x + b.w <= g.width && b.y + b.h <= g.height)); + assert.ok(box.ingest.y < box.bronze.y && box.bronze.y < box.transform.y && box.transform.y < box.validate.y); + assert.ok(g.bands[0].x < g.bands[1].x && g.bands[1].x < g.bands[2].x); + for (const b of g.nodes.filter((nd) => nd.lane)) { + const band = g.bands.find((x) => x.name === b.lane); + assert.ok(b.x >= band.x && b.x + b.w <= band.x + band.w, `${b.id} sits inside ${b.lane}`); + } + assert.ok(g.edges.filter((e) => e.cls === 'fb').every((e) => e.track < left)); + assert.match(renderSvg(m, g), /bronze<\/text>/); +}); + +test('edge labels on the fixture sit clear of every element and of each other, in both orientations', () => { + // arrange + const m = fixture(); + + // act + const gs = [lay(m, { orientation: 'LR' }), lay(m, { orientation: 'TB' })]; + + // assert + for (const g of gs) { + assert.equal(g.collisions, 0, g.orientation); + const labels = g.edges.filter((e) => e.lbox).map((e) => ({ id: e.label, ...e.lbox })); + assert.equal(labels.length, g.edges.filter((e) => e.label).length); + assert.deepEqual(overlaps(labels), [], g.orientation); + assert.deepEqual(labels.flatMap((l) => g.nodes.filter((b) => overlaps([l, b]).length).map((b) => `${l.id} on ${b.id}`)), [], g.orientation); + assert.ok(labels.every((l) => l.x >= 0 && l.y >= 0 && l.x + l.w <= g.width && l.y + l.h <= g.height)); + } +}); + +test('a label with no clear spot is counted as a collision, and one blocked at first is nudged clear', () => { + // arrange: twelve labelled flows squeezed between two one-element ranks + const many = doc([node('a'), node('b', 'store')], Array.from({ length: 12 }, (_, i) => flow(i % 2 ? 'a' : 'b', i % 2 ? 'b' : 'a', { label: `flow number ${i} with a long label` }))); + const pair = doc([node('a'), node('b'), node('c'), node('d')], [flow('a', 'c', { label: 'first label here' }), flow('b', 'd', { label: 'second label here' }), flow('a', 'd', { label: 'third label here' })]); + + // act + const crowded = lay(many); + const g = lay(pair); + + // assert + assert.equal(crowded.edges.length, 1); + assert.equal(crowded.collisions, 0, 'one bundle, one label'); + assert.equal(g.collisions, 0); + assert.deepEqual(overlaps(g.edges.map((e) => ({ id: e.label, ...e.lbox }))), []); +}); + +test('a retry label that the boundary name blocks is counted as a collision, not moved into a gutter', () => { + // arrange: the hull's name sits exactly where the label above the box would stand + const m = doc([node('a', 'process', { group: 'g' })], [flow('a', 'a', { label: 'retry the upstream call on timeout' })], + { groups: [{ id: 'g', label: 'a long boundary name', boundary: 'system' }] }); + + // act + const g = lay(m); + const box = g.nodes[0]; + + // assert + assert.equal(g.collisions, 1); + assert.ok(g.edges[0].lbox.x >= box.x && g.edges[0].lbox.x + g.edges[0].lbox.w <= box.x + box.w); +}); + +test('crossings: the fixture keeps no more than the sweeps alone leave, and transposition clears a dense two-rank case', () => { + // arrange + const m = fixture(); + const spec = 'a1>b4 a2>b1 a3>b3 a1>b3 a4>b0 a4>b4 a0>b2 a3>b1 a4>b2'; + const dense = doc([...[0, 1, 2, 3, 4].map((i) => node(`a${i}`)), ...[0, 1, 2, 3, 4].map((i) => node(`b${i}`, 'store'))], + spec.split(' ').map((p) => flow(...p.split('>')))); + + // act + const before = lay(m, { orientation: 'LR', transpose: false }).crossings; + const after = lay(m, { orientation: 'LR' }).crossings; + const d0 = lay(dense, { orientation: 'LR', transpose: false }).crossings; + const d1 = lay(dense, { orientation: 'LR' }).crossings; + + // assert + assert.ok(after <= before, `${after} > ${before}`); + assert.equal(after, 0); + assert.equal(d0, 4); + assert.equal(d1, 1); +}); + +test('countCrossings counts each crossing pair of segments between consecutive ranks once', () => { + // arrange: a->y and b->x cross; a->x shares an end with both + const a = { pos: 0, down: [] }; const b = { pos: 1, down: [] }; const x = { pos: 0, down: [] }; const y = { pos: 1, down: [] }; + a.down.push(y, x); b.down.push(x); + + // act + const c = countCrossings([[a, b], [x, y]]); + + // assert + assert.equal(c, 1); +}); + +test('group members stay contiguous within every rank after transposition', () => { + // arrange: two groups interleaved by declaration, fed from crossing sources + const m = doc( + [node('s1'), node('s2'), node('s3'), node('g1', 'store', { group: 'g' }), node('h1', 'store', { group: 'h' }), node('g2', 'store', { group: 'g' }), node('h2', 'store', { group: 'h' })], + [flow('s1', 'h1'), flow('s1', 'g2'), flow('s2', 'g1'), flow('s3', 'h2'), flow('s2', 'h2'), flow('s3', 'g1')], + ); + + // act + const g = lay(m); + const groupOf = new Map(m.nodes.map((nd, i) => [i, nd.group])); + + // assert + for (const rank of g.ranks) { + const seq = rank.filter((i) => i >= 0).map((i) => groupOf.get(i)).filter(Boolean); + const runs = seq.filter((x, i) => i === 0 || seq[i - 1] !== x); + assert.equal(runs.length, new Set(runs).size, `a group is split in ${seq.join(',')}`); + } +}); + +test('feedback risers never pass through a retry loop label, in either orientation', () => { + // arrange: the feedback source and target both carry a self-loop with a label + const m = doc([node('a'), node('b'), node('c')], [ + flow('a', 'b'), flow('b', 'c'), flow('c', 'a', { label: 'send back for repair' }), + flow('a', 'a', { label: 'retry the upstream call' }), flow('c', 'c', { label: 'retry the downstream write' }), + ]); + + // act + const gs = [lay(m, { orientation: 'LR' }), lay(m, { orientation: 'TB' })]; + + // assert + for (const g of gs) { + const selfLabels = g.edges.filter((e) => e.cls === 'self').map((e) => e.lbox); + const risers = g.edges.filter((e) => e.cls === 'fb').flatMap((e) => e.risers); + assert.equal(selfLabels.length, 2); + assert.equal(risers.length, 2); + for (const [[x1, y1], [x2, y2]] of risers) { + const seg = { id: 'riser', x: Math.min(x1, x2), y: Math.min(y1, y2), w: Math.abs(x2 - x1) || 0.1, h: Math.abs(y2 - y1) || 0.1 }; + assert.deepEqual(selfLabels.filter((l) => overlaps([seg, { id: 'l', ...l }]).length), [], `${g.orientation}: a riser crosses a self-loop label`); + } + for (const e of g.edges.filter((x) => x.cls === 'self')) { + const own = g.nodes.find((b) => b.id === e.from); + assert.ok(e.lbox.x >= own.x && e.lbox.x + e.lbox.w <= own.x + own.w && e.lbox.y + e.lbox.h <= own.y, `${g.orientation}: the label stands above its own box`); + } + } +}); + +// ── subtypes, roles, existence ─────────────────────────────────────────────────────────────────── + +test('every recognised subtype carries its mark, and an unrecognised one carries none', () => { + // arrange + const stores = ['database', 'broker', 'queue', 'topic', 'stream', 'log', 'file', 'lake']; + const exts = ['endpoint', 'scheduler', 'actor', 'system', 'env', 'binary', 'user']; + const m = doc([node('p'), ...stores.map((s) => node(`s-${s}`, 'store', { subtype: s })), ...exts.map((s) => node(`e-${s}`, 'external', { subtype: s }))], + [...stores.map((s) => flow('p', `s-${s}`)), ...exts.map((s) => flow(`e-${s}`, 'p'))]); + + // act + const svg = renderSvg(m, lay(m)); + const marked = (id) => new RegExp(`data-id="${id}">[^<]*(?:(?!).)*([\s\S]*?)<\/ul>/.exec(html)[1]; + + // assert + for (const w of ['database', 'broker, queue, topic or stream', 'log', 'file', 'endpoint', 'scheduler', 'actor', 'system', 'environment', 'binary', + 'error flow', 'retry flow', 'existence confirmed', 'declared, no evidence', 'existence unknown', 'contradicted', 'collapsed boundary']) { + assert.ok(legend.includes(` ${w}`), w); + } +}); + +test('an error flow and a retry flow are drawn with their own line, head and label mark', () => { + // arrange + const m = doc([node('a'), node('b'), node('c'), node('d')], [ + flow('a', 'b', { role: 'error', label: 'rejected' }), flow('b', 'c', { role: 'retry', label: 'again' }), + flow('c', 'd', { role: 'feedback' }), flow('a', 'c', { role: 'data', label: 'plain' }), + ]); + + // act + const svg = renderSvg(m, lay(m)); + const { html } = renderHtml(m); + + // assert + assert.match(svg, /]*>flow: a → b \(sync, existence declared, role error\) — rejected<\/title><path [^>]*marker-end="url\(#dfd-a-hi\)"\/><text class="el"[^>]*>✕ rejected<\/text>/); + assert.match(svg, /<g class="e fwd sync r-retry"[^>]*>.*?↻ again<\/text>/); + assert.match(svg, /<g class="e fwd sync r-feedback"/); + assert.match(svg, /<g class="e fwd sync" data-flows="3"/); + assert.match(html, /<td>forward · error<\/td><td>rejected<\/td>/); +}); + +test('flow existence comes from the flow, then its evidence; confirmed and unknown are drawn apart', () => { + // arrange + const m = doc([node('a'), node('b'), node('c'), node('d')], [ + flow('a', 'b', { existence: 'confirmed' }), flow('b', 'c', { existence: 'unknown', evidence: { existence: 'confirmed' } }), flow('c', 'd', { direction: 'outbound' }), + ]); + + // act + const svg = renderSvg(m, lay(m)); + const { html } = renderHtml(m); + + // assert + assert.equal(flowExistence(m.flows[1]), 'unknown'); + assert.equal(flowExistence(m.flows[2]), 'declared'); + assert.match(svg, /class="e fwd sync ok" data-flows="0"/); + assert.match(svg, /class="e fwd sync unk" data-flows="1"[^>]*>.*?marker-end="url\(#dfd-a-mut\)"/); + assert.match(svg, /<title>flow: c → d \(sync, existence declared, direction outbound\)<\/title>/); + assert.match(html, /<td>confirmed<\/td><\/tr>/); + assert.match(pageCss(), /\.dfd \.e\.unk\{opacity:\.55\}/); +}); + +// ── collapse ───────────────────────────────────────────────────────────────────────────────────── + +const collapsible = () => doc( + [node('in'), node('a', 'process', { group: 'g', layer: 'raw' }), node('b', 'store', { group: 'g', layer: 'raw' }), node('c', 'process', { group: 'g', layer: 'raw' }), + node('out', 'external'), node('h1', 'store', { group: 'h' }), node('h2', 'process', { group: 'h', layer: 'raw' })], + [ + flow('in', 'a', { label: 'req', evidence: { existence: 'confirmed', refs: ['x.mjs:1'] } }), + flow('in', 'c', { label: 'req2', evidence: { existence: 'unknown', refs: ['y.mjs:2'] } }), + flow('a', 'b'), flow('b', 'c'), flow('c', 'c', { label: 'retry' }), + flow('c', 'out', { label: 'result' }), flow('a', 'out', { mode: 'async', label: 'event' }), + flow('c', 'out', { role: 'error', label: 'fault' }), flow('a', 'h1', { directed: false }), flow('h2', 'b'), + flow('in', 'out'), + ], + { layers: ['raw'], groups: [{ id: 'g', label: 'Core', boundary: 'system' }] }, +); + +test('collapsing a group merges its crossing flows per other end, direction, mode and role, and drops its inside', () => { + // arrange + const m = collapsible(); + + // act + const c = collapseGroups(m, ['g']); + const core = c.nodes.find((nd) => nd.id === 'group:g'); + + // assert + assert.deepEqual(c.nodes.map((nd) => nd.id), ['in', 'group:g', 'out', 'h1', 'h2']); + assert.deepEqual(core, { id: 'group:g', kind: 'process', label: 'Core (3 elements)', layer: 'raw', collapsed: { group: 'g', members: 3 } }); + assert.deepEqual(c.flows.map((f) => [f.from, f.to, f.mode, f.role || 'data', f.label, f.merged]), [ + ['in', 'group:g', 'sync', 'data', 'req / req2 ×2', 2], + ['group:g', 'out', 'sync', 'data', 'result', 1], + ['group:g', 'out', 'async', 'data', 'event', 1], + ['group:g', 'out', 'sync', 'error', 'fault', 1], + ['group:g', 'h1', 'sync', 'data', undefined, 1], + ['h2', 'group:g', 'sync', 'data', undefined, 1], + ['in', 'out', 'sync', 'data', undefined, undefined], + ]); + assert.deepEqual(c.flows[0].evidence, { existence: 'unknown', refs: ['x.mjs:1', 'y.mjs:2'] }); + assert.equal(c.flows[1].evidence, undefined); + assert.equal(c.flows[4].directed, false); + assert.deepEqual(c.collapsedGroups, [{ group: 'g', id: 'group:g', label: 'Core', members: 3, internal: 3, merged: 7, layer: 'raw' }]); +}); + +test('two collapsed groups meet as two nodes, an id already taken is not reused, and no names is the model itself', () => { + // arrange + const m = collapsible(); + m.nodes.push(node('group:h')); + m.flows.push(flow('h1', 'group:h')); + + // act + const c = collapseGroups(m, ['g', 'h', 'g']); + const same = collapseGroups(m, []); + + // assert + assert.equal(same, m); + const hid = c.nodes.find((nd) => nd.collapsed && nd.collapsed.group === 'h'); + assert.equal(hid.id, 'group:h~'); + assert.equal(hid.layer, undefined, 'members on mixed layers leave the node unlayered'); + assert.equal(hid.label, 'h (2 elements)'); + assert.ok(c.flows.some((f) => f.from === 'group:g' && f.to === 'group:h~' && f.directed === false)); + assert.ok(c.flows.some((f) => f.from === 'group:h~' && f.to === 'group:g')); +}); + +test('collapsing a group with no member is refused by name', () => { + // arrange + const m = collapsible(); + + // act + const run = () => collapseGroups(m, ['g', 'nope']); + + // assert + assert.throws(run, /no element is in group "nope"/); +}); + +test('groupSizes counts members per group in first-member order', () => { + // arrange + const m = collapsible(); + + // act + const sizes = groupSizes(m); + + // assert + assert.deepEqual([...sizes], [['g', 3], ['h', 2]]); +}); + +test('renderHtml collapses the named groups, draws the node as a double box and says so in the banner', () => { + // arrange + const m = collapsible(); + + // act + const { html, stats } = renderHtml(m, { collapse: ['g'] }); + + // assert + assert.match(html, /<g class="n proc coll" data-id="group:g"><title>P\d process: Core \(3 elements\)[^<]*<\/title><rect [^>]*\/><rect class="in" /); + assert.match(html, /<strong>Collapsed:<\/strong> Core \(3 elements; 3 flow\(s\) inside it not drawn\)/); + assert.match(html, /collapsed group of 3/); + assert.equal(stats.elements, 5); + assert.equal(stats.flows, 7); +}); + +test('renderHtml reports the drawing it made and stays byte-identical in every orientation and with a collapse', () => { + // arrange + const opts = [{ orientation: 'LR' }, { orientation: 'TB' }, { collapse: ['platform'] }]; + + // act + const runs = opts.map((o) => [renderHtml(fixture(), o), renderHtml(fixture(), o)]); + + // assert + for (const [a, b] of runs) assert.equal(sha(a.html), sha(b.html)); + assert.deepEqual(runs[1][0].stats, { orientation: 'TB', width: 1460, height: 1776, crossings: 0, collisions: 0, elements: 16, flows: 20 }); + assert.equal(runs[0][0].stats.orientation, 'LR'); + assert.match(runs[1][0].html, /ranks top to bottom/); +}); + +test('the banner names the focus under either field name and a one-way filter direction', () => { + // arrange + const at = (filter) => doc([node('a')], [], { source: { producer: 'flow-graph', filter: { prefix: null, hops: 1, seeds: 1, nodesKept: 1, nodesOmitted: 0, flowsOmitted: 0, ...filter } } }); + const banner = (m) => /<p class="banner">(.*?)<\/p>/.exec(renderHtml(m).html)[1]; + + // act + const around = banner(at({ around: 'a', direction: 'in' })); + const out = banner(at({ focus: 'a', direction: 'out' })); + const both = banner(at({ around: 'a', direction: 'both' })); + const neither = banner(at({})); + + // assert + assert.match(around, /focus <code>a<\/code>, 1 hop\(s\) from 1 seed node\(s\), following flows into the seeds only;/); + assert.match(out, /following flows out of the seeds only/); + assert.doesNotMatch(both, /following/); + assert.doesNotMatch(neither, /focus/); +}); From afe7efa9ad3e5cdc44e3e77338cb207b96b01e6d Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:41:52 +1030 Subject: [PATCH 10/36] feat(dfd): map endpoints, databases, schedulers, scripts, logs, the reconcile and layer maps; add fromCodeFlows --- flow/dfd-adapter.mjs | 6 +- flow/test/dfd-adapter.test.mjs | 554 +++++++++++++++++++++++++++++++++ 2 files changed, 557 insertions(+), 3 deletions(-) create mode 100644 flow/test/dfd-adapter.test.mjs diff --git a/flow/dfd-adapter.mjs b/flow/dfd-adapter.mjs index b1b4d5e74..d08e76cad 100644 --- a/flow/dfd-adapter.mjs +++ b/flow/dfd-adapter.mjs @@ -329,14 +329,14 @@ export function fromCodeFlows(records, { title } = {}) { const input = node({ id: `input:${r.repo}:${at(p.source)}`, kind: 'external', subtype: 'actor', label: p.source.expr ? `${p.source.expr} (${at(p.source)})` : at(p.source), group: 'untrusted', note: `a source CodeQL treats as untrusted in ${r.repo}` }); const code = (file) => node({ id: `code:${r.repo}:${file}`, kind: 'process', label: file, group }); - const s = sinkKindOf(r.rule); - const sink = node({ id: `sink:${r.repo}:${s.category}`, kind: s.kind, subtype: s.subtype, label: s.label, - ...(s.unknown ? { unknown: true, note: `the rule id ${r.rule} does not say what its sink is` } : { note: `sink of ${r.repo}'s ${s.category} rules` }) }); const ref = `${r.rule} ${at(p.source)} -> ${at(p.sink)}`; const first = code(p.source.file); flow(input, first, p.source.expr || 'untrusted input', ref); let last = first; if (p.sink.file !== p.source.file) { last = code(p.sink.file); flow(first, last, `${p.steps} steps (${r.rule})`, ref); } + const s = sinkKindOf(r.rule); + const sink = node({ id: `sink:${r.repo}:${s.category}`, kind: s.kind, subtype: s.subtype, label: s.label, + ...(s.unknown ? { unknown: true, note: `the rule id ${r.rule} does not say what its sink is` } : { note: `sink of ${r.repo}'s ${s.category} rules` }) }); flow(last, sink, p.sink.expr ? `${p.sink.expr} (${r.rule})` : r.rule, ref); } return { diff --git a/flow/test/dfd-adapter.test.mjs b/flow/test/dfd-adapter.test.mjs new file mode 100644 index 000000000..d96dfc533 --- /dev/null +++ b/flow/test/dfd-adapter.test.mjs @@ -0,0 +1,554 @@ +// flow/dfd-adapter.mjs: the flow-graph adapter (fromFlowGraph, with the reconcile and a layer map) +// and the code-flow adapter (fromCodeFlows). Each mapping rule is asserted on a graph small enough to +// check by hand; every model the adapters emit is also run through validateDfd. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { validateDfd, analyse } from '../../lib/dfd.mjs'; +import { makeNode, makeEdge, nodeId } from '../graph.mjs'; +import { fromFlowGraph, fromCodeFlows, validateLayerMap, layerOf, globToRegExp, sinkKindOf } from '../dfd-adapter.mjs'; +import { dataflowFromResult } from '../../cra/dataflow-evidence.mjs'; + +const REPO = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const readJson = (rel) => JSON.parse(readFileSync(join(REPO, rel), 'utf8')); + +// ── the flow-graph adapter: the original mapping ───────────────────────────────────────────────── + +function flowGraph() { + const mod = (p, extra) => makeNode('module', p, extra); + const nodes = [ + mod('bin/tool.mjs', { analysis: 'analysed' }), + mod('bin/helper.mjs', { analysis: 'analysed' }), + mod('bin/broken.mjs', { analysis: 'unparseable' }), + mod('bin/test/tool.test.mjs', { analysis: 'analysed' }), + mod('alone.mjs', { analysis: 'analysed' }), + makeNode('store', 'reports/out.json'), + makeNode('artifact', 'rollup.json', { composed: true }), + makeNode('env', 'CW_HOME'), + makeNode('process', 'git'), + makeNode('process', 'CREATE TABLE t (a TEXT)'), + makeNode('process', 'COMMIT'), + makeNode('process', 'bin/helper.mjs'), + ]; + const m = (p) => nodeId('module', p); + const edges = [ + makeEdge(m('bin/tool.mjs'), nodeId('store', 'reports/out.json'), 'writes', { evidence: ['bin/tool.mjs:9 writeFileSync'] }), + makeEdge(m('bin/helper.mjs'), nodeId('store', 'reports/out.json'), 'reads', { witness: 'both', existence: 'confirmed', evidence: ['bin/helper.mjs:4 readFileSync'] }), + makeEdge(m('bin/tool.mjs'), nodeId('artifact', 'rollup.json'), 'touches', { evidence: ['bin/tool.mjs:12 literal'] }), + makeEdge(m('bin/tool.mjs'), nodeId('env', 'CW_HOME'), 'reads', { evidence: ['bin/tool.mjs:3 env'] }), + makeEdge(m('bin/tool.mjs'), m('bin/helper.mjs'), 'reads', { evidence: ['bin/tool.mjs static-import'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'git'), 'spawns', { evidence: ['bin/tool.mjs:20 execFileSync', 'bin/tool.mjs:31 spawnSync'] }), + makeEdge(m('bin/helper.mjs'), nodeId('process', 'git'), 'spawns', { evidence: ['bin/helper.mjs:7 spawn'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'CREATE TABLE t (a TEXT)'), 'spawns', { evidence: ['bin/tool.mjs:40 exec'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'COMMIT'), 'spawns', { evidence: ['bin/tool.mjs:41 exec'] }), + makeEdge(m('bin/tool.mjs'), nodeId('process', 'bin/helper.mjs'), 'spawns', { evidence: null }), + makeEdge(m('bin/test/tool.test.mjs'), nodeId('store', 'reports/out.json'), 'reads', { evidence: ['bin/test/tool.test.mjs:5 readFileSync'] }), + makeEdge(m('bin/broken.mjs'), nodeId('env', 'CW_HOME'), 'reads', { evidence: ['bin/broken.mjs:1 env'] }), + ]; + return { v: 1, generatedAt: '2026-11-11T00:00:00.000Z', nodes, edges }; +} + +test('fromFlowGraph maps modules, files, env keys and spawned binaries by the stated rules', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + const byLabel = Object.fromEntries(d.nodes.map((n) => [n.label, n])); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(Object.keys(d).slice(0, 2), ['v', 'kind']); + assert.deepEqual(byLabel['bin/tool.mjs'], { id: 'node:module:bin/tool.mjs', kind: 'process', label: 'bin/tool.mjs', group: 'bin' }); + assert.equal(byLabel['bin/broken.mjs'].unknown, true); + assert.match(byLabel['bin/broken.mjs'].note, /not analysed \(unparseable\)/); + assert.deepEqual(byLabel['reports/out.json'], { id: 'node:store:reports/out.json', kind: 'store', label: 'reports/out.json', subtype: 'file' }); + assert.equal(byLabel['rollup.json'].unknown, true); + assert.deepEqual(byLabel.CW_HOME, { id: 'node:env:CW_HOME', kind: 'external', label: 'CW_HOME', subtype: 'env' }); + assert.deepEqual(byLabel.git, { id: 'node:process:git', kind: 'external', label: 'git', subtype: 'binary' }); + assert.equal(d.nodes.some((n) => /CREATE TABLE|COMMIT|test/.test(n.label)), false); +}); + +test('fromFlowGraph turns a read around, keeps a write, and leaves a touch undirected', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + const flow = (label, from) => d.flows.find((f) => f.label === label && f.from === from); + + // assert + assert.deepEqual(flow('writes', 'node:module:bin/tool.mjs'), { + from: 'node:module:bin/tool.mjs', to: 'node:store:reports/out.json', label: 'writes', mode: 'sync', + evidence: { witness: 'static', existence: 'unknown', refs: ['bin/tool.mjs:9 writeFileSync'] }, existence: 'unknown', + }); + assert.deepEqual(flow('reads', 'node:store:reports/out.json'), { + from: 'node:store:reports/out.json', to: 'node:module:bin/helper.mjs', label: 'reads', mode: 'sync', + evidence: { witness: 'both', existence: 'confirmed', refs: ['bin/helper.mjs:4 readFileSync'] }, existence: 'confirmed', + }); + assert.equal(flow('reads', 'node:env:CW_HOME').to, 'node:module:bin/tool.mjs'); + assert.equal(flow('touches', 'node:module:bin/tool.mjs').directed, false); +}); + +test('fromFlowGraph reads a spawn as sync only when every witness is a Sync call', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + const spawns = d.flows.filter((f) => f.label === 'spawns').map((f) => [f.from, f.to, f.mode]); + + // assert + assert.deepEqual(spawns, [ + ['node:module:bin/tool.mjs', 'node:process:git', 'sync'], + ['node:module:bin/helper.mjs', 'node:process:git', 'async'], + ['node:module:bin/tool.mjs', 'node:module:bin/helper.mjs', 'async'], + ]); +}); + +test('fromFlowGraph counts everything it leaves out', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual(d.source, { + producer: 'flow-graph', + graph: { v: 1, generatedAt: '2026-11-11T00:00:00.000Z', nodes: 12, edges: 12 }, + options: { tests: false, imports: false, reconcile: null, layers: null, + merged: { pathsThatAreModules: 0, storesRenamed: 0, storesSplit: 0, directionsResolved: 0, reconcileMatched: 0 } }, + omitted: { testModules: 1, importEdges: 1, spawnTargetsNotCommands: 1, ambiguousExecEdges: 1, edgesOfOmittedNodes: 2, isolatedNodes: 2, + selfReferences: 0, unknownNodeKinds: 0, unknownEdgeKinds: 0 }, + }); + assert.equal(d.title, 'flow graph'); +}); + +test('fromFlowGraph keeps tests and imports when asked, and takes a title', () => { + // arrange + const graph = flowGraph(); + delete graph.v; + delete graph.generatedAt; + + // act + const d = fromFlowGraph(graph, { tests: true, imports: true, title: 'everything' }); + + // assert + assert.equal(d.title, 'everything'); + assert.equal(d.source.omitted.testModules, 0); + assert.equal(d.source.omitted.importEdges, 0); + assert.deepEqual(d.source.graph, { v: null, generatedAt: null, nodes: 12, edges: 12 }); + assert.ok(d.nodes.some((n) => n.label === 'bin/test/tool.test.mjs')); + assert.deepEqual(d.flows.find((f) => f.label === 'imports'), { + from: 'node:module:bin/helper.mjs', to: 'node:module:bin/tool.mjs', label: 'imports', mode: 'sync', + evidence: { witness: 'static', existence: 'unknown', refs: ['bin/tool.mjs static-import'] }, existence: 'unknown', + }); +}); + +test('include.tests and include.imports are the same switches as the older top-level ones', () => { + // arrange + const graph = flowGraph(); + + // act + const d = fromFlowGraph(graph, { include: { tests: true, imports: true } }); + + // assert + assert.deepEqual([d.source.options.tests, d.source.options.imports], [true, true]); + assert.equal(d.source.omitted.importEdges, 0); +}); + +// ── the flow-graph adapter: endpoints, databases, schedulers, scripts, logs ────────────────────── + +// The new node and edge kinds are written raw: flow/graph.mjs's constructors gain them in the static +// pass's own change, and this adapter is coded against the contract, not against that change. +const raw = (kind, key, extra = {}) => ({ v: 1, id: `node:${kind}:${key}`, kind, path: key, lastWriteAt: null, ...extra }); +const edge = (from, to, kind, extra = {}) => ({ v: 1, from, to, kind, witness: 'static', existence: 'unknown', evidence: [`${kind} witness`], ...extra }); +const M = (p) => nodeId('module', p); +const graphOf = (nodes, edges) => ({ v: 1, generatedAt: null, nodes, edges }); +const mods = (...paths) => paths.map((p) => makeNode('module', p, { analysis: 'analysed' })); +const flowsOf = (d) => d.flows.map((f) => `${f.from} > ${f.to} ${f.label} ${f.mode}`); + +test('an endpoint is an external; a connection flows out to it and a listener is fed from it', () => { + // arrange + const graph = graphOf([...mods('srv/api.mjs'), raw('endpoint', 'https://api.example:443'), raw('endpoint', 'listen:8080')], [ + edge(M('srv/api.mjs'), 'node:endpoint:https://api.example:443', 'connects'), + edge(M('srv/api.mjs'), 'node:endpoint:listen:8080', 'listens'), + edge('node:endpoint:listen:8080', M('srv/api.mjs'), 'listens'), + ]); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(d.nodes.filter((n) => n.kind === 'external'), [ + { id: 'node:endpoint:https://api.example:443', kind: 'external', label: 'https://api.example:443', subtype: 'endpoint' }, + { id: 'node:endpoint:listen:8080', kind: 'external', label: 'listen:8080', subtype: 'endpoint', note: 'a port this tree listens on (8080)' }, + ]); + assert.deepEqual(flowsOf(d), [ + 'node:module:srv/api.mjs > node:endpoint:https://api.example:443 connects sync', + 'node:endpoint:listen:8080 > node:module:srv/api.mjs listens sync', + 'node:endpoint:listen:8080 > node:module:srv/api.mjs listens sync', + ]); +}); + +test('a database is a store of subtype database, and the tables a flow names go into its label', () => { + // arrange + const db = raw('database', 'state/app.db'); + const graph = graphOf([...mods('a.mjs', 'b.mjs'), db], [ + edge(M('a.mjs'), db.id, 'writes', { tables: ['users', 'orders'] }), + edge(M('b.mjs'), db.id, 'reads', { tables: ['users'] }), + edge(M('b.mjs'), db.id, 'writes', { tables: [] }), + ]); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual(d.nodes.find((n) => n.id === db.id), { id: db.id, kind: 'store', label: 'state/app.db', subtype: 'database' }); + assert.deepEqual(d.flows.map((f) => f.label), ['writes users, orders', 'reads users', 'writes']); +}); + +test('a scheduler is an external that triggers its module by event, whichever end the edge names first', () => { + // arrange + const nodes = [...mods('monitor/sweep.mjs', 'lib/poll.mjs'), raw('scheduler', 'launchd:com.example.sweep'), raw('scheduler', 'timer:lib/poll.mjs'), raw('scheduler', 'at:noon')]; + const graph = graphOf(nodes, [ + edge('node:scheduler:launchd:com.example.sweep', M('monitor/sweep.mjs'), 'triggers'), + edge(M('lib/poll.mjs'), 'node:scheduler:timer:lib/poll.mjs', 'triggers'), + edge('node:scheduler:at:noon', M('lib/poll.mjs'), 'triggers'), + ]); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(flowsOf(d), [ + 'node:scheduler:launchd:com.example.sweep > node:module:monitor/sweep.mjs triggers event', + 'node:scheduler:timer:lib/poll.mjs > node:module:lib/poll.mjs triggers event', + 'node:scheduler:at:noon > node:module:lib/poll.mjs triggers event', + ]); + assert.deepEqual(d.nodes.filter((n) => n.subtype === 'scheduler').map((n) => n.note), ['launchd agent', 'in-process timer', undefined]); +}); + +test('a spawn of node with a script flows to the script\'s process, not to the node binary', () => { + // arrange + const graph = graphOf([...mods('bin/run.mjs', 'bin/child.mjs'), makeNode('process', 'node')], [ + edge(M('bin/run.mjs'), nodeId('process', 'node'), 'spawns', { script: 'bin/child.mjs', evidence: ['bin/run.mjs:3 spawn'] }), + edge(M('bin/child.mjs'), nodeId('process', 'node'), 'spawns', { script: 'vendor/tool.js', evidence: ['bin/child.mjs:9 spawnSync'] }), + ]); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual(flowsOf(d), [ + 'node:module:bin/run.mjs > node:module:bin/child.mjs spawns async', + 'node:module:bin/child.mjs > node:process:node spawns vendor/tool.js sync', + ]); +}); + +test('an appended file another module tails is a log: a broker, with async flows either side', () => { + // arrange + const log = makeNode('store', 'state/events.jsonl'); + const plain = makeNode('store', 'state/notes.txt'); + const graph = graphOf([...mods('w.mjs', 'r.mjs'), log, plain], [ + edge(M('w.mjs'), log.id, 'writes', { append: true }), + edge(M('r.mjs'), log.id, 'reads', { tail: true }), + edge(M('w.mjs'), plain.id, 'writes'), // written, not appended: a tail alone does not make a log + edge(M('r.mjs'), plain.id, 'reads', { tail: true }), + edge(M('r.mjs'), plain.id, 'reads'), + ]); + + // act + const d = fromFlowGraph(graph); + const a = analyse(d); + + // assert + assert.deepEqual(d.nodes.filter((n) => n.kind === 'store').map((n) => [n.label, n.subtype]), [['state/events.jsonl', 'log'], ['state/notes.txt', 'file']]); + assert.deepEqual(flowsOf(d), [ + 'node:module:w.mjs > node:store:state/events.jsonl appends async', + 'node:store:state/events.jsonl > node:module:r.mjs tails async', + 'node:module:w.mjs > node:store:state/notes.txt writes sync', + 'node:store:state/notes.txt > node:module:r.mjs reads sync', + 'node:store:state/notes.txt > node:module:r.mjs reads sync', + ]); + assert.deepEqual(a.brokered, [{ from: 'node:module:w.mjs', via: 'node:store:state/events.jsonl', to: 'node:module:r.mjs' }]); +}); + +test('an append to a file nobody tails stays a file and a sync flow', () => { + // arrange + const s = makeNode('store', 'out.log'); + const graph = graphOf([...mods('w.mjs'), s], [edge(M('w.mjs'), s.id, 'writes', { append: true })]); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual([d.nodes[1].subtype, d.flows[0].label, d.flows[0].mode], ['file', 'appends', 'sync']); +}); + +// ── the flow-graph adapter: the double element, the reconcile and layers ───────────────────────── + +test('a path literal that names a module is that module\'s process, not a second element', () => { + // arrange + const graph = graphOf([...mods('bin/a.mjs', 'bin/b.mjs'), makeNode('artifact', 'bin/b.mjs'), makeNode('artifact', 'bin/a.mjs')], [ + edge(M('bin/a.mjs'), nodeId('artifact', 'bin/b.mjs'), 'reads'), + edge(M('bin/a.mjs'), nodeId('artifact', 'bin/a.mjs'), 'reads'), + ]); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual(d.nodes.map((n) => [n.id, n.kind]), [['node:module:bin/a.mjs', 'process'], ['node:module:bin/b.mjs', 'process']]); + assert.deepEqual(flowsOf(d), ['node:module:bin/b.mjs > node:module:bin/a.mjs reads sync']); + assert.equal(d.source.options.merged.pathsThatAreModules, 2); + assert.equal(d.source.omitted.selfReferences, 1); +}); + +test('the reconcile replaces witness and existence, and existence is confirmed only when it is', () => { + // arrange + const s = makeNode('store', 'out.json'); + const graph = graphOf([...mods('a.mjs', 'b.mjs'), s], [ + makeEdge(M('a.mjs'), s.id, 'writes', { evidence: ['a.mjs:1 writeFileSync'] }), + makeEdge(M('b.mjs'), s.id, 'reads', { evidence: ['b.mjs:1 readFileSync'] }), + ]); + const reconcile = { v: 1, generatedAt: '2026-11-12T00:00:00.000Z', coverage: { modules: 2 }, observedPaths: {}, + edges: [{ v: 1, from: M('a.mjs'), to: s.id, kind: 'writes', witness: 'both', existence: 'confirmed', runtimeCoverage: 'visited' }, + { v: 1, from: M('b.mjs'), to: s.id, kind: 'reads', witness: 'static', existence: 'unknown', runtimeCoverage: 'unvisited' }] }; + + // act + const d = fromFlowGraph(graph, { reconcile }); + + // assert + assert.deepEqual(d.flows.map((f) => [f.label, f.existence, f.evidence.witness, f.evidence.existence]), [['writes', 'confirmed', 'both', 'confirmed'], ['reads', 'unknown', 'static', 'unknown']]); + assert.deepEqual(d.source.options.reconcile, { edges: 2, generatedAt: '2026-11-12T00:00:00.000Z' }); + assert.equal(d.source.options.merged.reconcileMatched, 2); +}); + +test('a touch the runtime resolved becomes directed flows carrying the direction it saw', () => { + // arrange + const stores = ['r.json', 'w.json', 'rw.json', 'x.json', 'y.json'].map((p) => makeNode('artifact', p)); + const graph = graphOf([...mods('m.mjs'), ...stores], stores.map((s) => makeEdge(M('m.mjs'), s.id, 'touches'))); + const dir = { 'r.json': 'reads', 'w.json': 'write', 'rw.json': 'both', 'x.json': 'sideways' }; + const reconcile = { edges: Object.entries(dir).map(([p, direction]) => ({ from: M('m.mjs'), to: nodeId('artifact', p), kind: 'touches', witness: 'both', existence: 'confirmed', direction })) }; + + // act + const d = fromFlowGraph(graph, { reconcile }); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(d.flows.map((f) => [f.from.replace('node:', ''), f.to.replace('node:', ''), f.label, f.direction ?? null, f.directed ?? true]), [ + ['artifact:r.json', 'module:m.mjs', 'reads', 'reads', true], + ['module:m.mjs', 'artifact:w.json', 'writes', 'writes', true], + ['artifact:rw.json', 'module:m.mjs', 'reads', 'both', true], + ['module:m.mjs', 'artifact:rw.json', 'writes', 'both', true], + ['module:m.mjs', 'artifact:x.json', 'touches', null, false], + ['module:m.mjs', 'artifact:y.json', 'touches', null, false], + ]); + assert.equal(d.source.options.merged.directionsResolved, 3); + assert.deepEqual(analyse(d).bidirectional.map((b) => [b.a, b.b]), [['node:module:m.mjs', 'node:artifact:rw.json']]); +}); + +test('observedPaths renames a bare filename with one location and splits one with several', () => { + // arrange + const one = makeNode('artifact', 'rollup.json', { composed: true }); + const many = makeNode('artifact', 'state.json', { composed: true }); + const none = makeNode('artifact', 'ghost.json', { composed: true }); + const graph = graphOf([...mods('a.mjs'), one, many, none], [ + makeEdge(M('a.mjs'), one.id, 'writes', { witness: 'both', existence: 'confirmed' }), + makeEdge(M('a.mjs'), many.id, 'reads', { witness: 'both', existence: 'confirmed' }), + makeEdge(M('a.mjs'), none.id, 'reads'), + ]); + const reconcile = { edges: [], observedPaths: { 'rollup.json': ['reports/x/rollup.json'], [many.id]: ['reports/a/state.json', 'reports/b/state.json'] } }; + + // act + const d = fromFlowGraph(graph, { reconcile }); + const stores = d.nodes.filter((n) => n.kind === 'store'); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(stores.map((n) => [n.id, n.label, n.unknown ?? false]), [ + [one.id, 'reports/x/rollup.json', false], + [`${many.id}@reports/a/state.json`, 'reports/a/state.json', false], + [`${many.id}@reports/b/state.json`, 'reports/b/state.json', false], + [none.id, 'ghost.json', true], + ]); + assert.match(stores[1].note, /one of 2 paths the runtime opened under that name; which module used which path was not observed/); + assert.deepEqual(d.flows.map((f) => [f.to === M('a.mjs') ? f.from : f.to, f.existence]), [ + [one.id, 'confirmed'], + [`${many.id}@reports/a/state.json`, 'unknown'], + [`${many.id}@reports/b/state.json`, 'unknown'], + [none.id, 'unknown'], + ]); + assert.deepEqual([d.source.options.merged.storesRenamed, d.source.options.merged.storesSplit], [1, 1]); +}); + +test('a layer map puts processes and stores in maturity layers and leaves externals out of them', () => { + // arrange + const graph = graphOf([...mods('bin/scan.mjs', 'monitor/rollup.mjs'), makeNode('store', 'reports/raw.sarif'), makeNode('env', 'CW_HOME')], [ + makeEdge(M('bin/scan.mjs'), nodeId('store', 'reports/raw.sarif'), 'writes'), + makeEdge(M('monitor/rollup.mjs'), nodeId('store', 'reports/raw.sarif'), 'reads'), + makeEdge(M('monitor/rollup.mjs'), nodeId('env', 'CW_HOME'), 'reads'), + ]); + const layers = { layers: ['raw', 'normalised'], rules: [{ glob: '**/*.sarif', layer: 'raw' }, { prefix: 'bin/', layer: 'raw' }, { prefix: 'monitor/', layer: 'normalised' }] }; + + // act + const d = fromFlowGraph(graph, { layers }); + const asRules = fromFlowGraph(graph, { layers: layers.rules }); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(d.layers, ['raw', 'normalised']); + assert.deepEqual(d.nodes.map((n) => [n.label, n.layer ?? null]), [['bin/scan.mjs', 'raw'], ['monitor/rollup.mjs', 'normalised'], ['reports/raw.sarif', 'raw'], ['CW_HOME', null]]); + assert.equal(d.source.options.layers, 3); + assert.deepEqual(asRules.layers, ['raw', 'normalised']); + assert.throws(() => fromFlowGraph(graph, { layers: { layers: ['raw'], rules: [{ prefix: 'bin/', layer: 'gold' }] } }), /names layer "gold"/); +}); + +test('a node or edge kind the adapter does not know is counted, never drawn as a guess', () => { + // arrange + const graph = graphOf([...mods('a.mjs'), raw('socket', '/tmp/s'), makeNode('store', 'x.json')], [ + edge(M('a.mjs'), 'node:socket:/tmp/s', 'writes'), + edge(M('a.mjs'), nodeId('store', 'x.json'), 'mutates'), + ]); + + // act + const d = fromFlowGraph(graph); + + // assert + assert.deepEqual([d.source.omitted.unknownNodeKinds, d.source.omitted.unknownEdgeKinds, d.source.omitted.edgesOfOmittedNodes], [1, 1, 1]); + assert.deepEqual(d.flows, []); +}); + +// ── layer maps ─────────────────────────────────────────────────────────────────────────────────── + +test('a glob matches within a segment with *, at any depth with **, and one character with ?', () => { + // arrange + const cases = [['reports/*.json', 'reports/a.json', true], ['reports/*.json', 'reports/x/a.json', false], ['**/rollup.json', 'rollup.json', true], + ['**/rollup.json', 'reports/x/rollup.json', true], ['bin/**', 'bin/a/b.mjs', true], ['a?.mjs', 'ab.mjs', true], ['a?.mjs', 'a/.mjs', false], ['a+b.json', 'a+b.json', true]]; + + // act + const got = cases.map(([g, p]) => globToRegExp(g).test(p)); + + // assert + assert.deepEqual(got, cases.map((c) => c[2])); +}); + +test('layerOf takes the first rule that matches, and a path no rule matches has no layer', () => { + // arrange + const map = { layers: ['a', 'b'], rules: [{ prefix: 'x/y', layer: 'b' }, { glob: 'x/**', layer: 'a' }] }; + + // act + const got = ['x/y/z', 'x/q', 'elsewhere'].map((p) => layerOf(p, map)); + + // assert + assert.deepEqual(got, ['b', 'a', undefined]); + assert.equal(layerOf('x/q', map.rules), 'a'); +}); + +test('validateLayerMap refuses every malformed shape, by name', () => { + // arrange + const bad = [null, [], { layers: [], rules: [] }, { layers: ['a', 'a'], rules: [{ prefix: 'x', layer: 'a' }] }, + { v: 2, kind: 'other', layers: ['a'], rules: [{ prefix: 'x', glob: 'y', layer: 'a', extra: 1 }], more: true }]; + + // act + const errors = bad.map((m) => validateLayerMap(m).errors.join(' | ')); + + // assert + assert.match(errors[0], /not an object/); + assert.match(errors[1], /not an object/); + assert.match(errors[2], /layers must be a non-empty array.*rules must be a non-empty array/); + assert.match(errors[3], /a layer is declared more than once/); + assert.match(errors[4], /unknown field "more".*v must be 1.*kind must be "dfd-layers".*unknown field\(s\) extra.*exactly one of prefix and glob/); + assert.deepEqual(validateLayerMap({ v: 1, kind: 'dfd-layers', layers: ['a'], rules: [{ prefix: 'x', layer: 'a', why: 'because' }] }), { ok: true, errors: [] }); +}); + +// ── the code-flow adapter ──────────────────────────────────────────────────────────────────────── + +const step = (uri, line, text) => ({ location: { physicalLocation: { artifactLocation: { uri }, region: { startLine: line } }, message: { text } } }); +const sarif = (rule, locs) => ({ ruleId: rule, locations: [{ physicalLocation: { artifactLocation: { uri: locs[locs.length - 1][0] } } }], + codeFlows: [{ threadFlows: [{ locations: locs.map(([u, l, t]) => step(u, l, t)) }] }] }); +const NOW = '2026-11-12T00:00:00.000Z'; + +test('a CodeQL trace becomes input -> source file -> sink file -> sink, typed by what the rule sinks into', () => { + // arrange + const records = [ + dataflowFromResult(sarif('js/sql-injection', [['api/routes.js', 4, 'req.query.id'], ['api/x.js', 7, 'id'], ['api/db.js', 12, 'query(sql)']]), 'shop', NOW), + dataflowFromResult(sarif('js/reflected-xss', [['api/routes.js', 4, 'req.query.id'], ['api/routes.js', 9, 'res.send(id)']]), 'shop', NOW), + ]; + + // act + const d = fromCodeFlows(records, { title: 'shop' }); + + // assert + assert.deepEqual(validateDfd(d).errors, []); + assert.equal(d.title, 'shop'); + assert.deepEqual(d.groups, [{ id: 'repo:shop', label: 'shop', boundary: 'system' }, { id: 'untrusted', label: 'Untrusted input', boundary: 'trust' }]); + assert.deepEqual(d.nodes.map((n) => [n.id, n.kind, n.subtype ?? null]), [ + ['input:shop:api/routes.js:4', 'external', 'actor'], + ['code:shop:api/routes.js', 'process', null], + ['code:shop:api/db.js', 'process', null], + ['sink:shop:database query', 'store', 'database'], + ['sink:shop:browser that renders the response', 'external', 'actor'], + ]); + assert.deepEqual(d.flows.map((f) => `${f.from} > ${f.to}: ${f.label}`), [ + 'input:shop:api/routes.js:4 > code:shop:api/routes.js: req.query.id', + 'code:shop:api/routes.js > code:shop:api/db.js: 3 steps (js/sql-injection)', + 'code:shop:api/db.js > sink:shop:database query: query(sql) (js/sql-injection)', + 'code:shop:api/routes.js > sink:shop:browser that renders the response: res.send(id) (js/reflected-xss)', + ]); + assert.deepEqual(d.flows[0].evidence, { witness: 'static', existence: 'unknown', refs: ['js/sql-injection api/routes.js:4 -> api/db.js:12', 'js/reflected-xss api/routes.js:4 -> api/routes.js:9'] }); + assert.ok(d.flows.every((f) => f.existence === 'unknown')); + assert.equal(analyse(d).paths.total, 2); +}); + +test('a record without a path draws nothing and is counted; so is one that is not a record', () => { + // arrange + const records = [ + dataflowFromResult({ ruleId: 'js/incomplete-sanitization', locations: [] }, 'shop', NOW), + null, + { repo: 'shop' }, + dataflowFromResult(sarif('js/command-line-injection', [['cli.js', 1, null], ['cli.js', 2, null]]), 'shop', NOW), + ]; + + // act + const d = fromCodeFlows(records); + + // assert + assert.deepEqual(d.source.omitted, { noPath: 1, unreadable: 2 }); + assert.equal(d.title, 'code flows'); + assert.equal(d.source.producer, 'codeql-code-flows'); + assert.deepEqual(d.flows.map((f) => f.label), ['untrusted input', 'js/command-line-injection']); + assert.equal(d.nodes[0].label, 'cli.js:1'); + assert.deepEqual(validateDfd(d).errors, []); + assert.deepEqual(fromCodeFlows(undefined).nodes, []); +}); + +test('sinkKindOf names the sink from the rule id, and an unnamed one is an unknown system', () => { + // arrange + const rules = ['js/sql-injection', 'py/log-injection', 'js/path-injection', 'go/command-injection', 'js/request-forgery', 'js/xss-through-dom', 'js/code-injection', 'java/weird-rule', undefined]; + + // act + const got = rules.map((r) => { const s = sinkKindOf(r); return `${s.kind}:${s.subtype}${s.unknown ? ':unknown' : ''}`; }); + + // assert + assert.deepEqual(got, ['store:database', 'store:log', 'store:file', 'external:binary', 'external:endpoint', 'external:actor', 'external:system', 'external:system:unknown', 'external:system:unknown']); +}); + +test('an unclassified sink is marked unknown and says which rule could not be read', () => { + // arrange + const records = [dataflowFromResult(sarif('java/weird-rule', [['A.java', 1, 'in'], ['B.java', 2, 'out']]), 'svc', NOW)]; + + // act + const d = fromCodeFlows(records); + const sink = d.nodes.find((n) => n.id.startsWith('sink:')); + + // assert + assert.equal(sink.unknown, true); + assert.match(sink.note, /the rule id java\/weird-rule does not say what its sink is/); +}); From a42c015a276cc83503cb80b58bcd4d647bd0db4f Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:42:15 +1030 Subject: [PATCH 11/36] fix(flow): keep each wrapped builtin's name and length under the trace hook --- flow/test/runtime-network.test.mjs | 18 ++++++++++++++++++ flow/trace-hook.cjs | 26 +++++++++++++++++--------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/flow/test/runtime-network.test.mjs b/flow/test/runtime-network.test.mjs index 0a8fe1578..d2fe52356 100644 --- a/flow/test/runtime-network.test.mjs +++ b/flow/test/runtime-network.test.mjs @@ -143,3 +143,21 @@ test('an explicit listen port is recorded at once, as listen:<port>, not ephemer assert.equal(listens[0].ephemeral, undefined, 'named once is named: a literal could have matched it'); } finally { rmSync(d, { recursive: true, force: true }); } }); + +test('a wrapped builtin keeps its name and length — tracing must not change what code can inspect', () => { + // Arrange + const d = fixtureRepo(); + try { + writeFileSync(join(d, 'bin', 'shape.mjs'), + "import * as fs from 'node:fs';\nimport http from 'node:http';\nimport net from 'node:net';\nimport cp from 'node:child_process';\n" + + "const fns = { readFileSync: fs.readFileSync, openSync: fs.openSync, request: http.request, connect: net.connect, listen: net.Server.prototype.listen, spawnSync: cp.spawnSync, fetch: globalThis.fetch };\n" + + "console.log(JSON.stringify(Object.fromEntries(Object.entries(fns).map(([k, f]) => [k, [f.name, f.length]]))));\n"); + // Act + const plain = spawnSync(process.execPath, ['bin/shape.mjs'], { cwd: d, encoding: 'utf8' }); + const traced = spawnSync(process.execPath, ['-r', hookPath(), 'bin/shape.mjs'], + { cwd: d, encoding: 'utf8', env: { ...process.env, CW_FLOW_TRACE: join(d, 't.jsonl'), CW_FLOW_ROOT: d } }); + // Assert + assert.ok(readTrace(join(d, 't.jsonl')).state === 'usable', 'the hook was really on'); + assert.deepEqual(JSON.parse(traced.stdout), JSON.parse(plain.stdout)); + } finally { rmSync(d, { recursive: true, force: true }); } +}); diff --git a/flow/trace-hook.cjs b/flow/trace-hook.cjs index 57d13b0ff..83b4c2b81 100644 --- a/flow/trace-hook.cjs +++ b/flow/trace-hook.cjs @@ -169,6 +169,17 @@ function install() { let probing = false; let depth = 0; // network calls nest: http.request -> net -> Socket#connect const wrappers = new WeakSet(); + // A wrapper keeps the original's `name` and `length`: code that inspects a function (a table of fs + // methods checked by name, a promisify keyed on length) must see the same function shape traced + // as untraced, or the trace changes the behaviour it is measuring. + const like = (w, orig) => { + try { + Object.defineProperty(w, 'name', { value: orig.name, configurable: true }); + Object.defineProperty(w, 'length', { value: orig.length, configurable: true }); + } catch { /* a frozen function property is cosmetic */ } + wrappers.add(w); + return w; + }; try { mkdirSync.call(fs, path.dirname(OUT), { recursive: true }); } catch { /* the append will report it */ } @@ -230,7 +241,7 @@ function install() { } return orig.apply(this, args); }; - wrappers.add(obj[name]); + like(obj[name], orig); }; const READS = ['readFileSync', 'readFile', 'createReadStream', 'readdirSync', 'readdir', @@ -245,10 +256,10 @@ function install() { for (const n of SPAWNS) { const orig = cp[n]; if (typeof orig !== 'function') continue; - cp[n] = function (...args) { + cp[n] = like(function (...args) { emit({ t: 'spawn', path: String(args[0]), actor: actorOf(), via: n, at: Date.now() }); return orig.apply(this, args); - }; + }, orig); } // ── network ────────────────────────────────────────────────────────────────────────────────── @@ -268,8 +279,7 @@ function install() { depth += 1; try { return orig.apply(this, args); } finally { depth -= 1; } }; - obj[name] = w; - wrappers.add(w); + obj[name] = like(w, orig); }; netWrap(net, 'connect', 'net.connect', (a) => socketKey(a, 'tcp')); @@ -293,9 +303,7 @@ function install() { emit({ t: 'connect', key, actor: actorOf(), via: 'fetch', at: Date.now() }); return Reflect.apply(orig, this, args); }; - Object.defineProperty(fetch, 'length', { value: orig.length }); - Object.defineProperty(globalThis, 'fetch', { ...fetchDesc, value: fetch }); - wrappers.add(fetch); + Object.defineProperty(globalThis, 'fetch', { ...fetchDesc, value: like(fetch, orig) }); } const origListen = net.Server.prototype.listen; @@ -316,7 +324,7 @@ function install() { } return origListen.apply(this, args); }; - wrappers.add(net.Server.prototype.listen); + like(net.Server.prototype.listen, origListen); // Non-vacuity, written by the hook itself: a trace with no `installed` row means the preload never // ran, which flow/runtime.mjs reports as UNUSABLE rather than as a clean, edgeless run. From 9028f8c638dd295e0458be011a541e3fe6b64f03 Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:42:22 +1030 Subject: [PATCH 12/36] Add --direction, --collapse, --collapse-all-groups-over, --orientation and --help to the dfd command --- flow/dfd-cli.mjs | 145 +++++++++++++++++++++++++++---------- flow/test/dfd-cli.test.mjs | 145 +++++++++++++++++++++++++++++++++++-- 2 files changed, 247 insertions(+), 43 deletions(-) diff --git a/flow/dfd-cli.mjs b/flow/dfd-cli.mjs index e97267d76..d49f7aaef 100644 --- a/flow/dfd-cli.mjs +++ b/flow/dfd-cli.mjs @@ -3,38 +3,70 @@ // usage: node flow/dfd-cli.mjs --from <dfd.json> [options] // node flow/dfd-cli.mjs --graph [<flow.json>] [--tests] [--imports] [options] // -// --from <file> a dfd@1 document, hand-authored or emitted by a tool (schema/dfd.schema.json) -// --graph [<file>] the flow graph through the adapter; default <CW_HARNESS_STORE>/flow.json, -// which `node flow/static.mjs` or `node flow/report.mjs` writes -// --tests keep test modules and fixtures (left out by default) -// --imports keep module-to-module imports as flows (left out by default: not data flows) -// --prefix <p> seed the filter with every node whose label or id starts with <p> -// --focus <node> seed the filter with one node, by id or label -// --hops <n> keep everything within n flows of the seeds (default 1) -// --max <n> refuse to draw more than n elements (default 150); filter, or raise it -// --emit <what> html (default) | model (the dfd@1 document) | analysis (the analysis as JSON) -// --fonts <mode> inline (default) | none | served — see lib/house-css.mjs -// --title <text> the diagram's title -// --out <file> write there (tmp+rename) and print one summary line; default is stdout +// Options are listed in HELP below (`--help` prints them). // // exit: 0 written · 2 usage · 20 the document is not a valid dfd@1 · 21 an input is absent, unreadable -// or not a flow graph · 22 the filter matched no node, or the model is larger than --max +// or not a flow graph · 22 the filter matched no node, --collapse named a group with no element, +// or the model is larger than --max // env, read at call time: CW_HARNESS_STORE (the flow store directory, default reports/harness; --graph // without a path reads flow.json there) import { readJson, storePath } from './store.mjs'; import { validateDfd, analyse, fromFlowGraph, filterModel } from '../lib/dfd.mjs'; -import { renderHtml } from '../lib/dfd-render.mjs'; +import { renderHtml, groupSizes, ORIENTATIONS } from '../lib/dfd-render.mjs'; import { writeAtomic } from '../monitor/lockfile.mjs'; import { isMainModule } from '../lib/is-main.mjs'; export const EXIT = Object.freeze({ OK: 0, USAGE: 2, INVALID: 20, INPUT: 21, SIZE: 22 }); export const DEFAULT_MAX = 150; -const VALUE_FLAGS = ['from', 'prefix', 'focus', 'hops', 'max', 'emit', 'fonts', 'title', 'out']; -const BOOL_FLAGS = ['tests', 'imports']; +const VALUE_FLAGS = ['from', 'prefix', 'focus', 'hops', 'direction', 'collapse', 'collapse-all-groups-over', 'orientation', 'max', 'emit', 'fonts', 'title', 'out']; +const BOOL_FLAGS = ['tests', 'imports', 'help']; const EMIT = ['html', 'model', 'analysis']; const FONTS = ['inline', 'none', 'served']; +const DIRECTIONS = ['in', 'out', 'both']; +const WHOLE = { hops: 'hops', max: 'max', 'collapse-all-groups-over': 'collapseOver' }; + +export const USAGE = 'usage: node flow/dfd-cli.mjs --from <dfd.json> | --graph [<flow.json>] [--prefix p] [--focus node] [--hops n] [--direction in|out|both] ' + + '[--collapse g[,g]] [--collapse-all-groups-over n] [--orientation LR|TB|auto] [--emit html|model|analysis] [--out file]'; + +export const HELP = `Draw a data flow diagram (Gane–Sarson) from a dfd@1 file or from the flow graph. + +${USAGE} + +source (exactly one) + --from <file> a dfd@1 document, hand-authored or emitted by a tool (schema/dfd.schema.json) + --graph [<file>] the flow graph through the adapter; default <CW_HARNESS_STORE>/flow.json, + which \`node flow/static.mjs\` or \`node flow/report.mjs\` writes + --tests keep test modules and fixtures (left out by default) + --imports keep module-to-module imports as flows (left out by default: not data flows) + +filter + --prefix <p> seed the filter with every node whose label or id starts with <p> + --focus <node> seed the filter with one node, by id or label + --hops <n> keep everything within n flows of the seeds (default 1) + --direction <d> in: follow flows into the seeds only · out: out of them only · both (default) + +drawing + --collapse <g,...> draw each named group (boundary) as one node; flows across it merge per other + end, mode and role, with the count in the label + --collapse-all-groups-over <n> + collapse every group with more than n elements + --orientation <o> LR (ranks left to right) | TB (top to bottom) | auto (default: TB when LR is + wider than 1600 units and TB is narrower) + --max <n> refuse to draw more than n elements (default ${DEFAULT_MAX}), counted after collapsing + --fonts <mode> inline (default) | none | served — see lib/house-css.mjs + --title <text> the diagram's title + +output + --emit <what> html (default) | model (the filtered dfd@1 document, before any collapse) | + analysis (the analysis as JSON) + --out <file> write there (tmp+rename) and print one summary line; default is stdout + --help this text + +exit: 0 written · 2 usage · 20 not a valid dfd@1 · 21 an input is absent, unreadable or not a flow +graph · 22 the filter matched no node, --collapse named a group with no element, or more than --max +`; /** -> { opts } or { error }. `--graph` takes an optional path. */ export function parseArgs(argv) { @@ -44,17 +76,30 @@ export function parseArgs(argv) { const name = a.startsWith('--') ? a.slice(2) : null; const next = argv[i + 1]; if (name === 'graph') opts.graph = next !== undefined && !next.startsWith('--') ? argv[++i] : true; + else if (a === '-h') opts.help = true; else if (BOOL_FLAGS.includes(name)) opts[name] = true; else if (VALUE_FLAGS.includes(name)) { if (next === undefined) return { error: `--${name} needs a value` }; opts[name] = argv[++i]; } else return { error: `unknown argument ${a}` }; } + if (opts.help) return { opts }; if ((opts.from === undefined) === (opts.graph === undefined)) return { error: 'give exactly one of --from <dfd.json> and --graph [<flow.json>]' }; - for (const k of ['hops', 'max']) { - if (opts[k] === undefined) continue; - if (!/^\d+$/.test(opts[k])) return { error: `--${k} takes a whole number, got ${opts[k]}` }; - opts[k] = Number(opts[k]); + for (const [flag, key] of Object.entries(WHOLE)) { + if (opts[flag] === undefined) continue; + if (!/^\d+$/.test(opts[flag])) return { error: `--${flag} takes a whole number, got ${opts[flag]}` }; + const value = Number(opts[flag]); + delete opts[flag]; + opts[key] = value; + } + if (opts.direction !== undefined) { + if (!DIRECTIONS.includes(opts.direction)) return { error: `--direction takes one of ${DIRECTIONS.join(', ')}` }; + if (opts.prefix === undefined && opts.focus === undefined) return { error: '--direction needs --prefix or --focus: it says which way to walk from the seeds' }; + } + if (opts.orientation !== undefined && !ORIENTATIONS.includes(opts.orientation)) return { error: `--orientation takes one of ${ORIENTATIONS.join(', ')}` }; + if (opts.collapse !== undefined) { + opts.collapse = opts.collapse.split(',').map((g) => g.trim()).filter(Boolean); + if (!opts.collapse.length) return { error: '--collapse takes one or more group ids, comma-separated' }; } if (opts.emit !== undefined && !EMIT.includes(opts.emit)) return { error: `--emit takes one of ${EMIT.join(', ')}` }; if (opts.fonts !== undefined && !FONTS.includes(opts.fonts)) return { error: `--fonts takes one of ${FONTS.join(', ')}` }; @@ -68,41 +113,65 @@ function load(path, what) { return { data: r.data }; } -export function main(argv = process.argv.slice(2), { env = process.env, stdout = process.stdout, stderr = process.stderr } = {}) { - const fail = (code, msg) => { stderr.write(`dfd: ${msg}\n`); return code; }; - const { opts, error } = parseArgs(argv); - if (error) return fail(EXIT.USAGE, `${error}\n usage: node flow/dfd-cli.mjs --from <dfd.json> | --graph [<flow.json>] [--prefix p] [--focus node] [--hops n] [--emit html|model|analysis] [--out file]`); - +/** + * Parsed options -> { model, collapse } (the filtered dfd@1 document and the groups to draw as one + * node each) or { code, error }. Shared with the panel route, so the two read and refuse alike. + */ +export function prepare(opts, { env = process.env, filter = filterModel } = {}) { let model; if (opts.from !== undefined) { const got = load(opts.from, 'dfd document'); - if (got.error) return fail(EXIT.INPUT, got.error); + if (got.error) return { code: EXIT.INPUT, error: got.error }; model = got.data; } else { const path = opts.graph === true ? storePath('flow.json', env) : opts.graph; const got = load(path, 'flow graph'); - if (got.error) return fail(EXIT.INPUT, `${got.error}${opts.graph === true ? ' — run `node flow/static.mjs` to write it' : ''}`); - if (!got.data || !Array.isArray(got.data.nodes) || !Array.isArray(got.data.edges)) return fail(EXIT.INPUT, `${path} is not a flow graph (no nodes and edges arrays)`); + if (got.error) return { code: EXIT.INPUT, error: `${got.error}${opts.graph === true ? ' — run `node flow/static.mjs` to write it' : ''}` }; + if (!got.data || !Array.isArray(got.data.nodes) || !Array.isArray(got.data.edges)) return { code: EXIT.INPUT, error: `${path} is not a flow graph (no nodes and edges arrays)` }; model = fromFlowGraph(got.data, { tests: !!opts.tests, imports: !!opts.imports, title: opts.title }); } const valid = validateDfd(model); - if (!valid.ok) return fail(EXIT.INVALID, `not a valid dfd@1 document:\n${valid.errors.map((e) => ` ${e}`).join('\n')}`); + if (!valid.ok) return { code: EXIT.INVALID, error: `not a valid dfd@1 document:\n${valid.errors.map((e) => ` ${e}`).join('\n')}` }; if (opts.title !== undefined) model = { ...model, title: opts.title }; - model = filterModel(model, { prefix: opts.prefix, focus: opts.focus, hops: opts.hops ?? 1 }); - if (model === null) return fail(EXIT.SIZE, `the filter matched no node (prefix ${JSON.stringify(opts.prefix ?? null)}, focus ${JSON.stringify(opts.focus ?? null)})`); + // `around` is the contract's name for the focus seed; `focus` is passed too until lib/dfd.mjs settles on one + model = filter(model, { prefix: opts.prefix, focus: opts.focus, around: opts.focus, hops: opts.hops ?? 1, + ...(opts.direction === undefined ? {} : { direction: opts.direction }) }); + if (model === null) return { code: EXIT.SIZE, error: `the filter matched no node (prefix ${JSON.stringify(opts.prefix ?? null)}, focus ${JSON.stringify(opts.focus ?? null)})` }; + + const sizes = groupSizes(model); + const named = opts.collapse || []; + const absent = named.filter((g) => !sizes.has(g)); + if (absent.length) return { code: EXIT.SIZE, error: `--collapse names no group with an element in the drawing: ${absent.join(', ')}${sizes.size ? ` (groups: ${[...sizes.keys()].join(', ')})` : ''}` }; + const over = opts.collapseOver === undefined ? [] : [...sizes].filter(([, k]) => k > opts.collapseOver).map(([g]) => g); + const collapse = [...new Set([...named, ...over])]; + return { model, collapse, drawn: model.nodes.length - collapse.reduce((s, g) => s + sizes.get(g) - 1, 0) }; +} + +export function main(argv = process.argv.slice(2), { env = process.env, stdout = process.stdout, stderr = process.stderr, filter = filterModel } = {}) { + const fail = (code, msg) => { stderr.write(`dfd: ${msg}\n`); return code; }; + const { opts, error } = parseArgs(argv); + if (error) return fail(EXIT.USAGE, `${error}\n ${USAGE}\n (--help lists every option)`); + if (opts.help) { stdout.write(HELP); return EXIT.OK; } + + const got = prepare(opts, { env, filter }); + if (got.error) return fail(got.code, got.error); + const { model, collapse, drawn } = got; const emit = opts.emit || 'html'; const max = opts.max ?? DEFAULT_MAX; - if (emit === 'html' && model.nodes.length > max) { - return fail(EXIT.SIZE, `${model.nodes.length} elements is more than --max ${max}: a diagram that size is not readable. Narrow it with --prefix or --focus (and --hops), or raise --max.`); + if (emit === 'html' && drawn > max) { + return fail(EXIT.SIZE, `${drawn} elements is more than --max ${max}: a diagram that size is not readable. Narrow it with --prefix or --focus (and --hops), collapse groups with --collapse or --collapse-all-groups-over, or raise --max.`); } - const text = emit === 'model' ? `${JSON.stringify(model, null, 2)}\n` - : emit === 'analysis' ? `${JSON.stringify(analyse(model), null, 2)}\n` - : renderHtml(model, { fonts: opts.fonts || 'inline' }).html; + let text; + let stats = null; + if (emit === 'model') text = `${JSON.stringify(model, null, 2)}\n`; + else if (emit === 'analysis') text = `${JSON.stringify(analyse(model), null, 2)}\n`; + else ({ html: text, stats } = renderHtml(model, { fonts: opts.fonts || 'inline', collapse, orientation: opts.orientation || 'auto' })); if (opts.out === undefined) { stdout.write(text); return EXIT.OK; } writeAtomic(opts.out, text, { mkdir: true }); - stdout.write(`dfd: ${model.nodes.length} elements, ${model.flows.length} flows -> ${opts.out} (${emit})\n`); + const drawing = stats ? `, ${stats.orientation} ${stats.width}x${stats.height}, ${stats.crossings} crossing(s), ${stats.collisions} label collision(s)` : ''; + stdout.write(`dfd: ${stats ? stats.elements : model.nodes.length} elements, ${stats ? stats.flows : model.flows.length} flows -> ${opts.out} (${emit}${drawing})\n`); return EXIT.OK; } diff --git a/flow/test/dfd-cli.test.mjs b/flow/test/dfd-cli.test.mjs index bad846b59..685572b04 100644 --- a/flow/test/dfd-cli.test.mjs +++ b/flow/test/dfd-cli.test.mjs @@ -7,7 +7,7 @@ import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { main, parseArgs, EXIT, DEFAULT_MAX } from '../dfd-cli.mjs'; +import { main, parseArgs, prepare, EXIT, DEFAULT_MAX, HELP, USAGE } from '../dfd-cli.mjs'; import { validateDfd } from '../../lib/dfd.mjs'; import { makeNode, makeEdge, nodeId } from '../graph.mjs'; @@ -15,10 +15,10 @@ const HERE = dirname(fileURLToPath(import.meta.url)); const FIXTURE = join(HERE, '..', '..', 'lib', 'test', 'fixtures', 'dfd-synthetic.json'); const tmp = () => mkdtempSync(join(tmpdir(), 'flow-dfd-')); -function run(argv, env = {}) { +function run(argv, env = {}, extra = {}) { const out = []; const err = []; - const code = main(argv, { env, stdout: { write: (s) => out.push(s) }, stderr: { write: (s) => err.push(s) } }); + const code = main(argv, { env, stdout: { write: (s) => out.push(s) }, stderr: { write: (s) => err.push(s) }, ...extra }); return { code, out: out.join(''), err: err.join('') }; } @@ -110,7 +110,7 @@ test('--out writes the page whole, leaves no temp file and prints one line', () // assert assert.equal(r.code, EXIT.OK); - assert.equal(r.out, `dfd: 16 elements, 20 flows -> ${out} (html)\n`); + assert.equal(r.out, `dfd: 16 elements, 20 flows -> ${out} (html, TB 1460x1776, 0 crossing(s), 0 label collision(s))\n`); assert.match(readFileSync(out, 'utf8'), /url\("data:font\/woff2;base64,/, 'the default page carries its fonts'); assert.deepEqual(readdirSync(join(dir, 'deep')), ['page.html']); }); @@ -186,7 +186,7 @@ test('a usage error exits 2 and shows the usage line', () => { // assert assert.equal(r.code, EXIT.USAGE); - assert.match(r.err, /--emit takes one of[\s\S]*usage: node flow\/dfd-cli\.mjs/); + assert.match(r.err, /--emit takes one of[\s\S]*usage: node flow\/dfd-cli\.mjs[\s\S]*--help lists every option/); }); test('a filter that matches nothing, or a model past --max, exits 22 and draws nothing', () => { @@ -277,3 +277,138 @@ test('run as a command it exits with main\'s code', () => { assert.equal(bad.status, 2); assert.match(bad.stderr, /^dfd: give exactly one of/); }); + +// ── direction, collapse, orientation, help ─────────────────────────────────────────────────────── + +test('--help prints every option and exits 0 without needing a source', () => { + // arrange + const argv = ['--help']; + + // act + const r = run(argv); + const short = run(['-h', '--from']); + + // assert + assert.equal(r.code, EXIT.OK); + assert.equal(r.out, HELP); + for (const flag of ['--from', '--graph', '--prefix', '--focus', '--hops', '--direction', '--collapse ', '--collapse-all-groups-over', '--orientation', '--max', '--emit', '--out']) { + assert.ok(HELP.includes(flag), flag); + } + assert.ok(HELP.includes(USAGE)); + assert.equal(short.code, EXIT.USAGE, 'a flag missing its value is still a usage error'); +}); + +test('parseArgs reads --direction, --collapse, --collapse-all-groups-over and --orientation', () => { + // arrange + const argv = ['--from', 'f.json', '--focus', 'n', '--direction', 'in', '--collapse', 'a, b,,', '--collapse-all-groups-over', '12', '--orientation', 'TB']; + + // act + const { opts, error } = parseArgs(argv); + + // assert + assert.equal(error, undefined); + assert.deepEqual(opts, { from: 'f.json', focus: 'n', direction: 'in', collapse: ['a', 'b'], collapseOver: 12, orientation: 'TB' }); +}); + +test('parseArgs refuses a bad direction, a direction with no seed, a bad orientation, an empty collapse and a fractional threshold', () => { + // arrange + const cases = [ + [['--from', 'f', '--prefix', 'p', '--direction', 'up'], /--direction takes one of in, out, both/], + [['--from', 'f', '--direction', 'in'], /--direction needs --prefix or --focus/], + [['--from', 'f', '--orientation', 'RL'], /--orientation takes one of LR, TB, auto/], + [['--from', 'f', '--collapse', ','], /--collapse takes one or more group ids/], + [['--from', 'f', '--collapse-all-groups-over', '2.5'], /--collapse-all-groups-over takes a whole number, got 2\.5/], + ]; + + // act + const errors = cases.map(([argv]) => parseArgs(argv).error); + + // assert + cases.forEach(([, re], i) => assert.match(errors[i], re)); +}); + +test('--direction and the focus pass through to the filter unchanged', () => { + // arrange + const seen = []; + const stub = (model, args) => { seen.push(args); return model; }; + + // act + const r = run(['--from', FIXTURE, '--focus', 'validate', '--hops', '2', '--direction', 'out', '--emit', 'model'], {}, { filter: stub }); + const plain = run(['--from', FIXTURE, '--emit', 'model'], {}, { filter: stub }); + + // assert + assert.equal(r.code, EXIT.OK); + assert.equal(plain.code, EXIT.OK); + assert.deepEqual(seen, [ + { prefix: undefined, focus: 'validate', around: 'validate', hops: 2, direction: 'out' }, + { prefix: undefined, focus: undefined, around: undefined, hops: 1 }, + ]); +}); + +test('--collapse draws the named group as one node and --collapse-all-groups-over picks groups by size', () => { + // arrange + const dir = tmp(); + const one = join(dir, 'one.html'); + const big = join(dir, 'big.html'); + + // act + const a = run(['--from', FIXTURE, '--collapse', 'platform', '--orientation', 'LR', '--out', one]); + const b = run(['--from', FIXTURE, '--collapse-all-groups-over', '1', '--fonts', 'none', '--out', big]); + const none = run(['--from', FIXTURE, '--collapse-all-groups-over', '3', '--emit', 'html', '--fonts', 'none']); + + // assert + assert.equal(a.code, EXIT.OK); + assert.match(a.out, /^dfd: 14 elements, 18 flows -> .*one\.html \(html, LR \d+x\d+, 0 crossing\(s\), 0 label collision\(s\)\)\n$/); + assert.match(readFileSync(one, 'utf8'), /data-id="group:platform"/); + assert.equal(b.code, EXIT.OK); + assert.doesNotMatch(readFileSync(big, 'utf8'), /data-id="group:internet"/, 'internet has one element, not more than one'); + assert.match(readFileSync(big, 'utf8'), /data-id="group:platform"/); + assert.doesNotMatch(none.out, /<strong>Collapsed:/); +}); + +test('--collapse naming a group with no element exits 22 and lists the groups there are', () => { + // arrange + const argv = ['--from', FIXTURE, '--collapse', 'platform,nope']; + const dir = tmp(); + const bare = join(dir, 'bare.json'); + writeFileSync(bare, JSON.stringify({ v: 1, kind: 'dfd', nodes: [{ id: 'a', kind: 'process', label: 'a' }], flows: [] })); + + // act + const r = run(argv); + const empty = run(['--from', bare, '--collapse', 'g']); + + // assert + assert.equal(r.code, EXIT.SIZE); + assert.match(r.err, /--collapse names no group with an element in the drawing: nope \(groups: internet, platform\)/); + assert.match(empty.err, /drawing: g\n$/); + assert.equal(r.out + empty.out, ''); +}); + +test('--max counts the elements left after collapsing', () => { + // arrange + const argv = ['--from', FIXTURE, '--max', '14', '--fonts', 'none']; + + // act + const flat = run(argv); + const folded = run([...argv, '--collapse', 'platform']); + + // assert + assert.equal(flat.code, EXIT.SIZE); + assert.match(flat.err, /16 elements is more than --max 14[\s\S]*--collapse/); + assert.equal(folded.code, EXIT.OK); +}); + +test('prepare returns the filtered model and the groups to collapse, or the code a refusal exits with', () => { + // arrange + const opts = parseArgs(['--from', FIXTURE, '--collapse-all-groups-over', '2']).opts; + + // act + const ok = prepare(opts); + const missing = prepare({ from: join(tmp(), 'none.json') }); + + // assert + assert.deepEqual(ok.collapse, ['platform']); + assert.equal(ok.drawn, 14); + assert.equal(ok.model.nodes.length, 16); + assert.equal(missing.code, EXIT.INPUT); +}); From 2efddfb1e7fe4357d07eed0781fd0be779d03be6 Mon Sep 17 00:00:00 2001 From: Portll <john@portll.net> Date: Thu, 12 Nov 2026 00:44:09 +1030 Subject: [PATCH 13/36] Serve the data flow diagram in the panel at /dfd/ and /api/dfd, and ship flow/ so it can --- admin/routes/dfd.mjs | 74 +++++++++++ admin/serve.mjs | 3 +- admin/test/dfd-route.test.mjs | 133 ++++++++++++++++++++ admin/test/dfd-routes-entry.test.mjs | 175 +++++++++++++++++++++++++++ admin/test/route-auth.test.mjs | 5 +- admin/test/route-inventory.test.mjs | 8 +- package.json | 1 + 7 files changed, 393 insertions(+), 6 deletions(-) create mode 100644 admin/routes/dfd.mjs create mode 100644 admin/test/dfd-route.test.mjs create mode 100644 admin/test/dfd-routes-entry.test.mjs diff --git a/admin/routes/dfd.mjs b/admin/routes/dfd.mjs new file mode 100644 index 000000000..cc2589b8b --- /dev/null +++ b/admin/routes/dfd.mjs @@ -0,0 +1,74 @@ +// admin/routes/dfd.mjs — the data flow diagram behind the panel login. +// +// GET /dfd/ the page (renderHtml), sent under the page's own hash-pinned CSP +// GET /api/dfd the filtered dfd@1 document as JSON +// +// Source, read at call time: the dfd@1 file CW_DFD_FILE names, else the flow store's graph +// (<CW_HARNESS_STORE>/flow.json) through the adapter. The query mirrors the command's flags — +// prefix, focus, hops, direction, collapse, collapse-all-groups-over, orientation, max, tests, +// imports — and goes through the command's own parser and loader (flow/dfd-cli.mjs), so the page +// and `node flow/dfd-cli.mjs` read, filter and refuse alike. collapse and orientation shape the +// drawing only; /api/dfd returns the document before either. +// +// Dispatched below the login gate (admin/serve.mjs MODULAR_ROUTES), so every route here already +// has a session. + +import { parseArgs, prepare, EXIT, DEFAULT_MAX } from '../../flow/dfd-cli.mjs'; +import { renderHtml } from '../../lib/dfd-render.mjs'; +import { esc } from '../../lib/html-escape.mjs'; + +const FILTER_PARAMS = ['prefix', 'focus', 'hops', 'direction']; +const DRAWING_PARAMS = ['collapse', 'collapse-all-groups-over', 'orientation', 'max']; +const BOOL_PARAMS = ['tests', 'imports']; +const STATUS = { [EXIT.USAGE]: 400, [EXIT.INVALID]: 422, [EXIT.INPUT]: 404, [EXIT.SIZE]: 422 }; + +/** The request's query -> the command's argv, its source fixed by the environment. */ +export function argvFor(url, env = process.env, { drawing = true } = {}) { + const q = new URL(url, 'http://panel.invalid').searchParams; + const argv = env.CW_DFD_FILE ? ['--from', env.CW_DFD_FILE] : ['--graph']; + for (const k of [...FILTER_PARAMS, ...(drawing ? DRAWING_PARAMS : [])]) if (q.has(k)) argv.push(`--${k}`, q.get(k)); + for (const k of BOOL_PARAMS) if (q.get(k) === '1' || q.get(k) === 'true') argv.push(`--${k}`); + return argv; +} + +/** -> { status, opts, model, collapse, drawn } or { status, error }. `drawing: false` ignores the drawing keys. */ +export function load(url, env = process.env, { drawing = true } = {}) { + // a value flag always takes the next argument as its value, so a query value cannot become a flag + const { opts, error } = parseArgs(argvFor(url, env, { drawing })); + if (error) return { status: 400, error }; + const got = prepare(opts, { env }); + if (got.error) return { status: STATUS[got.code], error: got.error }; + return { status: 200, opts, ...got }; +} + +const errorPage = (status, msg) => `<!doctype html><html lang="en"><head><meta charset="utf-8"><title>data flow diagram unavailable` + + `

    data flow diagram unavailable

    ${esc(msg)}

    Status ${status}.

    `; + +function page(ctx) { + const got = load(ctx.req.url); + const max = got.opts?.max ?? DEFAULT_MAX; + if (got.status === 200 && got.drawn > max) { + got.status = 422; + got.error = `${got.drawn} elements is more than max ${max}: narrow it with prefix or focus, collapse groups, or raise max`; + } + if (got.status !== 200) return ctx.send(got.status, errorPage(got.status, got.error), 'text/html; charset=utf-8', { 'cache-control': 'no-store' }); + const { html, csp } = renderHtml(got.model, { fonts: 'served', collapse: got.collapse, orientation: got.opts.orientation || 'auto' }); + return ctx.send(200, html, 'text/html; charset=utf-8', { + 'content-security-policy': csp, + 'content-security-policy-report-only': csp, + 'cache-control': 'no-store', + 'x-robots-tag': 'noindex, nofollow', + }); +} + +function api(ctx) { + const got = load(ctx.req.url, process.env, { drawing: false }); + if (got.status !== 200) return ctx.send(got.status, { ok: false, error: got.error }); + return ctx.send(200, got.model, 'application/json', { 'cache-control': 'no-store' }); +} + +export const routes = [ + { method: 'GET', path: '/dfd/', handle: page }, + { method: 'GET', path: '/dfd', handle: page }, + { method: 'GET', path: '/api/dfd', handle: api }, +]; diff --git a/admin/serve.mjs b/admin/serve.mjs index 7f20cb214..536e993f1 100644 --- a/admin/serve.mjs +++ b/admin/serve.mjs @@ -93,6 +93,7 @@ import { routeFlagOff, offBody } from '../lib/feature-flags.mjs'; import { routes as overwatchLayerRoutes } from './routes/overwatch-layer.mjs'; import { routes as docsiteRoutes, docsiteHandle, docsiteOrigins, docsiteHosts } from './routes/docsite.mjs'; import { routes as launchlistRoutes, launchlistHosts } from './routes/launchlist.mjs'; +import { routes as dfdRoutes } from './routes/dfd.mjs'; import { routes as offboxRoutes, offboxHosts, handleIngest as offboxIngest } from './routes/offbox.mjs'; import { routes as jobRoutes, initJobRoutes } from './routes/jobs.mjs'; import { routes as panelProcessRoutes, initPanelProcessRoutes } from './routes/panel-process.mjs'; @@ -831,7 +832,7 @@ function errorPage(code, title, detail, { back = '/' } = {}) { // throw there took the process down for everyone rather than refusing one request. Three unguarded // decodeURIComponent calls have the same shape. Fail closed, per request, not per process. // Every modular route group, in dispatch order. Adding a group is one import and one entry here. -const MODULAR_ROUTES = [...secretsRoutes, ...leaksCheckRoutes, ...leaksVerifyRoutes, ...postureRoutes, ...reportRoutes, ...remediationRoutes, ...a11yRoutes, ...profileRoutes, ...configEditRoutes, ...scanConfigRoutes, ...ingestRoutes, ...issueDetailRoutes, ...craRoutes, ...determinationRoutes, ...annotationRoutes, ...oversightRoutes, ...correlationRoutes, ...remediationPolicyRoutes, ...llmRuntimeRoutes, ...hostRoutes, ...verdictRoutes, ...packageRoutes, ...updateRoutes, ...codeqlRemediationRoutes, ...cobolworkRemediationRoutes, ...projectsViewRoutes, ...fleetOverviewRoutes, ...settingsRoutes, ...agentSurfaceRoutes, ...perfRoutes, ...scannerRoutes, ...scanPathRoutes, ...turnsRoutes, ...commentRoutes, ...learningRoutes, ...featuresRoutes, ...overwatchLayerRoutes, ...docsiteRoutes, ...launchlistRoutes, ...offboxRoutes, ...rollupsRoutes, ...jobRoutes, ...panelProcessRoutes, ...panelStateRoutes, ...issuesRoutes, ...dailyRoutes, ...renovateRoutes, ...paletteRoutes, ...feedRoutes, ...journeyRoutes, ...graphRoutes, ...pullRequestRoutes, ...inboxRoutes, ...runnerRoutes]; +const MODULAR_ROUTES = [...secretsRoutes, ...leaksCheckRoutes, ...leaksVerifyRoutes, ...postureRoutes, ...reportRoutes, ...remediationRoutes, ...a11yRoutes, ...profileRoutes, ...configEditRoutes, ...scanConfigRoutes, ...ingestRoutes, ...issueDetailRoutes, ...craRoutes, ...determinationRoutes, ...annotationRoutes, ...oversightRoutes, ...correlationRoutes, ...remediationPolicyRoutes, ...llmRuntimeRoutes, ...hostRoutes, ...verdictRoutes, ...packageRoutes, ...updateRoutes, ...codeqlRemediationRoutes, ...cobolworkRemediationRoutes, ...projectsViewRoutes, ...fleetOverviewRoutes, ...settingsRoutes, ...agentSurfaceRoutes, ...perfRoutes, ...scannerRoutes, ...scanPathRoutes, ...turnsRoutes, ...commentRoutes, ...learningRoutes, ...featuresRoutes, ...overwatchLayerRoutes, ...docsiteRoutes, ...launchlistRoutes, ...dfdRoutes, ...offboxRoutes, ...rollupsRoutes, ...jobRoutes, ...panelProcessRoutes, ...panelStateRoutes, ...issuesRoutes, ...dailyRoutes, ...renovateRoutes, ...paletteRoutes, ...feedRoutes, ...journeyRoutes, ...graphRoutes, ...pullRequestRoutes, ...inboxRoutes, ...runnerRoutes]; // Server copy of the client's VALID_VIEWS — panel-view-paths.test.mjs pins the two sets equal. // Includes RETIRED view names still served as aliases ('leaks' -> the secrets view). The server's diff --git a/admin/test/dfd-route.test.mjs b/admin/test/dfd-route.test.mjs new file mode 100644 index 000000000..22ee1486e --- /dev/null +++ b/admin/test/dfd-route.test.mjs @@ -0,0 +1,133 @@ +// admin/routes/dfd.mjs over a really-spawned panel: the page and the document sit behind the login +// gate, and a signed-in request gets the page under its own hash-pinned policy. CW_AUTH_STORE / +// CW_REGISTRY / CW_DFD_FILE point at temp files and the synthetic fixture. + +import { test, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { createServer } from 'node:net'; +import { request } from 'node:http'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SERVE = join(HERE, '..', 'serve.mjs'); +const FIXTURE = join(HERE, '..', '..', 'lib', 'test', 'fixtures', 'dfd-synthetic.json'); +const TMP = mkdtempSync(join(tmpdir(), 'cw-dfd-route-')); + +let port, localPort, child, csrf, cookie = ''; + +const freePort = () => new Promise((res, rej) => { + const s = createServer(); + s.on('error', rej); + s.listen(0, '127.0.0.1', () => { const p = s.address().port; s.close(() => res(p)); }); +}); + +function hit(path, { method = 'GET', headers = {}, body = null, operator = true, auth = false } = {}) { + return new Promise((resolve, reject) => { + const h = { host: 'localhost', ...headers }; + if (auth && cookie) h.cookie = cookie; + if (body != null) { h['content-type'] = 'application/json'; h['content-length'] = Buffer.byteLength(body); } + const req = request({ host: '127.0.0.1', port: operator ? localPort : port, path, method, headers: h }, (res) => { + let buf = ''; + res.setEncoding('utf8'); + res.on('data', (d) => { buf += d; }); + res.on('end', () => { let json = null; try { json = JSON.parse(buf); } catch { /* html */ } resolve({ status: res.statusCode, headers: res.headers, body: buf, json }); }); + }); + req.on('error', reject); + if (body != null) req.write(body); + req.end(); + }); +} +const post = (path, payload, opts = {}) => + hit(path, { ...opts, method: 'POST', body: JSON.stringify(payload), headers: { 'x-cw-csrf': csrf, ...(opts.headers || {}) } }); + +before(async () => { + writeFileSync(join(TMP, 'projects.json'), JSON.stringify({ + reportsRoot: join(TMP, 'reports'), monitorOutput: 'fixarea', defaultManifest: 'security-baseline', roots: [], + projects: [{ name: 'fixrepo', area: 'fixarea', path: join(TMP, 'src'), manifest: 'security-baseline' }], + areas: [{ slug: 'fixarea', label: 'fixarea', out: 'fixarea', primary: true, members: ['fixrepo'] }], + })); + mkdirSync(join(TMP, 'src'), { recursive: true }); + + let lastErr = ''; + for (let attempt = 1; attempt <= 3; attempt++) { + port = await freePort(); + localPort = await freePort(); + let err = ''; + child = spawn(process.execPath, [SERVE], { + env: { + ...process.env, + CW_AUTH_STORE: join(TMP, 'users.json'), CW_REGISTRY: join(TMP, 'projects.json'), CW_DFD_FILE: FIXTURE, + CW_ADMIN_PORT: String(port), CW_ADMIN_LOCAL_PORT: String(localPort), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stderr.on('data', (d) => { err += String(d); }); + for (let i = 0; i < 100 && !csrf; i++) { + try { const r = await hit('/api/csrf'); if (r.status === 200) { csrf = r.json.token; break; } } catch { /* not up yet */ } + await new Promise((r) => setTimeout(r, 100)); + } + if (csrf) break; + child.kill('SIGKILL'); + lastErr = err; + } + assert.ok(csrf, `panel did not come up${lastErr ? ` — last child stderr:\n${lastErr}` : ''}`); + // a user must exist first: with none, the zero-user posture refuses every remote caller before the gate + const boot = await post('/auth/bootstrap', { email: 'op@example.com', password: 'correct horse battery' }); + assert.equal(boot.status, 200, `bootstrap failed: ${boot.body}`); + const login = await post('/auth/login', { email: 'op@example.com', password: 'correct horse battery' }); + assert.equal(login.status, 200, `login failed: ${login.body}`); + cookie = String(login.headers['set-cookie'] || '').split(';')[0]; + assert.ok(cookie, 'login set no cookie'); +}); + +after(() => { child?.kill('SIGKILL'); rmSync(TMP, { recursive: true, force: true }); }); + +describe('dfd route — gated', () => { + test('without a session the published port answers neither the page nor the document', async () => { + // arrange + const paths = ['/dfd/', '/api/dfd']; + + // act + const rs = await Promise.all(paths.map((p) => hit(p, { operator: false }))); + + // assert + for (const r of rs) { + assert.notEqual(r.status, 200); + assert.doesNotMatch(r.body, / { + test('GET /dfd/ serves the diagram under the page\'s own hash-pinned CSP', async () => { + // arrange + const path = '/dfd/?orientation=TB'; + + // act + const r = await hit(path, { operator: false, auth: true }); + + // assert + assert.equal(r.status, 200, r.body.slice(0, 200)); + assert.match(r.body, / { + // arrange + const path = '/api/dfd?prefix=val&hops=0'; + + // act + const r = await hit(path, { operator: false, auth: true }); + + // assert + assert.equal(r.status, 200, r.body.slice(0, 200)); + assert.equal(r.json.kind, 'dfd'); + assert.deepEqual(r.json.nodes.map((nd) => nd.id), ['validate']); + }); +}); diff --git a/admin/test/dfd-routes-entry.test.mjs b/admin/test/dfd-routes-entry.test.mjs new file mode 100644 index 000000000..b2bd616db --- /dev/null +++ b/admin/test/dfd-routes-entry.test.mjs @@ -0,0 +1,175 @@ +// admin/routes/dfd.mjs through its handlers: the source the environment names, the query as the +// command's flags, and the status each refusal becomes. The login gate is tested over a spawned +// panel in dfd-route.test.mjs. +import { test, describe, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { routes, argvFor, load } from '../routes/dfd.mjs'; +import { makeNode, makeEdge, nodeId } from '../../flow/graph.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FIXTURE = join(HERE, '..', '..', 'lib', 'test', 'fixtures', 'dfd-synthetic.json'); +const TMP = mkdtempSync(join(tmpdir(), 'cw-dfd-entry-')); +after(() => rmSync(TMP, { recursive: true, force: true })); + +const handler = (method, path) => routes.find((r) => r.method === method && r.path === path).handle; + +function call(path, url, env) { + const saved = { ...process.env }; + for (const k of ['CW_DFD_FILE', 'CW_HARNESS_STORE']) delete process.env[k]; + Object.assign(process.env, env); + let out = null; + try { + handler('GET', path)({ req: { url }, send: (status, body, ct, headers) => { out = { status, body, ct, headers }; } }); + } finally { + for (const k of ['CW_DFD_FILE', 'CW_HARNESS_STORE']) delete process.env[k]; + Object.assign(process.env, { ...saved }); + } + assert.ok(out, 'the handler answered nothing'); + return out; +} + +describe('dfd routes — declaration', () => { + test('the page and the document are declared once each, the page with and without its slash', () => { + // arrange + const want = ['GET /dfd/', 'GET /dfd', 'GET /api/dfd']; + + // act + const got = routes.map((r) => `${r.method} ${r.path}`); + + // assert + assert.deepEqual(got, want); + assert.equal(handler('GET', '/dfd'), handler('GET', '/dfd/')); + }); +}); + +describe('dfd routes — the query is the command line', () => { + test('a dfd file named by CW_DFD_FILE is the source, and each known query key becomes its flag', () => { + // arrange + const url = '/dfd/?prefix=api&focus=x&hops=2&direction=out&collapse=g,h&collapse-all-groups-over=3&orientation=TB&max=9&tests=1&imports=true&title=nope&from=/etc/passwd'; + + // act + const argv = argvFor(url, { CW_DFD_FILE: '/data/model.json' }); + + // assert + assert.deepEqual(argv, ['--from', '/data/model.json', '--prefix', 'api', '--focus', 'x', '--hops', '2', '--direction', 'out', + '--collapse', 'g,h', '--collapse-all-groups-over', '3', '--orientation', 'TB', '--max', '9', '--tests', '--imports']); + }); + + test('without the drawing keys only the filter reaches the command', () => { + // arrange + const url = '/api/dfd?prefix=api&collapse=nope&orientation=XX&max=1'; + + // act + const argv = argvFor(url, {}, { drawing: false }); + + // assert + assert.deepEqual(argv, ['--graph', '--prefix', 'api']); + }); + + test('with no dfd file the source is the flow store, and a flag-shaped value stays a value', () => { + // arrange + const url = '/api/dfd?prefix=--graph&tests=0'; + + // act + const argv = argvFor(url, {}); + const got = load(url, { CW_DFD_FILE: FIXTURE }); + + // assert + assert.deepEqual(argv, ['--graph', '--prefix', '--graph']); + assert.equal(got.status, 422); + assert.match(got.error, /the filter matched no node \(prefix "--graph"/); + }); +}); + +describe('GET /dfd/ — the page', () => { + test('serves the diagram under the page\'s own policy, with served fonts', () => { + // arrange + const env = { CW_DFD_FILE: FIXTURE }; + + // act + const r = call('/dfd/', '/dfd/?orientation=LR&collapse=platform', env); + + // assert + assert.equal(r.status, 200); + assert.equal(r.ct, 'text/html; charset=utf-8'); + assert.match(r.body, / { + // arrange + const store = join(TMP, 'store'); + mkdirSync(store, { recursive: true }); + writeFileSync(join(store, 'flow.json'), JSON.stringify({ v: 1, generatedAt: '2026-11-12T00:00:00.000Z', + nodes: [makeNode('module', 'bin/a.mjs', { analysis: 'analysed' }), makeNode('store', 'reports/x.json')], + edges: [makeEdge(nodeId('module', 'bin/a.mjs'), nodeId('store', 'reports/x.json'), 'writes', { evidence: ['bin/a.mjs:1 writeFileSync'] })] })); + + // act + const r = call('/dfd/', '/dfd/', { CW_HARNESS_STORE: store }); + + // assert + assert.equal(r.status, 200, r.body.slice(0, 300)); + assert.match(r.body, /Producer: flow-graph<\/code>/); + }); + + test('each refusal is a status and an escaped page, never a diagram', () => { + // arrange + const bad = join(TMP, 'bad.json'); + writeFileSync(bad, JSON.stringify({ v: 1, kind: 'dfd', nodes: [], flows: [{ from: 'a', to: 'b', mode: 'sync' }] })); + + // act + const usage = call('/dfd/', '/dfd/?direction=sideways&prefix=a', { CW_DFD_FILE: FIXTURE }); + const absent = call('/dfd/', '/dfd/', { CW_HARNESS_STORE: join(TMP, 'empty') }); + const invalid = call('/dfd/', '/dfd/', { CW_DFD_FILE: bad }); + const nomatch = call('/dfd/', '/dfd/?prefix=