diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml new file mode 100644 index 0000000..a91bece --- /dev/null +++ b/.github/workflows/links.yml @@ -0,0 +1,25 @@ +name: Check external links + +# Weekly, not on pull requests, so a flaky third-party site never blocks a merge. +# Settings (exclusions, accepted codes, user agent, retries) are in lychee.toml. +on: + schedule: + - cron: '17 6 * * 1' # Mondays 06:17 UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + lychee: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check links + uses: lycheeverse/lychee-action@v2 + with: + lycheeVersion: v0.24.2 + args: --config lychee.toml --no-progress 'docs/**/*.md' 'src/**/*.tsx' + # The report goes to the job summary; the job fails if any link is broken. + jobSummary: true + fail: true diff --git a/.github/workflows/snippets.yml b/.github/workflows/snippets.yml index 010835b..6d6fe6f 100644 --- a/.github/workflows/snippets.yml +++ b/.github/workflows/snippets.yml @@ -29,3 +29,12 @@ jobs: { echo '```'; cat snippets.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" tail -2 snippets.txt exit $status + - name: Check bridge configs + if: success() || failure() + run: | + set +e + npm run --silent check:bridges -- ${{ vars.SNIPPETS_STRICT == '1' && '--strict' || '' }} > bridges.txt + status=$? + { echo '```'; cat bridges.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + tail -2 bridges.txt + exit $status diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 0000000..08b8c8a --- /dev/null +++ b/lychee.toml @@ -0,0 +1,37 @@ +# lychee config for the weekly external link check (.github/workflows/links.yml). +# Internal links and anchors are checked by the Docusaurus build (onBrokenLinks: throw), so only +# http(s) links are checked here. +scheme = ["https", "http"] +# Resolve root-relative links (/docs/...) to the site itself, which is excluded below. +base_url = "https://matrixdocs.github.io/" + +# Links inside code blocks are examples (http://synapse:8008, https://matrix.example.org/...), not links. +include_verbatim = false + +# Rate limiting means the page exists. +accept = ["200..=299", "429"] + +# crates.io and gitlab.gnome.org reject requests without a browser user agent. +user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0" + +max_retries = 3 +retry_wait_time = 5 +timeout = 30 +max_concurrency = 16 +host_concurrency = 4 + +exclude_all_private = true + +exclude = [ + # This site: internal links are the build's job, and new pages aren't deployed yet. + '^https://matrixdocs\.github\.io/', + # Placeholder hosts used in the docs (RFC 2606/6761 names and docs-only subdomains of them). + '^https?://([a-z0-9-]+\.)*example(\.(com|org|net))?([:/?#]|$)', + '^https?://(localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1?\])([:/?#]|$)', + # Single-label hosts: Docker service names such as http://synapse:8008 or http://mautrix-telegram:29317. + '^https?://[a-z0-9-]+([:/?#]|$)', + # Templated URLs such as https:///... or https://{domain}/... + '^https?://[^/]*[<>{}$]', + # matrix.to links are resolved in the browser from the fragment; the server only ever serves the app. + '^https://matrix\.to/#', +] diff --git a/package.json b/package.json index 0f85e1a..6661217 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "write-heading-ids": "docusaurus write-heading-ids", "typecheck": "tsc", "check": "MATRIX_SPEC=${MATRIX_SPEC:-.upstream/matrix-spec} SYNAPSE_SRC=${SYNAPSE_SRC:-.upstream/synapse} node scripts/check-snippets.mjs", + "check:bridges": "MATRIX_SPEC=${MATRIX_SPEC:-.upstream/matrix-spec} BRIDGES_SRC=${BRIDGES_SRC:-.upstream/bridges} node scripts/check-bridge-configs.mjs", "fetch-upstream": "scripts/fetch-upstream.sh" }, "dependencies": { diff --git a/scripts/check-bridge-configs.mjs b/scripts/check-bridge-configs.mjs new file mode 100644 index 0000000..5336626 --- /dev/null +++ b/scripts/check-bridge-configs.mjs @@ -0,0 +1,255 @@ +#!/usr/bin/env node +// Checks bridge config blocks in the docs against each bridge's example config at the tag pinned in +// scripts/upstream-versions.json (fetched by scripts/fetch-upstream.sh into BRIDGES_SRC): +// bridge - every key path in a bridge config.yaml block exists in that bridge's example config +// (keys that are only commented out in the example count as existing) +// registration - every key in an appservice registration block is in the spec's registration schema +// (MATRIX_SPEC/data/api/application-service/definitions/registration.yaml) +// The bridge comes from a `mautrix/` or ` ` comment in the block, else from the page. +// bridgev2 bridges: `network:` is checked against /pkg/connector/example-config.yaml and the other +// sections against mautrix/go bridgev2/matrix/mxmain/example-config.yaml. +// Opt out of one fenced block with a line `{/* snippet:skip: reason */}` right before it. +// Usage: BRIDGES_SRC=... MATRIX_SPEC=... [DOCS_DIR=...] node scripts/check-bridge-configs.mjs [--json] [--strict] +import fs from 'node:fs'; +import path from 'node:path'; +import yaml from 'js-yaml'; + +const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const SRC = process.env.BRIDGES_SRC; +const SPEC = process.env.MATRIX_SPEC; +if (!SRC || !SPEC) { + console.error('set BRIDGES_SRC and MATRIX_SPEC to the pinned upstream files (run scripts/fetch-upstream.sh)'); + process.exit(2); +} +const args = new Set(process.argv.slice(2)); +const PINS = JSON.parse(fs.readFileSync(path.join(ROOT, 'scripts/upstream-versions.json'), 'utf8')).bridges; + +// Bridges whose config is mautrix/go bridgev2 (shared sections) plus their own `network:` section. +const BRIDGEV2 = new Set(['mautrix-whatsapp', 'mautrix-signal', 'mautrix-slack', 'mautrix-telegram', 'mautrix-meta', 'mautrix-gmessages']); +// Default bridge per page when the block itself doesn't say. 'bridgev2' = shared sections only. +const PAGES = { + 'bridges/discord.md': 'mautrix-discord', + 'bridges/telegram.md': 'mautrix-telegram', + 'bridges/whatsapp.md': 'mautrix-whatsapp', + 'bridges/signal.md': 'mautrix-signal', + 'bridges/slack.md': 'mautrix-slack', + 'bridges/overview.md': 'bridgev2', + 'bridges/irc.md': null, // Heisenbridge has no config file; only its registration is checked + 'deployment/docker.md': 'mautrix-telegram', + 'bots/hookshot.md': 'matrix-hookshot', +}; + +// ---------- ground truth ---------- +const GT = {}; +for (const [name, pin] of Object.entries(PINS)) { + const file = path.join(SRC, name, pin.config); + if (!fs.existsSync(file)) { console.error(`missing ${file}; run scripts/fetch-upstream.sh`); process.exit(2); } + const text = fs.readFileSync(file, 'utf8'); + GT[name] = {tree: yaml.load(text), loose: loosePaths(text, true).paths, label: `${pin.repo} ${pin.tag} ${pin.config}`}; +} + +// Key paths by indentation, including keys that are only commented out (`#key:` / ` # key:`). +// List markers are dropped, so `a:\n - b: 1` gives a.b. Returns paths and the line of each path. +function loosePaths(text, withComments) { + const paths = new Set(), lines = new Map(), stack = []; + text.split('\n').forEach((raw, i) => { + let l = raw; + const c = l.match(/^(\s*)#(.*)$/); + if (c) { + if (!withComments) return; + let rest = c[2]; + if (!c[1] && /^ \S/.test(rest)) rest = rest.slice(1); // "# key:" at column 0 + else if (c[1] && /^ /.test(rest)) rest = rest.slice(1); // " # key:" indented + l = c[1] + rest; + } + const m = l.match(/^(\s*)((?:-\s+)*)(["']?)([A-Za-z_][\w.-]*|[^\s"':#][^"':]*)\3\s*:(\s|$)/); + if (!m || /^\s*#/.test(l)) return; + const indent = m[1].length + m[2].length; + while (stack.length && stack[stack.length - 1].indent >= indent) stack.pop(); + const p = [...stack.map((s) => s.key), m[4]].join('\u0000'); + paths.add(p); + if (!lines.has(p)) lines.set(p, i); + stack.push({indent, key: m[4]}); + }); + return {paths, lines}; +} + +const reg = yaml.load(fs.readFileSync(path.join(SPEC, 'data/api/application-service/definitions/registration.yaml'), 'utf8')); +const nsItem = yaml.load(fs.readFileSync(path.join(SPEC, 'data/api/application-service/definitions/namespace_list.yaml'), 'utf8')); +const REG = { + top: new Set(Object.keys(reg.properties)), + ns: new Set(Object.keys(reg.properties.namespaces.properties)), + item: new Set(Object.keys(nsItem.items.properties)), +}; + +// ---------- matching ---------- +const isMap = (v) => v && typeof v === 'object' && !Array.isArray(v); +// A map whose keys are user data (server names, user IDs, "*") rather than settings. +const freeForm = (v) => isMap(v) && (!Object.keys(v).length || Object.keys(v).some((k) => /[.@*:/]/.test(k))); +function mergeItems(list) { + const out = {}; + for (const it of list) if (isMap(it)) for (const [k, v] of Object.entries(it)) if (!(k in out) || isMap(v)) out[k] = v; + return out; +} + +// Walk the doc block; ex is the example node (undefined when only the commented-out example covers it). +function checkNode(node, ex, gt, keyPath, out) { + if (Array.isArray(node)) { + const exItem = Array.isArray(ex) ? mergeItems(ex) : undefined; + for (const it of node) checkNode(it, exItem, gt, keyPath, out); + return; + } + if (!isMap(node)) return; + if (freeForm(ex)) return; + for (const [k, v] of Object.entries(node)) { + const p = [...keyPath, k]; + if (isMap(ex) && k in ex) { + const e = ex[k]; + if ((isMap(v) || Array.isArray(v)) && e !== null && typeof e !== 'object' && !gt.loose.has([...p, Object.keys(v)[0] ?? ''].join('\u0000'))) { + out.push({path: p, msg: `is a ${typeof e} in the example config, not a ${Array.isArray(v) ? 'list' : 'map'}`}); + continue; + } + checkNode(v, e, gt, p, out); + } else if (gt.loose.has(p.join('\u0000'))) { + checkNode(v, undefined, gt, p, out); + } else { + out.push({path: p, msg: 'is not in'}); + } + } +} + +function checkRegistration(doc) { + const out = []; + for (const [k, v] of Object.entries(doc)) { + if (k.includes('.')) continue; // namespaced extensions such as org.matrix.msc3202 + if (!REG.top.has(k)) { out.push({path: [k]}); continue; } + if (k === 'namespaces' && isMap(v)) { + for (const [n, list] of Object.entries(v)) { + if (!REG.ns.has(n)) { out.push({path: [k, n]}); continue; } + if (Array.isArray(list)) for (const it of list) if (isMap(it)) for (const f of Object.keys(it)) if (!REG.item.has(f)) out.push({path: [k, n, f]}); + } + } + } + return out; +} + +// Which bridge a block belongs to: a mention in the block's comments wins over the page default. +function identify(body, title, page) { + const named = []; + for (const m of body.matchAll(/^\s*#.*$/gm)) { + for (const r of m[0].matchAll(/\b(mautrix\/[a-z]+|matrix-hookshot|matrix-appservice-irc)\b/g)) { + const n = r[1].replace('/', '-'); + if (n !== 'mautrix-go' && PINS[n]) named.push(n); + } + } + const t = title.match(/\b(mautrix-[a-z]+|matrix-hookshot|matrix-appservice-irc)\b/); + if (t && PINS[t[1]]) named.push(t[1]); + if (named.length) return named[0]; + if (/mautrix\/go\b|bridgev2/.test(body + title)) return PAGES[page] && BRIDGEV2.has(PAGES[page]) ? PAGES[page] : 'bridgev2'; + return PAGES[page]; +} + +// Split a doc into [section, example] pairs for the bridge. +function sections(bridge, doc) { + if (bridge === 'bridgev2' || BRIDGEV2.has(bridge)) { + const out = []; + for (const [k, v] of Object.entries(doc)) { + if (k === 'network') { if (bridge !== 'bridgev2') out.push({gt: bridge, node: v, prefix: ['network'], ex: GT[bridge].tree}); } + else out.push({gt: 'mautrix-go', node: {[k]: v}, prefix: [], ex: GT['mautrix-go'].tree}); + } + return out; + } + return [{gt: bridge, node: doc, prefix: [], ex: GT[bridge].tree}]; +} + +// ---------- scan ---------- +function walk(dir, out = []) { + for (const e of fs.readdirSync(dir, {withFileTypes: true})) { + const p = path.join(dir, e.name); + if (e.isDirectory()) walk(p, out); + else if (/\.mdx?$/.test(e.name)) out.push(p); + } + return out; +} + +const DOCS = process.env.DOCS_DIR ? path.resolve(process.env.DOCS_DIR) : path.join(ROOT, 'docs'); +const report = []; +const totals = {bridge: 0, registration: 0}; +let blocks = 0; + +for (const file of walk(DOCS).sort()) { + const rel = path.relative(DOCS, file).split(path.sep).join('/'); + const lines = fs.readFileSync(file, 'utf8').split('\n'); + const findings = []; + const add = (kind, line, msg) => { findings.push({kind, line, msg}); totals[kind]++; }; + let inFence = false, lang = '', title = '', start = 0, buf = [], skip = false, skipNext = false; + + for (let i = 0; i < lines.length; i++) { + const l = lines[i]; + const fence = l.match(/^\s*(```+|~~~+)\s*([\w+-]*)(.*)$/); + if (!inFence && /\{\/\*\s*snippet:skip/.test(l)) { skipNext = true; continue; } + if (fence && !inFence) { + inFence = true; lang = fence[2]; title = (fence[3].match(/title="([^"]*)"/) || [, ''])[1]; + start = i + 1; buf = []; skip = skipNext; skipNext = false; continue; + } + if (inFence && /^\s*(```+|~~~+)\s*$/.test(l)) { + inFence = false; + if (skip || !/^ya?ml$/.test(lang)) continue; + const body = buf.join('\n'); + let doc; + try { doc = yaml.load(body); } catch { continue; } + if (!isMap(doc) || 'services' in doc || /homeserver\.yaml|compose|prometheus/i.test(title)) continue; + const where = loosePaths(body, false).lines; + const lineOf = (p) => start + 1 + (where.get(p.join('\u0000')) ?? 0); + const keys = Object.keys(doc); + + // Appservice registration files. + if (/registration/i.test(title) || ['id', 'as_token', 'hs_token'].every((k) => k in doc)) { + blocks++; + for (const f of checkRegistration(doc)) add('registration', lineOf(f.path), `registration key \`${f.path.join('.')}\` is not in the spec v1.19 registration schema`); + continue; + } + + const bridge = identify(body, title, rel); + if (!bridge) continue; + const looksLikeConfig = /config\.ya?ml/i.test(title) || /example-config|config\.sample/.test(body); + if (!looksLikeConfig) { + // Untitled block: only check it if most top-level keys are the bridge's own sections. + const top = new Set(sections(bridge, doc).flatMap((s) => Object.keys(s.ex)).concat(BRIDGEV2.has(bridge) ? ['network'] : [])); + if (keys.filter((k) => top.has(k)).length / keys.length < 0.5) continue; + } + blocks++; + for (const s of sections(bridge, doc)) { + const out = []; + const gt = GT[s.gt]; + if (s.prefix.length) { + // network: section. Its keys are the connector config's top level. + checkNode(s.node, s.ex, {loose: gt.loose}, [], out); + for (const f of out) f.path = [...s.prefix, ...f.path]; + } else checkNode(s.node, s.ex, gt, [], out); + for (const f of out) { + const shown = s.prefix.length ? f.path.slice(1) : f.path; + const msg = f.msg === 'is not in' ? `is not in ${gt.label}` : f.msg + ` (${gt.label})`; + add('bridge', lineOf(f.path), `${bridge} key \`${f.path.join('.')}\` ${msg}${s.prefix.length ? ` (as \`${shown.join('.')}\`)` : ''}`); + } + } + continue; + } + if (inFence) buf.push(l); + } + if (findings.length) report.push({file: rel, findings}); +} + +if (args.has('--json')) { + console.log(JSON.stringify({totals, blocks, report}, null, 2)); +} else { + for (const r of report) { + console.log(`\n${r.file}`); + for (const f of r.findings) console.log(` L${f.line} [${f.kind}] ${f.msg}`); + } + const n = Object.values(totals).reduce((a, b) => a + b, 0); + console.log(`\n${n} findings in ${blocks} bridge config blocks: ${Object.entries(totals).map(([k, v]) => `${k} ${v}`).join(', ')}`); + console.log(`pages with findings: ${report.length}`); +} +const n = Object.values(totals).reduce((a, b) => a + b, 0); +process.exit(args.has('--strict') && n ? 1 : 0); diff --git a/scripts/fetch-upstream.sh b/scripts/fetch-upstream.sh index 639193e..158e748 100755 --- a/scripts/fetch-upstream.sh +++ b/scripts/fetch-upstream.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash -# Shallow-clone the pinned upstream sources the gate checks against into ${1:-.upstream}. -# To bump a pin: edit scripts/upstream-versions.json, rerun this, run `npm run check`, fix, merge. +# Shallow-clone the pinned upstream sources the gate checks against into ${1:-.upstream}, and fetch each +# bridge's example config at its pinned tag into ${1:-.upstream}/bridges//. +# To bump a pin: edit scripts/upstream-versions.json, rerun this, run `npm run check` and +# `npm run check:bridges`, fix, merge. Heisenbridge has no config file (settings live on the homeserver), +# so it has no entry; its registration file is checked against the spec instead. set -euo pipefail DEST=${1:-.upstream} mkdir -p "$DEST" @@ -10,4 +13,10 @@ for name in matrix-spec synapse; do rm -rf "$DEST/$name" git clone -q --depth 1 --branch "$tag" "$repo" "$DEST/$name" done -echo "MATRIX_SPEC=$DEST/matrix-spec SYNAPSE_SRC=$DEST/synapse" +rm -rf "$DEST/bridges" +node -e 'for (const [n, b] of Object.entries(require("./scripts/upstream-versions.json").bridges)) console.log(n, b.repo, b.tag, b.config)' | + while read -r name repo tag file; do + mkdir -p "$DEST/bridges/$name/$(dirname "$file")" + curl -fsSL --retry 3 -o "$DEST/bridges/$name/$file" "https://raw.githubusercontent.com/$repo/$tag/$file" + done +echo "MATRIX_SPEC=$DEST/matrix-spec SYNAPSE_SRC=$DEST/synapse BRIDGES_SRC=$DEST/bridges" diff --git a/scripts/upstream-versions.json b/scripts/upstream-versions.json index e98077e..5149e06 100644 --- a/scripts/upstream-versions.json +++ b/scripts/upstream-versions.json @@ -1,4 +1,16 @@ { "matrix-spec": {"repo": "https://github.com/matrix-org/matrix-spec.git", "tag": "v1.19"}, - "synapse": {"repo": "https://github.com/element-hq/synapse.git", "tag": "v1.163.0"} + "synapse": {"repo": "https://github.com/element-hq/synapse.git", "tag": "v1.163.0"}, + "bridges": { + "mautrix-go": {"repo": "mautrix/go", "tag": "v0.32.0", "config": "bridgev2/matrix/mxmain/example-config.yaml"}, + "mautrix-discord": {"repo": "mautrix/discord", "tag": "v0.7.7", "config": "example-config.yaml"}, + "mautrix-whatsapp": {"repo": "mautrix/whatsapp", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"}, + "mautrix-signal": {"repo": "mautrix/signal", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"}, + "mautrix-slack": {"repo": "mautrix/slack", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"}, + "mautrix-telegram": {"repo": "mautrix/telegram", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"}, + "mautrix-meta": {"repo": "mautrix/meta", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"}, + "mautrix-gmessages": {"repo": "mautrix/gmessages", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"}, + "matrix-appservice-irc": {"repo": "matrix-org/matrix-appservice-irc", "tag": "4.0.0", "config": "config.sample.yaml"}, + "matrix-hookshot": {"repo": "matrix-org/matrix-hookshot", "tag": "7.5.0", "config": "config.sample.yml"} + } }