Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/workflows/links.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Check external links

# Weekly, not on pull requests, so a flaky third-party site never blocks a merge.
# Settings (exclusions, accepted codes, user agent, retries) are in lychee.toml.
on:
schedule:
- cron: '17 6 * * 1' # Mondays 06:17 UTC
workflow_dispatch:

permissions:
contents: read

jobs:
lychee:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Check links
uses: lycheeverse/lychee-action@v2
with:
lycheeVersion: v0.24.2
args: --config lychee.toml --no-progress 'docs/**/*.md' 'src/**/*.tsx'
# The report goes to the job summary; the job fails if any link is broken.
jobSummary: true
fail: true
9 changes: 9 additions & 0 deletions .github/workflows/snippets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,12 @@ jobs:
{ echo '```'; cat snippets.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
tail -2 snippets.txt
exit $status
- name: Check bridge configs
if: success() || failure()
run: |
set +e
npm run --silent check:bridges -- ${{ vars.SNIPPETS_STRICT == '1' && '--strict' || '' }} > bridges.txt
status=$?
{ echo '```'; cat bridges.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
tail -2 bridges.txt
exit $status
37 changes: 37 additions & 0 deletions lychee.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# lychee config for the weekly external link check (.github/workflows/links.yml).
# Internal links and anchors are checked by the Docusaurus build (onBrokenLinks: throw), so only
# http(s) links are checked here.
scheme = ["https", "http"]
# Resolve root-relative links (/docs/...) to the site itself, which is excluded below.
base_url = "https://matrixdocs.github.io/"

# Links inside code blocks are examples (http://synapse:8008, https://matrix.example.org/...), not links.
include_verbatim = false

# Rate limiting means the page exists.
accept = ["200..=299", "429"]

# crates.io and gitlab.gnome.org reject requests without a browser user agent.
user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0"

max_retries = 3
retry_wait_time = 5
timeout = 30
max_concurrency = 16
host_concurrency = 4

exclude_all_private = true

exclude = [
# This site: internal links are the build's job, and new pages aren't deployed yet.
'^https://matrixdocs\.github\.io/',
# Placeholder hosts used in the docs (RFC 2606/6761 names and docs-only subdomains of them).
'^https?://([a-z0-9-]+\.)*example(\.(com|org|net))?([:/?#]|$)',
'^https?://(localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1?\])([:/?#]|$)',
# Single-label hosts: Docker service names such as http://synapse:8008 or http://mautrix-telegram:29317.
'^https?://[a-z0-9-]+([:/?#]|$)',
# Templated URLs such as https://<server>/... or https://{domain}/...
'^https?://[^/]*[<>{}$]',
# matrix.to links are resolved in the browser from the fragment; the server only ever serves the app.
'^https://matrix\.to/#',
]
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"write-heading-ids": "docusaurus write-heading-ids",
"typecheck": "tsc",
"check": "MATRIX_SPEC=${MATRIX_SPEC:-.upstream/matrix-spec} SYNAPSE_SRC=${SYNAPSE_SRC:-.upstream/synapse} node scripts/check-snippets.mjs",
"check:bridges": "MATRIX_SPEC=${MATRIX_SPEC:-.upstream/matrix-spec} BRIDGES_SRC=${BRIDGES_SRC:-.upstream/bridges} node scripts/check-bridge-configs.mjs",
"fetch-upstream": "scripts/fetch-upstream.sh"
},
"dependencies": {
Expand Down
255 changes: 255 additions & 0 deletions scripts/check-bridge-configs.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
#!/usr/bin/env node
// Checks bridge config blocks in the docs against each bridge's example config at the tag pinned in
// scripts/upstream-versions.json (fetched by scripts/fetch-upstream.sh into BRIDGES_SRC):
// bridge - every key path in a bridge config.yaml block exists in that bridge's example config
// (keys that are only commented out in the example count as existing)
// registration - every key in an appservice registration block is in the spec's registration schema
// (MATRIX_SPEC/data/api/application-service/definitions/registration.yaml)
// The bridge comes from a `mautrix/<name>` or `<project> <version>` comment in the block, else from the page.
// bridgev2 bridges: `network:` is checked against <bridge>/pkg/connector/example-config.yaml and the other
// sections against mautrix/go bridgev2/matrix/mxmain/example-config.yaml.
// Opt out of one fenced block with a line `{/* snippet:skip: reason */}` right before it.
// Usage: BRIDGES_SRC=... MATRIX_SPEC=... [DOCS_DIR=...] node scripts/check-bridge-configs.mjs [--json] [--strict]
import fs from 'node:fs';
import path from 'node:path';
import yaml from 'js-yaml';

const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..');
const SRC = process.env.BRIDGES_SRC;
const SPEC = process.env.MATRIX_SPEC;
if (!SRC || !SPEC) {
console.error('set BRIDGES_SRC and MATRIX_SPEC to the pinned upstream files (run scripts/fetch-upstream.sh)');
process.exit(2);
}
const args = new Set(process.argv.slice(2));
const PINS = JSON.parse(fs.readFileSync(path.join(ROOT, 'scripts/upstream-versions.json'), 'utf8')).bridges;

// Bridges whose config is mautrix/go bridgev2 (shared sections) plus their own `network:` section.
const BRIDGEV2 = new Set(['mautrix-whatsapp', 'mautrix-signal', 'mautrix-slack', 'mautrix-telegram', 'mautrix-meta', 'mautrix-gmessages']);
// Default bridge per page when the block itself doesn't say. 'bridgev2' = shared sections only.
const PAGES = {
'bridges/discord.md': 'mautrix-discord',
'bridges/telegram.md': 'mautrix-telegram',
'bridges/whatsapp.md': 'mautrix-whatsapp',
'bridges/signal.md': 'mautrix-signal',
'bridges/slack.md': 'mautrix-slack',
'bridges/overview.md': 'bridgev2',
'bridges/irc.md': null, // Heisenbridge has no config file; only its registration is checked
'deployment/docker.md': 'mautrix-telegram',
'bots/hookshot.md': 'matrix-hookshot',
};

// ---------- ground truth ----------
const GT = {};
for (const [name, pin] of Object.entries(PINS)) {
const file = path.join(SRC, name, pin.config);
if (!fs.existsSync(file)) { console.error(`missing ${file}; run scripts/fetch-upstream.sh`); process.exit(2); }
const text = fs.readFileSync(file, 'utf8');
GT[name] = {tree: yaml.load(text), loose: loosePaths(text, true).paths, label: `${pin.repo} ${pin.tag} ${pin.config}`};
}

// Key paths by indentation, including keys that are only commented out (`#key:` / ` # key:`).
// List markers are dropped, so `a:\n - b: 1` gives a.b. Returns paths and the line of each path.
function loosePaths(text, withComments) {
const paths = new Set(), lines = new Map(), stack = [];
text.split('\n').forEach((raw, i) => {
let l = raw;
const c = l.match(/^(\s*)#(.*)$/);
if (c) {
if (!withComments) return;
let rest = c[2];
if (!c[1] && /^ \S/.test(rest)) rest = rest.slice(1); // "# key:" at column 0
else if (c[1] && /^ /.test(rest)) rest = rest.slice(1); // " # key:" indented
l = c[1] + rest;
}
const m = l.match(/^(\s*)((?:-\s+)*)(["']?)([A-Za-z_][\w.-]*|[^\s"':#][^"':]*)\3\s*:(\s|$)/);
if (!m || /^\s*#/.test(l)) return;
const indent = m[1].length + m[2].length;
while (stack.length && stack[stack.length - 1].indent >= indent) stack.pop();
const p = [...stack.map((s) => s.key), m[4]].join('\u0000');
paths.add(p);
if (!lines.has(p)) lines.set(p, i);
stack.push({indent, key: m[4]});
});
return {paths, lines};
}

const reg = yaml.load(fs.readFileSync(path.join(SPEC, 'data/api/application-service/definitions/registration.yaml'), 'utf8'));
const nsItem = yaml.load(fs.readFileSync(path.join(SPEC, 'data/api/application-service/definitions/namespace_list.yaml'), 'utf8'));
const REG = {
top: new Set(Object.keys(reg.properties)),
ns: new Set(Object.keys(reg.properties.namespaces.properties)),
item: new Set(Object.keys(nsItem.items.properties)),
};

// ---------- matching ----------
const isMap = (v) => v && typeof v === 'object' && !Array.isArray(v);
// A map whose keys are user data (server names, user IDs, "*") rather than settings.
const freeForm = (v) => isMap(v) && (!Object.keys(v).length || Object.keys(v).some((k) => /[.@*:/]/.test(k)));
function mergeItems(list) {
const out = {};
for (const it of list) if (isMap(it)) for (const [k, v] of Object.entries(it)) if (!(k in out) || isMap(v)) out[k] = v;
return out;
}

// Walk the doc block; ex is the example node (undefined when only the commented-out example covers it).
function checkNode(node, ex, gt, keyPath, out) {
if (Array.isArray(node)) {
const exItem = Array.isArray(ex) ? mergeItems(ex) : undefined;
for (const it of node) checkNode(it, exItem, gt, keyPath, out);
return;
}
if (!isMap(node)) return;
if (freeForm(ex)) return;
for (const [k, v] of Object.entries(node)) {
const p = [...keyPath, k];
if (isMap(ex) && k in ex) {
const e = ex[k];
if ((isMap(v) || Array.isArray(v)) && e !== null && typeof e !== 'object' && !gt.loose.has([...p, Object.keys(v)[0] ?? ''].join('\u0000'))) {
out.push({path: p, msg: `is a ${typeof e} in the example config, not a ${Array.isArray(v) ? 'list' : 'map'}`});
continue;
}
checkNode(v, e, gt, p, out);
} else if (gt.loose.has(p.join('\u0000'))) {
checkNode(v, undefined, gt, p, out);
} else {
out.push({path: p, msg: 'is not in'});
}
}
}

function checkRegistration(doc) {
const out = [];
for (const [k, v] of Object.entries(doc)) {
if (k.includes('.')) continue; // namespaced extensions such as org.matrix.msc3202
if (!REG.top.has(k)) { out.push({path: [k]}); continue; }
if (k === 'namespaces' && isMap(v)) {
for (const [n, list] of Object.entries(v)) {
if (!REG.ns.has(n)) { out.push({path: [k, n]}); continue; }
if (Array.isArray(list)) for (const it of list) if (isMap(it)) for (const f of Object.keys(it)) if (!REG.item.has(f)) out.push({path: [k, n, f]});
}
}
}
return out;
}

// Which bridge a block belongs to: a mention in the block's comments wins over the page default.
function identify(body, title, page) {
const named = [];
for (const m of body.matchAll(/^\s*#.*$/gm)) {
for (const r of m[0].matchAll(/\b(mautrix\/[a-z]+|matrix-hookshot|matrix-appservice-irc)\b/g)) {
const n = r[1].replace('/', '-');
if (n !== 'mautrix-go' && PINS[n]) named.push(n);
}
}
const t = title.match(/\b(mautrix-[a-z]+|matrix-hookshot|matrix-appservice-irc)\b/);
if (t && PINS[t[1]]) named.push(t[1]);
if (named.length) return named[0];
if (/mautrix\/go\b|bridgev2/.test(body + title)) return PAGES[page] && BRIDGEV2.has(PAGES[page]) ? PAGES[page] : 'bridgev2';
return PAGES[page];
}

// Split a doc into [section, example] pairs for the bridge.
function sections(bridge, doc) {
if (bridge === 'bridgev2' || BRIDGEV2.has(bridge)) {
const out = [];
for (const [k, v] of Object.entries(doc)) {
if (k === 'network') { if (bridge !== 'bridgev2') out.push({gt: bridge, node: v, prefix: ['network'], ex: GT[bridge].tree}); }
else out.push({gt: 'mautrix-go', node: {[k]: v}, prefix: [], ex: GT['mautrix-go'].tree});
}
return out;
}
return [{gt: bridge, node: doc, prefix: [], ex: GT[bridge].tree}];
}

// ---------- scan ----------
function walk(dir, out = []) {
for (const e of fs.readdirSync(dir, {withFileTypes: true})) {
const p = path.join(dir, e.name);
if (e.isDirectory()) walk(p, out);
else if (/\.mdx?$/.test(e.name)) out.push(p);
}
return out;
}

const DOCS = process.env.DOCS_DIR ? path.resolve(process.env.DOCS_DIR) : path.join(ROOT, 'docs');
const report = [];
const totals = {bridge: 0, registration: 0};
let blocks = 0;

for (const file of walk(DOCS).sort()) {
const rel = path.relative(DOCS, file).split(path.sep).join('/');
const lines = fs.readFileSync(file, 'utf8').split('\n');
const findings = [];
const add = (kind, line, msg) => { findings.push({kind, line, msg}); totals[kind]++; };
let inFence = false, lang = '', title = '', start = 0, buf = [], skip = false, skipNext = false;

for (let i = 0; i < lines.length; i++) {
const l = lines[i];
const fence = l.match(/^\s*(```+|~~~+)\s*([\w+-]*)(.*)$/);
if (!inFence && /\{\/\*\s*snippet:skip/.test(l)) { skipNext = true; continue; }
if (fence && !inFence) {
inFence = true; lang = fence[2]; title = (fence[3].match(/title="([^"]*)"/) || [, ''])[1];
start = i + 1; buf = []; skip = skipNext; skipNext = false; continue;
}
if (inFence && /^\s*(```+|~~~+)\s*$/.test(l)) {
inFence = false;
if (skip || !/^ya?ml$/.test(lang)) continue;
const body = buf.join('\n');
let doc;
try { doc = yaml.load(body); } catch { continue; }
if (!isMap(doc) || 'services' in doc || /homeserver\.yaml|compose|prometheus/i.test(title)) continue;
const where = loosePaths(body, false).lines;
const lineOf = (p) => start + 1 + (where.get(p.join('\u0000')) ?? 0);
const keys = Object.keys(doc);

// Appservice registration files.
if (/registration/i.test(title) || ['id', 'as_token', 'hs_token'].every((k) => k in doc)) {
blocks++;
for (const f of checkRegistration(doc)) add('registration', lineOf(f.path), `registration key \`${f.path.join('.')}\` is not in the spec v1.19 registration schema`);
continue;
}

const bridge = identify(body, title, rel);
if (!bridge) continue;
const looksLikeConfig = /config\.ya?ml/i.test(title) || /example-config|config\.sample/.test(body);
if (!looksLikeConfig) {
// Untitled block: only check it if most top-level keys are the bridge's own sections.
const top = new Set(sections(bridge, doc).flatMap((s) => Object.keys(s.ex)).concat(BRIDGEV2.has(bridge) ? ['network'] : []));
if (keys.filter((k) => top.has(k)).length / keys.length < 0.5) continue;
}
blocks++;
for (const s of sections(bridge, doc)) {
const out = [];
const gt = GT[s.gt];
if (s.prefix.length) {
// network: section. Its keys are the connector config's top level.
checkNode(s.node, s.ex, {loose: gt.loose}, [], out);
for (const f of out) f.path = [...s.prefix, ...f.path];
} else checkNode(s.node, s.ex, gt, [], out);
for (const f of out) {
const shown = s.prefix.length ? f.path.slice(1) : f.path;
const msg = f.msg === 'is not in' ? `is not in ${gt.label}` : f.msg + ` (${gt.label})`;
add('bridge', lineOf(f.path), `${bridge} key \`${f.path.join('.')}\` ${msg}${s.prefix.length ? ` (as \`${shown.join('.')}\`)` : ''}`);
}
}
continue;
}
if (inFence) buf.push(l);
}
if (findings.length) report.push({file: rel, findings});
}

if (args.has('--json')) {
console.log(JSON.stringify({totals, blocks, report}, null, 2));
} else {
for (const r of report) {
console.log(`\n${r.file}`);
for (const f of r.findings) console.log(` L${f.line} [${f.kind}] ${f.msg}`);
}
const n = Object.values(totals).reduce((a, b) => a + b, 0);
console.log(`\n${n} findings in ${blocks} bridge config blocks: ${Object.entries(totals).map(([k, v]) => `${k} ${v}`).join(', ')}`);
console.log(`pages with findings: ${report.length}`);
}
const n = Object.values(totals).reduce((a, b) => a + b, 0);
process.exit(args.has('--strict') && n ? 1 : 0);
15 changes: 12 additions & 3 deletions scripts/fetch-upstream.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
#!/usr/bin/env bash
# Shallow-clone the pinned upstream sources the gate checks against into ${1:-.upstream}.
# To bump a pin: edit scripts/upstream-versions.json, rerun this, run `npm run check`, fix, merge.
# Shallow-clone the pinned upstream sources the gate checks against into ${1:-.upstream}, and fetch each
# bridge's example config at its pinned tag into ${1:-.upstream}/bridges/<name>/<path>.
# To bump a pin: edit scripts/upstream-versions.json, rerun this, run `npm run check` and
# `npm run check:bridges`, fix, merge. Heisenbridge has no config file (settings live on the homeserver),
# so it has no entry; its registration file is checked against the spec instead.
set -euo pipefail
DEST=${1:-.upstream}
mkdir -p "$DEST"
Expand All @@ -10,4 +13,10 @@ for name in matrix-spec synapse; do
rm -rf "$DEST/$name"
git clone -q --depth 1 --branch "$tag" "$repo" "$DEST/$name"
done
echo "MATRIX_SPEC=$DEST/matrix-spec SYNAPSE_SRC=$DEST/synapse"
rm -rf "$DEST/bridges"
node -e 'for (const [n, b] of Object.entries(require("./scripts/upstream-versions.json").bridges)) console.log(n, b.repo, b.tag, b.config)' |
while read -r name repo tag file; do
mkdir -p "$DEST/bridges/$name/$(dirname "$file")"
curl -fsSL --retry 3 -o "$DEST/bridges/$name/$file" "https://raw.githubusercontent.com/$repo/$tag/$file"
done
echo "MATRIX_SPEC=$DEST/matrix-spec SYNAPSE_SRC=$DEST/synapse BRIDGES_SRC=$DEST/bridges"
14 changes: 13 additions & 1 deletion scripts/upstream-versions.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,16 @@
{
"matrix-spec": {"repo": "https://github.com/matrix-org/matrix-spec.git", "tag": "v1.19"},
"synapse": {"repo": "https://github.com/element-hq/synapse.git", "tag": "v1.163.0"}
"synapse": {"repo": "https://github.com/element-hq/synapse.git", "tag": "v1.163.0"},
"bridges": {
"mautrix-go": {"repo": "mautrix/go", "tag": "v0.32.0", "config": "bridgev2/matrix/mxmain/example-config.yaml"},
"mautrix-discord": {"repo": "mautrix/discord", "tag": "v0.7.7", "config": "example-config.yaml"},
"mautrix-whatsapp": {"repo": "mautrix/whatsapp", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"},
"mautrix-signal": {"repo": "mautrix/signal", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"},
"mautrix-slack": {"repo": "mautrix/slack", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"},
"mautrix-telegram": {"repo": "mautrix/telegram", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"},
"mautrix-meta": {"repo": "mautrix/meta", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"},
"mautrix-gmessages": {"repo": "mautrix/gmessages", "tag": "v0.2610.0", "config": "pkg/connector/example-config.yaml"},
"matrix-appservice-irc": {"repo": "matrix-org/matrix-appservice-irc", "tag": "4.0.0", "config": "config.sample.yaml"},
"matrix-hookshot": {"repo": "matrix-org/matrix-hookshot", "tag": "7.5.0", "config": "config.sample.yml"}
}
}
Loading