From 391802da82e0f50e86a3ec08ef00a25fe3612b8d Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:53:00 +0800 Subject: [PATCH 1/4] feat(coordination): add a reproducible PostgreSQL and TLS self-hosting example Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../remote-authority-task-service-v0.md | 5 + .../remote-authority-compose/.dockerignore | 5 + .../remote-authority-compose/Dockerfile | 14 ++ .../remote-authority-compose/README.en.md | 58 +++++ .../remote-authority-compose/README.md | 58 +++++ .../remote-authority-compose/build_bundle.py | 102 ++++++++ .../remote-authority-compose/client.sh | 10 + .../remote-authority-compose/compose.yaml | 147 ++++++++++++ .../remote-authority-compose/launch.mjs | 44 ++++ .../remote-authority-compose/provision.mjs | 223 ++++++++++++++++++ .../remote-authority-compose/rehearse.mjs | 166 +++++++++++++ .../remote-authority-compose/start.sh | 15 ++ .../remote-authority-compose/state.sh | 74 ++++++ tests/test_remote_authority_compose_bundle.py | 55 +++++ 14 files changed, 976 insertions(+) create mode 100644 examples/coordination/remote-authority-compose/.dockerignore create mode 100644 examples/coordination/remote-authority-compose/Dockerfile create mode 100644 examples/coordination/remote-authority-compose/README.en.md create mode 100644 examples/coordination/remote-authority-compose/README.md create mode 100644 examples/coordination/remote-authority-compose/build_bundle.py create mode 100755 examples/coordination/remote-authority-compose/client.sh create mode 100644 examples/coordination/remote-authority-compose/compose.yaml create mode 100644 examples/coordination/remote-authority-compose/launch.mjs create mode 100644 examples/coordination/remote-authority-compose/provision.mjs create mode 100644 examples/coordination/remote-authority-compose/rehearse.mjs create mode 100755 examples/coordination/remote-authority-compose/start.sh create mode 100755 examples/coordination/remote-authority-compose/state.sh create mode 100644 tests/test_remote_authority_compose_bundle.py diff --git a/docs/reference/remote-authority-task-service-v0.md b/docs/reference/remote-authority-task-service-v0.md index aa4063fa51..44fcf18f45 100644 --- a/docs/reference/remote-authority-task-service-v0.md +++ b/docs/reference/remote-authority-task-service-v0.md @@ -225,6 +225,11 @@ intent can then query retained historical receipts with that new binding. ## Running the candidate +For a local PostgreSQL 16 server and HTTPS gateway, build the +[self-hosted Compose bundle](../../examples/coordination/remote-authority-compose/README.en.md) +from a pinned commit. The bundle boots an isolated starter Goal with two +independent client credentials. [简体中文](../../examples/coordination/remote-authority-compose/README.md) + Use the repository's qualified Node runtime and `npm ci --ignore-scripts`. Provision an isolated PostgreSQL authority schema/head using the existing provider administration path. The listener never installs schemas or seeds diff --git a/examples/coordination/remote-authority-compose/.dockerignore b/examples/coordination/remote-authority-compose/.dockerignore new file mode 100644 index 0000000000..322bd34d38 --- /dev/null +++ b/examples/coordination/remote-authority-compose/.dockerignore @@ -0,0 +1,5 @@ +* +!loopx-source.tar.gz +!Dockerfile +!provision.mjs +!launch.mjs diff --git a/examples/coordination/remote-authority-compose/Dockerfile b/examples/coordination/remote-authority-compose/Dockerfile new file mode 100644 index 0000000000..f1f1186164 --- /dev/null +++ b/examples/coordination/remote-authority-compose/Dockerfile @@ -0,0 +1,14 @@ +FROM public.ecr.aws/docker/library/node:24.21.0-bookworm@sha256:3d27e5c11e5786e309ec3e03f93ae536eb36e6e5eb3714d5eb3300a36157add0 +ARG LOOPX_REVISION +ARG LOOPX_SOURCE_SHA256 +WORKDIR /opt/loopx +LABEL org.opencontainers.image.source="https://github.com/loopx-project/loopx" \ + org.opencontainers.image.revision="${LOOPX_REVISION}" +COPY loopx-source.tar.gz /tmp/loopx-source.tar.gz +RUN printf '%s %s\n' "$LOOPX_SOURCE_SHA256" /tmp/loopx-source.tar.gz | sha256sum -c - \ + && tar -xzf /tmp/loopx-source.tar.gz \ + && rm /tmp/loopx-source.tar.gz \ + && command -v openssl \ + && npm ci --ignore-scripts --no-audit --no-fund --include=dev +COPY provision.mjs launch.mjs ./ +USER 1000:1000 diff --git a/examples/coordination/remote-authority-compose/README.en.md b/examples/coordination/remote-authority-compose/README.en.md new file mode 100644 index 0000000000..97a840944c --- /dev/null +++ b/examples/coordination/remote-authority-compose/README.en.md @@ -0,0 +1,58 @@ +# Self-hosted LoopX coordination + +Extract a built bundle on a machine with Docker Engine and Compose, then run the commands below. The first start uses container registries and npm. Select a free loopback port with `LOOPX_PORT`, which defaults to 8443. + +```sh +./start.sh +./client.sh a read +./client.sh b read +./client.sh reader read +``` + +Two trusted writers have distinct credentials and can hand off the same starter Todo. An additional `reader` has its own credential and can read only the allowed starter task. The first start provisions PostgreSQL authority state, a private CA and clients. Later starts reuse the task, store identity and policies from the volumes. The database runs on an internal network. + +Use `./client.sh a descriptor` to get the tenant, Goal, actor and store identity. Pass the request on standard input to `./client.sh a request`. When a reply is lost, keep the original request and `request_id` and pass them to `./client.sh a recover`. + +A remote client can forward the server's loopback port over SSH and use the same revision of `remote-authority-client.ts` with `--ca`. Export the CA with `./client.sh a ca` and deliver the client's `credential` and `request.json` privately from its volume. The CLI reads the credential from `LOOPX_AUTHORITY_CREDENTIAL`. [Gateway requests and recovery](https://github.com/loopx-project/loopx/blob/main/docs/reference/remote-authority-task-service-v0.md) + +Both TLS gateways publish loopback ports: trusted writers use `LOOPX_PORT` (8443 by default), while the scoped reader uses `LOOPX_READ_PORT` (8444 by default). Its private policy binds the exact Goal lifetime, Todo and original creation/binding receipt. Unlisted resources are denied. Snapshots return only allowed state, without notes, lease keys or a global cursor. Todo access does not authorize historical operation receipts. + +The trusted writer endpoint retains v0 scope/operation grants; a/b can read other tasks in the same scope. The v1 resource ACL supports read/snapshot, not resource-scoped writes. Give observers only the reader credential and read port. Both listeners reuse the existing service and restricted database role. Reader isolation does not qualify resource isolation for writers. TLS covers client-to-gateway connections; database connections use the isolated backend network. + +## Build and qualify + +Run this from the reviewed template checkout. Supply the complete Git commit to use for the gateway build. + +```sh +python3 examples/coordination/remote-authority-compose/build_bundle.py \ + --source-revision "$(git rev-parse HEAD)" --output /tmp/loopx-authority.tar.gz +``` + +The same source commit and template contents produce the same archive. The bundle contains `LICENSE`, `NOTICE`, the source digest and pinned image manifests. Docker verifies the digest before unpacking the source. Keep generated bundles, credentials, data and backups in their respective release or runtime storage. + +The source revision must include scoped v1 reads (#8861). Qualify from an unpacked bundle on a disposable Docker host with a unique project name and two free loopback ports. On amd64, verify the architecture of the host and running services. Time cold and cached starts through image pull, build, readiness and client reads. Check that PostgreSQL has no published port, clients have no route to the backend network and they have no database credential. + +Run the rehearsal from the repository. It creates a temporary project, exercises writer claim, expiry, takeover and receipt recovery after restart, plus reader filtering, write denial, persistent revocation and missing/malformed policy refusal, then removes its own containers and volumes. + +```sh +node examples/coordination/remote-authority-compose/rehearse.mjs /path/to/unpacked-bundle +``` + +Compose limits CPU, memory, processes and retained logs. Check the Docker data disk and current load before a shared-host trial. Use dedicated projects, ports, networks and volumes. Tests with two local clients qualify the protocol. Qualifying two physical hosts requires a client on the second host. + +## Back up, restore and rotate + +```sh +./state.sh backup /absolute/path/to/new-backup +./start.sh +COMPOSE_PROJECT_NAME=loopx-restored ./state.sh restore /absolute/path/to/new-backup +COMPOSE_PROJECT_NAME=loopx-restored LOOPX_PORT=8445 LOOPX_READ_PORT=8446 ./start.sh +``` + +Backup stops the project and captures its database and credential volumes together. Keep backups encrypted and access-controlled. Restore verifies the digests and bundle version, and requires a new project with no existing target volumes. Read back tasks and original operation receipts before switching clients, and stop admitting requests on the old project first. + +- Revoke access by atomically replacing `policy.json` in the relevant gateway volume: `reader-gateway` for scoped reads and `gateway` for trusted writes. Removing a resource entry or credential rejects the next request. Replacing a running v1 listener's policy with v0 cannot downgrade its permissions. Rotate high-entropy credentials together with their private policy digest and client file. Earlier backups still contain earlier credentials. +- The gateway certificate lasts 397 days. Reissue it with the CA material in `pg-admin` and the same SANs before expiry. Replace `cert.pem` and `key.pem` in the gateway volume, restart the gateway and check both clients. Distribute trust before rotating a CA. +- Back up before changing database credentials or versions. Rehearse migration and rollback in an isolated project. Update database roles and credential files together. Follow PostgreSQL's supported migration procedure across major versions. + +[简体中文](README.md) diff --git a/examples/coordination/remote-authority-compose/README.md b/examples/coordination/remote-authority-compose/README.md new file mode 100644 index 0000000000..f96d36bd64 --- /dev/null +++ b/examples/coordination/remote-authority-compose/README.md @@ -0,0 +1,58 @@ +# LoopX 远程协同自托管 + +在装有 Docker Engine 和 Compose 的机器上解压构建好的包,进入目录运行。首次启动需要访问镜像仓库和 npm。选一个空闲的回环端口,默认使用 8443。 + +```sh +./start.sh +./client.sh a read +./client.sh b read +./client.sh reader read +``` + +两个可信写客户端拥有不同凭据,可以交接同一个示例任务。额外的 `reader` 使用独立凭据,只能读取获准的示例任务。首次启动创建 PostgreSQL 权威库、私有 CA 和客户端。重复启动沿用卷里的任务、存储身份和撤权策略。数据库在内部网络中运行。 + +用 `./client.sh a descriptor` 取得请求的租户、Goal、Actor 和存储实例。把请求放在标准输入上,用 `./client.sh a request` 执行。发送超时后,保留原请求及 `request_id`,通过 `./client.sh a recover` 查询回执。 + +远端客户端可用 SSH 将网关的回环端口转发到本机,再用同一版本的 `remote-authority-client.ts` 和 `--ca` 连接。管理员可导出 `./client.sh a ca`,并私下交付对应客户端卷中的 `credential` 和 `request.json`。客户端通过 `LOOPX_AUTHORITY_CREDENTIAL` 接收凭据。 [网关请求与重试说明(英文)](https://github.com/loopx-project/loopx/blob/main/docs/reference/remote-authority-task-service-v0.md) + +两个 TLS 网关都只发布回环端口:可信写客户端使用 `LOOPX_PORT`(默认 8443),受限读者使用 `LOOPX_READ_PORT`(默认 8444)。`reader` 的私有策略绑定当前 Goal 实例、具体 Todo 及其原始创建/绑定回执;未列入的资源默认拒绝。snapshot 仅返回获准状态,不含 note、租约密钥或全局游标。读授权不授予历史操作回执访问。 + +可信写入口保留 v0 的范围和操作授权,a/b 可读取同范围任务;当前 v1 资源 ACL 只支持 read/snapshot,不能提供资源级写隔离。向观察者仅交付 `reader` 的凭据和只读端口。两入口共用原服务和受限数据库角色,不能将读者隔离表述为写客户端也被资源 ACL 限制。客户端到网关使用 TLS,数据库连接走隔离 backend 网络。 + +## 构建和验收 + +从经审阅的模板 checkout 运行。选择用于构建网关的完整 Git 提交。 + +```sh +python3 examples/coordination/remote-authority-compose/build_bundle.py \ + --source-revision "$(git rev-parse HEAD)" --output /tmp/loopx-authority.tar.gz +``` + +同一源码提交和模板内容生成相同的归档。归档内含 `LICENSE`、`NOTICE`、源码摘要及固定镜像清单。Docker 构建前会校验源码摘要。生成物、运行凭据、数据库和备份应存放在各自的发布或运行位置。 + +源码版本须包含 scoped v1 读取实现(#8861)。在专用测试 Docker 环境中解压验证,使用唯一的 Compose 项目名和两个空闲回环端口。在 amd64 主机上验证本机架构和实际运行架构。冷启动与缓存启动分别记录镜像拉取、构建、就绪与客户端首次读回时间。确认数据库端口未发布,客户端无法访问 backend 网络,网关不向客户端提供数据库凭据。 + +从仓库运行下列演练。它会创建临时项目,验证写客户端的认领、到期接管和重启后的回执,以及只读客户端的资源过滤、拒写、撤权重启和策略缺失/损坏拒绝,最后清理自己的容器和卷。 + +```sh +node examples/coordination/remote-authority-compose/rehearse.mjs /path/to/unpacked-bundle +``` + +Compose 为数据库、网关及辅助容器设置 CPU、内存、进程数和日志轮转。共享主机还需核对 Docker 数据盘空间与已有负载。选择独立项目名、端口、网络和卷。两个同主机容器可以验证协议,跨物理主机资格需从另一台主机连接验证。 + +## 备份、恢复和轮换 + +```sh +./state.sh backup /absolute/path/to/new-backup +./start.sh +COMPOSE_PROJECT_NAME=loopx-restored ./state.sh restore /absolute/path/to/new-backup +COMPOSE_PROJECT_NAME=loopx-restored LOOPX_PORT=8445 LOOPX_READ_PORT=8446 ./start.sh +``` + +备份会停止当前 Compose 项目,保存同一时点的数据库和凭据卷。请将含私钥及数据库凭据的备份放在加密受限的位置。恢复前校验摘要和包版本,并要求新项目的目标卷尚未创建。启动后读回任务和原操作回执,确认旧项目已停止接收请求再切换入口。 + +- 撤权时原子替换对应网关卷中的 `policy.json`。只读策略位于 `reader-gateway` 卷,写策略位于 `gateway` 卷;撤销资源条目或凭据后下一次请求即拒绝。不能把 v0 策略放到已启动的 v1 监听器来降级权限。轮换密钥须生成新的高熵凭据,同步更新私有策略和客户端凭据。备份也含历史凭据。 +- 服务证书有效期 397 天。可在到期前用 `pg-admin` 中的 CA 签发保留原 SAN 的证书,更新网关卷中的 `cert.pem`、`key.pem`,重启网关并读回两个客户端。更换 CA 时需提前分发新信任。 +- 更新 PostgreSQL 口令或版本前先备份,在隔离项目演练迁移和回退。同步修改数据库角色与对应凭据文件。PostgreSQL 跨大版本需使用其正式迁移流程。 + +[English](README.en.md) diff --git a/examples/coordination/remote-authority-compose/build_bundle.py b/examples/coordination/remote-authority-compose/build_bundle.py new file mode 100644 index 0000000000..baa8f4beb0 --- /dev/null +++ b/examples/coordination/remote-authority-compose/build_bundle.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Build a source-pinned, reproducible LoopX self-hosting bundle.""" + +import argparse +import gzip +import hashlib +import io +import os +from pathlib import Path +import re +import subprocess +import tarfile +import tempfile + + +TEMPLATE = Path(__file__).resolve().parent +REPOSITORY = TEMPLATE.parents[2] +BUNDLE_NAME = "loopx-remote-authority-compose" +SOURCE_PATHS = ( + "LICENSE", + "NOTICE", + "package.json", + "package-lock.json", + "examples/coordination/remote-authority-client.ts", + "examples/coordination/remote-authority-service.ts", + "loopx/control_plane", +) +BUNDLE_FILES = ( + ".dockerignore", + "Dockerfile", + "README.en.md", + "README.md", + "client.sh", + "compose.yaml", + "launch.mjs", + "provision.mjs", + "start.sh", + "state.sh", +) + + +def git(*args: str) -> bytes: + return subprocess.check_output(["git", *args], cwd=REPOSITORY) + + +def gzip_archive(data: bytes) -> bytes: + encoded = io.BytesIO() + with gzip.GzipFile(filename="", fileobj=encoded, mode="wb", mtime=0) as compressed: + compressed.write(data) + return encoded.getvalue() + + +def bundle(revision: str) -> bytes: + source_tar = git("archive", "--format=tar", revision, "--", *SOURCE_PATHS) + source_archive = gzip_archive(source_tar) + source_digest = hashlib.sha256(source_archive).hexdigest() + files = {name: (TEMPLATE / name).read_bytes() for name in BUNDLE_FILES} + image_id = hashlib.sha256(b"".join( + hashlib.sha256(data).digest() + for data in (source_archive, files["Dockerfile"], files["compose.yaml"], + files["provision.mjs"], files["launch.mjs"]) + )).hexdigest() + files[".env"] = ( + f"LOOPX_REVISION={revision}\nLOOPX_SOURCE_SHA256={source_digest}\n" + f"LOOPX_IMAGE_ID={image_id}\n" + ).encode("ascii") + files["loopx-source.tar.gz"] = source_archive + for name in ("LICENSE", "NOTICE"): + files[name] = git("show", f"{revision}:{name}") + + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w", format=tarfile.USTAR_FORMAT) as output: + for name, contents in sorted(files.items()): + entry = tarfile.TarInfo(f"{BUNDLE_NAME}/{name}") + entry.size = len(contents) + entry.mode = 0o755 if name.endswith(".sh") else 0o644 + output.addfile(entry, io.BytesIO(contents)) + return gzip_archive(archive.getvalue()) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-revision", required=True, help="full LoopX Git commit") + parser.add_argument("--output", type=Path, required=True, help="new tar.gz file") + args = parser.parse_args() + if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision): + parser.error("--source-revision must be a full lowercase Git commit SHA") + try: + if git("cat-file", "-t", args.source_revision).strip() != b"commit": + parser.error("--source-revision must identify a commit") + contents = bundle(args.source_revision) + with tempfile.TemporaryDirectory(dir=args.output.parent) as work: + temporary = Path(work) / "bundle.tar.gz" + temporary.write_bytes(contents) + os.link(temporary, args.output) + except (OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"Could not build bundle: {error}\n") + print(f"{hashlib.sha256(contents).hexdigest()} {args.output}") + + +if __name__ == "__main__": + main() diff --git a/examples/coordination/remote-authority-compose/client.sh b/examples/coordination/remote-authority-compose/client.sh new file mode 100755 index 0000000000..97c343ddfb --- /dev/null +++ b/examples/coordination/remote-authority-compose/client.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +case "${1:-}" in + a|b|reader) actor=$1; shift ;; + *) printf '%s\n' "Usage: ./client.sh a|b|reader [read|request|recover|descriptor|ca]" >&2; exit 2 ;; +esac +directory=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$directory" +[ "$#" -gt 0 ] || set -- read +exec docker compose run --rm --no-deps -T "client-${actor}" "$@" diff --git a/examples/coordination/remote-authority-compose/compose.yaml b/examples/coordination/remote-authority-compose/compose.yaml new file mode 100644 index 0000000000..79b17816d4 --- /dev/null +++ b/examples/coordination/remote-authority-compose/compose.yaml @@ -0,0 +1,147 @@ +name: loopx-authority +x-app: &app + image: loopx/remote-authority:${LOOPX_IMAGE_ID:?unpack a generated bundle} + pull_policy: never + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + cpus: "0.50" + mem_limit: 512m + pids_limit: 128 + logging: &bounded-logs + driver: local + options: + max-size: "10m" + max-file: "3" +services: + secrets: + <<: *app + build: + context: . + args: + LOOPX_REVISION: ${LOOPX_REVISION:?unpack a generated bundle} + LOOPX_SOURCE_SHA256: ${LOOPX_SOURCE_SHA256:?unpack a generated bundle} + user: "0:0" + cap_add: [CHOWN, DAC_OVERRIDE] + network_mode: none + tmpfs: [/tmp] + command: [node, --experimental-strip-types, provision.mjs, secrets] + volumes: + - pg-admin:/secrets/admin + - pg-auth:/secrets/pg-auth + - pg-data:/secrets/database:ro + - gateway:/secrets/gateway + - reader-gateway:/secrets/reader-gateway + - client-a:/secrets/client-a + - client-b:/secrets/client-b + - reader:/secrets/reader + - health:/secrets/health + db: + image: public.ecr.aws/docker/library/postgres:16.16-bookworm@sha256:a49f7438ba03589f9b1ca062f2da82e899b48ea1dc50d2449fa09a70be83ebca + depends_on: + secrets: + condition: service_completed_successfully + environment: + POSTGRES_DB: authority + POSTGRES_USER: bootstrap + POSTGRES_PASSWORD_FILE: /secrets/pg-auth/postgres-password + volumes: + - pg-auth:/secrets/pg-auth:ro + - pg-data:/var/lib/postgresql/data + networks: [backend] + cpus: "0.75" + mem_limit: 768m + pids_limit: 128 + logging: *bounded-logs + restart: unless-stopped + healthcheck: + test: [CMD, pg_isready, -h, 127.0.0.1, -U, bootstrap, -d, authority] + interval: 2s + timeout: 2s + retries: 40 + setup: + <<: *app + user: "0:0" + cap_add: [DAC_OVERRIDE] + depends_on: + db: + condition: service_healthy + command: [node, --experimental-strip-types, provision.mjs, database] + volumes: + - pg-admin:/secrets/admin + - pg-auth:/secrets/pg-auth:ro + - gateway:/secrets/gateway:ro + networks: [backend] + gateway: + <<: *app + depends_on: + setup: + condition: service_completed_successfully + command: [node, --experimental-strip-types, launch.mjs, gateway] + volumes: + - gateway:/secrets/gateway:ro + - health:/secrets/health:ro + networks: [frontend, backend] + ports: + - "127.0.0.1:${LOOPX_PORT:-8443}:8443" + restart: unless-stopped + healthcheck: + test: [CMD, node, --experimental-strip-types, launch.mjs, health] + interval: 3s + timeout: 4s + retries: 25 + client-a: + <<: *app + profiles: [clients] + entrypoint: [node, --experimental-strip-types, launch.mjs, client] + volumes: + - client-a:/secrets/client:ro + networks: [frontend] + client-b: + <<: *app + profiles: [clients] + entrypoint: [node, --experimental-strip-types, launch.mjs, client] + volumes: + - client-b:/secrets/client:ro + networks: [frontend] + reader-gateway: + <<: *app + depends_on: + setup: + condition: service_completed_successfully + command: [node, --experimental-strip-types, launch.mjs, reader-gateway] + volumes: + - reader-gateway:/secrets/reader-gateway:ro + - reader:/secrets/client:ro + networks: [frontend, backend] + ports: + - "127.0.0.1:${LOOPX_READ_PORT:-8444}:8443" + restart: unless-stopped + healthcheck: + test: [CMD, node, --experimental-strip-types, launch.mjs, reader-health] + interval: 3s + timeout: 4s + retries: 25 + client-reader: + <<: *app + profiles: [clients] + entrypoint: [node, --experimental-strip-types, launch.mjs, client] + environment: + LOOPX_SERVICE_HOST: reader-gateway + volumes: + - reader:/secrets/client:ro + networks: [frontend] +networks: + backend: + internal: true + frontend: +volumes: + pg-admin: + pg-auth: + pg-data: + gateway: + reader-gateway: + client-a: + client-b: + reader: + health: diff --git a/examples/coordination/remote-authority-compose/launch.mjs b/examples/coordination/remote-authority-compose/launch.mjs new file mode 100644 index 0000000000..33bc880fe6 --- /dev/null +++ b/examples/coordination/remote-authority-compose/launch.mjs @@ -0,0 +1,44 @@ +import {readFileSync} from "node:fs"; +import {callRemoteAuthority} from "./loopx/control_plane/coordination/remote_authority_http.ts"; + +const [mode, action = "read"] = process.argv.slice(2); +const port = "8443"; +const read = (directory, file) => readFileSync(`/secrets/${directory}/${file}`, "utf8"); + +async function main() { + if (mode === "gateway" || mode === "reader-gateway") { + const password = read(mode, "database-password"); + process.env.LOOPX_AUTHORITY_DATABASE_URL = `postgresql://loopx_gateway:${encodeURIComponent(password)}@db:5432/authority`; + process.argv.splice(2, Infinity, "--policy", `/secrets/${mode}/policy.json`, + "--cert", `/secrets/${mode}/cert.pem`, "--key", `/secrets/${mode}/key.pem`, + "--host", "0.0.0.0", "--port", port); + await import("./examples/coordination/remote-authority-service.ts"); + return; + } + const directory = mode === "health" ? "health" : "client"; + const descriptor = read(directory, "request.json"); + const ca = read(directory, "ca.pem"); + if (action === "descriptor" || action === "ca") { + process.stdout.write(action === "descriptor" ? descriptor + "\n" : ca); + return; + } + const endpoint = mode === "health" || mode === "reader-health" ? `https://127.0.0.1:${port}` + : `https://${process.env.LOOPX_SERVICE_HOST ?? "gateway"}:${port}`; + const credential = read(directory, "credential"); + if (action === "request" || action === "recover") { + process.env.LOOPX_AUTHORITY_CREDENTIAL = credential; + process.argv.splice(2, Infinity, "--endpoint", endpoint, "--ca", `/secrets/${directory}/ca.pem`); + if (action === "recover") process.argv.push("--recover"); + await import("./examples/coordination/remote-authority-client.ts"); + return; + } + if (action !== "read") throw new Error("Unknown command"); + const result = await callRemoteAuthority(endpoint, credential, JSON.parse(descriptor), {ca: Buffer.from(ca)}); + if (mode === "client") process.stdout.write(JSON.stringify(result) + "\n"); + if (result.status !== "found") process.exitCode = 1; +} + +main().catch(() => { + console.error("LoopX command failed"); + process.exitCode = 1; +}); diff --git a/examples/coordination/remote-authority-compose/provision.mjs b/examples/coordination/remote-authority-compose/provision.mjs new file mode 100644 index 0000000000..0d1d68c718 --- /dev/null +++ b/examples/coordination/remote-authority-compose/provision.mjs @@ -0,0 +1,223 @@ +import {createHash, randomBytes} from "node:crypto"; +import {execFileSync} from "node:child_process"; +import {chmodSync, chownSync, existsSync, linkSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync} from "node:fs"; +import {dirname, join} from "node:path"; +import {REMOTE_AUTHORITY_POLICY, REMOTE_AUTHORITY_SCOPED_READ_POLICY} from "./loopx/control_plane/coordination/remote_authority_policy.ts"; + +const admin = "/secrets/admin"; +const pgAuth = "/secrets/pg-auth"; +const generationFile = join(admin, "generation.json"); +const provisionedFlag = join(admin, "volumes-ready"); +const installedFlag = join(admin, "database-ready"); +const tenant = "local"; +const goal = "collaboration"; +const todo = "todo_welcome"; +const actors = ["client-a", "client-b", "health", "reader"]; +const bindingOperation = "compose-bind-starter-goal"; +const goalRef = identity => ({goal_id: goal, goal_instance_id: `ginst_${identity.slice("postgresql:".length)}`}); +const secret = () => randomBytes(32).toString("hex"); +const read = path => readFileSync(path, "utf8"); +const policyDigest = value => createHash("sha256").update(value).digest("hex"); + +function createFile(path, value, uid = 0, replace = false) { + const temporary = mkdtempSync(join(dirname(path), ".init-")); + const entry = join(temporary, "file"); + try { + writeFileSync(entry, value, {mode: 0o400, flag: "wx", flush: true}); + chownSync(entry, uid, uid); + try { + if (replace) renameSync(entry, path); + else linkSync(entry, path); + } catch (error) { + if (error.code !== "EEXIST") throw error; + } + } finally { + rmSync(temporary, {recursive: true, force: true}); + } +} + +function newGeneration() { + const scratch = mkdtempSync("/tmp/authority-certs-"); + const openssl = (...args) => execFileSync("openssl", args, {cwd: scratch, stdio: "ignore", timeout: 15000}); + try { + openssl("req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256", "-nodes", + "-subj", "/CN=LoopX Local CA", "-days", "3650", "-sha256", + "-addext", "basicConstraints=critical,CA:TRUE,pathlen:0", + "-addext", "keyUsage=critical,keyCertSign,cRLSign", + "-keyout", "ca.key", "-out", "ca.pem"); + openssl("req", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256", "-nodes", + "-subj", "/CN=gateway", "-keyout", "server.key", "-out", "server.csr"); + writeFileSync(join(scratch, "server.ext"), "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:gateway,DNS:reader-gateway,DNS:localhost,IP:127.0.0.1\n"); + openssl("x509", "-req", "-in", "server.csr", "-CA", "ca.pem", "-CAkey", "ca.key", + "-CAcreateserial", "-days", "397", "-sha256", "-extfile", "server.ext", "-out", "server.pem"); + return { + identity: `postgresql:${randomBytes(16).toString("hex")}`, + databasePassword: secret(), + runtimePassword: secret(), + credentials: Object.fromEntries(actors.map(actor => [actor, secret()])), + root: read(join(scratch, "ca.pem")), + rootKey: read(join(scratch, "ca.key")), + key: read(join(scratch, "server.key")), + cert: read(join(scratch, "server.pem")), + }; + } finally { + rmSync(scratch, {recursive: true, force: true}); + } +} + +function prepareSecretVolumes() { + const requiredFiles = { + gateway: ["database-password", "policy.json", "cert.pem", "key.pem"], + "reader-gateway": ["database-password", "policy.json", "cert.pem", "key.pem"], + ...Object.fromEntries(actors.map(actor => [actor, ["credential", "ca.pem", "request.json"]])), + "pg-auth": ["postgres-password"], + }; + const hasDatabase = existsSync("/secrets/database/PG_VERSION"); + if (!existsSync(generationFile)) { + const previous = hasDatabase || Object.keys(requiredFiles).some(volume => existsSync(`/secrets/${volume}/identity`)); + if (previous) throw new Error("bootstrap identity missing for existing state"); + chmodSync(admin, 0o700); + createFile(generationFile, JSON.stringify(newGeneration())); + } + const generation = JSON.parse(read(generationFile)); + const {identity} = generation; + const initialized = existsSync(provisionedFlag); + if ((initialized && read(provisionedFlag) !== identity) || (existsSync(installedFlag) && !hasDatabase)) { + throw new Error("mismatched bootstrap identity or missing existing database volume"); + } + if (!initialized) { + populateSecretVolumes(generation); + createFile(join(admin, "ca.key"), generation.rootKey); + createFile(join(admin, "ca.pem"), generation.root); + } + for (const [volume, files] of Object.entries(requiredFiles)) { + const location = `/secrets/${volume}`; + if (read(join(location, "identity")) !== identity || files.some(file => !existsSync(join(location, file)))) { + throw new Error("credential volume needs restoration"); + } + } + createFile(provisionedFlag, identity); + // The generation record needs plaintext copies only to resume an interrupted first start. + if (generation.credentials) createFile(generationFile, JSON.stringify({identity}), 0, true); +} + +function populateSecretVolumes({identity, credentials, root, cert, key, runtimePassword, databasePassword}) { + const scopes = [{tenant_id: tenant, goal_id: goal, store_identity: identity, registered_agents: actors}]; + const principals = actors.filter(actor => actor !== "reader").map(actor => ({ + principal_id: `principal-${actor}`, + credential_sha256: policyDigest(credentials[actor]), + grants: [{ + tenant_id: tenant, goal_id: goal, actor_id: actor, + operations: actor === "health" ? ["read"] : [ + "read", "claim", "progress", "renew", "release", "complete", ...(actor === "client-a" ? ["todo_add"] : []), + ], + }], + })); + const files = { + gateway: {"database-password": runtimePassword, "policy.json": JSON.stringify({ + schema_version: REMOTE_AUTHORITY_POLICY, scopes, principals, + }), "cert.pem": cert, "key.pem": key}, + "reader-gateway": {"database-password": runtimePassword, "policy.json": JSON.stringify({ + schema_version: REMOTE_AUTHORITY_SCOPED_READ_POLICY, + scopes: [{...scopes[0], registered_agents: ["reader"], todo_read_acl: [{ + goal_ref: goalRef(identity), todo_id: todo, principal_id: "principal-reader", actor_id: "reader", + todo_origin_operation_id: bindingOperation, + }]}], + principals: [{principal_id: "principal-reader", credential_sha256: policyDigest(credentials.reader), + grants: [{tenant_id: tenant, goal_id: goal, actor_id: "reader", goal_ref: goalRef(identity), + operations: ["read", "snapshot"]}]}], + }), "cert.pem": cert, "key.pem": key}, + ...Object.fromEntries(actors.map(actor => [actor, { + credential: credentials[actor], "ca.pem": root, + "request.json": JSON.stringify({ + schema_version: "loopx_remote_authority_request_v0", tenant_id: tenant, goal_id: goal, + store_identity: identity, actor_id: actor, todo_id: todo, operation: "read", + }), + }])), + "pg-auth": {"postgres-password": databasePassword}, + }; + for (const [volume, contents] of Object.entries(files)) { + const location = `/secrets/${volume}`; + const owner = volume === "pg-auth" ? 0 : 1000; + chmodSync(location, 0o700); + chownSync(location, owner, owner); + const marker = join(location, "identity"); + createFile(marker, identity, owner); + if (read(marker) !== identity) throw new Error("mismatched credential volume"); + for (const [file, value] of Object.entries(contents)) createFile(join(location, file), value, owner); + } +} + +async function prepareDatabase() { + const {Pool} = await import("pg"); + const {setupPostgreSqlAuthoritySchemaAndRole} = await import("./loopx/control_plane/coordination/postgresql_authority_seed.ts"); + const {PostgreSqlAuthorityStore} = await import("./loopx/control_plane/coordination/postgresql_authority_store.ts"); + const {coordinationTodoReadModel} = await import("./loopx/control_plane/coordination/coordination_projection.ts"); + const {canonicalTodoDomainRecord, TODO_DOMAIN_ITEM_SCHEMA, TODO_DOMAIN_READ_RECORD_SCHEMA} = + await import("./loopx/control_plane/coordination/coordination_state_contract.ts"); + const {materializeTodoRecordForSchema} = await import("./loopx/control_plane/coordination/todo_presentation.ts"); + const {transitionGoalAcceptanceLifecycle} = await import("./loopx/control_plane/goals/acceptance_authority.ts"); + const {identity} = JSON.parse(read(generationFile)); + const runtimePassword = read("/secrets/gateway/database-password"); + const installed = existsSync(installedFlag); + if (read(provisionedFlag) !== identity || (installed && read(installedFlag) !== identity) + || !/^[a-f0-9]{64}$/.test(runtimePassword)) throw new Error("mismatched database initialization state"); + const opts = {host: "db", port: 5432, database: "authority", connectionTimeoutMillis: 5000, statement_timeout: 15000}; + const scopedStore = pool => new PostgreSqlAuthorityStore({connect: async () => { + const client = await pool.connect(); + return {query: (sql, args) => client.query(sql, args ? [...args] : undefined), release: error => client.release(error)}; + }}, {tenant_id: tenant, goal_id: goal, expected_store_identity: identity}); + const setup = new Pool({...opts, user: "bootstrap", password: read(join(pgAuth, "postgres-password"))}); + const runtime = new Pool({...opts, user: "loopx_gateway", password: runtimePassword}); + try { + if (!installed) { + const database = {connect: async () => { + const client = await setup.connect(); + return {query: (sql, args) => client.query(sql, args ? [...args] : undefined), release: error => client.release(error)}; + }}; + const schema = await setupPostgreSqlAuthoritySchemaAndRole({database, storeIdentity: identity, runtimeRole: "loopx_gateway"}); + if (schema.status !== "installed") throw new Error("authority schema installation failed"); + const role = await setup.query("SELECT rolcanlogin FROM pg_roles WHERE rolname = 'loopx_gateway'"); + if (!role.rows[0]?.rolcanlogin) { + await setup.query(`ALTER ROLE loopx_gateway LOGIN PASSWORD '${runtimePassword}'`); + } + await setup.query("REVOKE CONNECT, TEMPORARY ON DATABASE authority FROM PUBLIC; GRANT CONNECT ON DATABASE authority TO loopx_gateway"); + const store = scopedStore(runtime); + const before = await store.loadAuthority(); + if (before.status === "missing") { + const record = materializeTodoRecordForSchema(canonicalTodoDomainRecord({ + schema_version: TODO_DOMAIN_ITEM_SCHEMA, todo_id: todo, role: "agent", status: "open", + done: false, archive_state: "active", task_class: "advancement_task", + text: "Verify the two-client handoff from claim to accepted result", + }), TODO_DOMAIN_READ_RECORD_SCHEMA, "starter Todo"); + const todos = [record]; + const seeded = await store.commitAuthority({operation_id: "compose-initial-state", expected_provider_revision: null, + next_projection: {goal_id: goal, handoff_mode: "hard_lease", todos, leases: [], + todo_read_model: coordinationTodoReadModel(todos, TODO_DOMAIN_READ_RECORD_SCHEMA)}, events: [], receipts: []}); + if (seeded.status !== "applied") throw new Error("initial task could not be committed"); + } + const bound = await transitionGoalAcceptanceLifecycle(store, { + goal_id: goal, actor_agent_id: null, operation_id: bindingOperation, + transition: {kind: "bind_existing", goal_ref: goalRef(identity)}, + }); + if (bound.status !== "applied" && bound.status !== "replayed") throw new Error("starter Goal could not be bound"); + } + const after = await scopedStore(runtime).loadAuthority(); + if (after.status !== "loaded") throw new Error("initialized database is unavailable"); + createFile(installedFlag, identity); + } finally { + await Promise.all([setup.end(), runtime.end()]); + } +} + +const phase = process.argv[2]; +try { + if (phase === "secrets") prepareSecretVolumes(); + else if (phase === "database") await prepareDatabase(); + else throw new Error("unknown bootstrap command"); + process.stdout.write(`LoopX ${phase} ready\n`); +} catch (error) { + // Database/crypto errors may reveal credentials. Expose the phase and stable code only. + console.error(`LoopX ${phase} failed (${error.code ?? "init_state"})`); + process.exitCode = 1; +} diff --git a/examples/coordination/remote-authority-compose/rehearse.mjs b/examples/coordination/remote-authority-compose/rehearse.mjs new file mode 100644 index 0000000000..3b2727d3e4 --- /dev/null +++ b/examples/coordination/remote-authority-compose/rehearse.mjs @@ -0,0 +1,166 @@ +#!/usr/bin/env node +// Exercise the packaged entry points against a disposable Compose project. +import assert from "node:assert/strict"; +import {spawnSync} from "node:child_process"; +import {randomBytes} from "node:crypto"; +import {createServer} from "node:net"; +import {resolve} from "node:path"; +import {performance} from "node:perf_hooks"; +import {setTimeout as sleep} from "node:timers/promises"; + +const [bundle] = process.argv.slice(2); +if (!bundle || process.argv.length !== 3 || bundle === "--help") { + console.log("Usage: node rehearse.mjs "); + process.exit(bundle === "--help" ? 0 : 2); +} + +const project = `loopx-rehearsal-${randomBytes(6).toString("hex")}`; +const cwd = resolve(bundle); +const env = {...process.env, COMPOSE_PROJECT_NAME: project, LOOPX_PORT: await freeLoopbackPort(), + LOOPX_READ_PORT: await freeLoopbackPort()}; +const started = performance.now(); +let readySeconds; +let operationError; + +async function freeLoopbackPort() { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const port = server.address().port; + await new Promise(resolve => server.close(resolve)); + return String(port); +} + +function run(command, args, {input, allowRejected = false} = {}) { + const result = spawnSync(command, args, {cwd, env, input, encoding: "utf8", timeout: 150_000}); + if (result.error) throw result.error; + if (result.signal || (result.status !== 0 && !allowRejected)) { + throw new Error(`${command} ${args.join(" ")} exited ${result.status ?? result.signal}: ${result.stderr.slice(-2048)}`); + } + return result.stdout; +} + +function client(actor, action, request) { + return JSON.parse(run("./client.sh", [actor, action], { + input: request === undefined ? undefined : JSON.stringify(request), + allowRejected: true, + })); +} + +try { + const startup = run("./start.sh", []); + const a = client("a", "descriptor"); + const b = client("b", "descriptor"); + const reader = client("reader", "descriptor"); + const aStart = client("a", "read"); + const bStart = client("b", "read"); + assert.equal(aStart.status, "found"); + assert.equal(bStart.status, "found"); + assert.equal(aStart.provider_revision, bStart.provider_revision); + assert.deepEqual(aStart.todo, bStart.todo); + assert.notEqual(a.actor_id, b.actor_id); + assert.equal(client("reader", "read").schema_version, "loopx_remote_authority_scoped_read_response_v1"); + readySeconds = Math.round((performance.now() - started) / 100) / 10; + const wrongStore = {...b, store_identity: "postgresql:" + "0".repeat(32)}; + assert.equal(client("b", "request", wrongStore).status, "rejected"); + assert.equal(client("b", "request", {...b, actor_id: a.actor_id}).status, "rejected"); + const hidden = "todo_outside_read_grant"; + assert.equal(client("a", "request", {...a, todo_id: hidden, operation: "todo_add", + request_id: "rehearsal-create-hidden", text: "Task outside the reader grant", + note: "Private writer-only note"}).result.status, "applied"); + const beforeReadChecks = client("a", "read").provider_revision; + assert.equal(client("reader", "request", {...reader, todo_id: hidden}).status, "rejected"); + assert.equal(client("reader", "request", {...reader, operation: "claim", request_id: "reader-cannot-claim"}).status, "rejected"); + for (const scope of [{tenant_id: "other"}, {goal_id: "other"}, {actor_id: a.actor_id}, + {store_identity: "postgresql:" + "0".repeat(32)}]) { + assert.equal(client("reader", "request", {...reader, ...scope}).status, "rejected"); + } + const {todo_id: ignored, ...readerScope} = reader; + const snapshot = client("reader", "request", {...readerScope, operation: "snapshot"}); + assert.equal(snapshot.status, "snapshot"); + assert.deepEqual(snapshot.todos.map(row => row.todo_id), [reader.todo_id]); + assert.equal(Object.hasOwn(snapshot, "provider_revision"), false); + assert.equal(JSON.stringify(snapshot).includes("Private writer-only note"), false); + assert.equal(client("a", "read").provider_revision, beforeReadChecks); + + const claimA = {...a, operation: "claim", request_id: "rehearsal-claim-a"}; + const acquired = client("a", "request", claimA).result; + assert.equal(acquired.status, "applied"); + const progressA = {...a, operation: "progress", request_id: "rehearsal-progress-a", + claim_request_id: claimA.request_id, note: "First client prepared the handoff"}; + assert.equal(client("a", "request", progressA).result.status, "applied"); + const readA = client("a", "read"); + const renewed = client("a", "request", {...a, operation: "renew", ttl_seconds: 20, + idempotency_key: acquired.lease.idempotency_key, expected_version: readA.lease.version}).result; + assert.equal(renewed.status, "applied"); + const beforeTakeover = client("b", "request", {...b, operation: "claim", request_id: "rehearsal-too-early"}); + assert.equal(beforeTakeover.result.status, "failed"); + const remaining = Date.parse(renewed.lease.expires_at) - Date.now(); + assert(Number.isFinite(remaining) && remaining <= 21_000); + await sleep(Math.max(remaining, 0) + 250); + + const claimB = {...b, operation: "claim", request_id: "rehearsal-claim-b"}; + const takeover = client("b", "request", claimB).result; + assert.equal(takeover.status, "applied"); + assert(takeover.lease.lease_epoch > renewed.lease.lease_epoch); + assert.equal(client("a", "request", {...progressA, request_id: "rehearsal-stale-owner"}).result.status, "failed"); + const complete = {...b, operation: "complete", request_id: "rehearsal-complete", + claim_request_id: claimB.request_id, evidence: "evidence:compose-rehearsal"}; + assert.equal(client("b", "request", complete).result.status, "applied"); + const recover = client("b", "recover", complete); + assert.equal(recover.status, "found"); + assert.equal(recover.historical_only, true); + assert.match(recover.receipt_sha256, /^[a-f0-9]{64}$/); + + run("docker", ["compose", "down", "--timeout", "5"]); + run("./start.sh", []); + assert.equal(client("b", "recover", complete).receipt_sha256, recover.receipt_sha256); + assert.equal(client("b", "request", complete).result.status, "replayed"); + const aEnd = client("a", "read"); + const bEnd = client("b", "read"); + assert.equal(aEnd.todo.status, "done"); + assert.deepEqual(aEnd.todo, bEnd.todo); + assert.equal(aEnd.provider_revision, bEnd.provider_revision); + assert.equal(client("reader", "read").todo.status, "done"); + assert.equal(client("reader", "recover", {...complete, actor_id: reader.actor_id}).status, "rejected"); + function policyScript(code) { + run("docker", ["compose", "run", "--rm", "--no-deps", "-T", "--entrypoint", "node", "secrets", "-e", code]); + } + const readerPolicy = "/secrets/reader-gateway/policy.json"; + policyScript(`const fs = require('node:fs'); const p = '${readerPolicy}'; + fs.renameSync(p, p + '.saved'); fs.writeFileSync(p + '.next', JSON.stringify({ + ...JSON.parse(fs.readFileSync(p + '.saved')), principals: []})); fs.renameSync(p + '.next', p);`); + assert.equal(client("reader", "read").status, "rejected"); + run("docker", ["compose", "restart", "reader-gateway"]); + const listeningDeadline = performance.now() + 15_000; + let revoked = client("reader", "read"); + while (revoked.status === "unavailable" && performance.now() < listeningDeadline) { + await sleep(200); + revoked = client("reader", "read"); + } + assert.equal(revoked.status, "rejected"); + assert.equal(revoked.reason_code, "principal_unauthenticated"); + policyScript(`const fs = require('node:fs'); const p = '${readerPolicy}'; fs.unlinkSync(p);`); + assert.equal(client("reader", "read").status, "unavailable"); + policyScript(`const fs = require('node:fs'); const p = '${readerPolicy}'; fs.writeFileSync(p, '{');`); + assert.equal(client("reader", "read").status, "unavailable"); + policyScript(`const fs = require('node:fs'); const p = '${readerPolicy}'; fs.renameSync(p + '.saved', p);`); + assert.equal(client("reader", "read").todo.status, "done"); + assert.equal(client("a", "read").provider_revision, aEnd.provider_revision); + const architecture = run("docker", ["compose", "exec", "-T", "gateway", "uname", "-m"]).trim(); + console.log(JSON.stringify({ok: true, architecture, twoClientReadySeconds: readySeconds, + startup: startup.trim(), leaseTakeover: true, receiptPreserved: true, + scopedReadDefaultDeny: true, scopedWriteUnavailable: true, revocationPreserved: true})); +} catch (error) { + operationError = error; + throw error; +} finally { + try { + run("docker", ["compose", "down", "--timeout", "5", "--volumes"]); + } catch (error) { + if (!operationError) throw error; + console.error(`Cleanup of ${project} also failed: ${error.message}`); + } +} diff --git a/examples/coordination/remote-authority-compose/start.sh b/examples/coordination/remote-authority-compose/start.sh new file mode 100755 index 0000000000..d9052909a9 --- /dev/null +++ b/examples/coordination/remote-authority-compose/start.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +directory=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$directory" +started=$(date +%s) +docker compose pull --policy missing db +pulled=$(date +%s) +docker compose build secrets +built=$(date +%s) +docker compose up --detach --no-build --wait --wait-timeout 120 +ready=$(date +%s) +printf 'LoopX ready at https://127.0.0.1:%s (DB image %ss, build %ss, startup %ss, total %ss)\n' \ + "${LOOPX_PORT:-8443}" "$((pulled - started))" "$((built - pulled))" "$((ready - built))" "$((ready - started))" +printf 'Inspect the starter Todo with ./client.sh a read\n' +printf 'Scoped reads: ./client.sh reader read (https://127.0.0.1:%s)\n' "${LOOPX_READ_PORT:-8444}" diff --git a/examples/coordination/remote-authority-compose/state.sh b/examples/coordination/remote-authority-compose/state.sh new file mode 100755 index 0000000000..db4433a831 --- /dev/null +++ b/examples/coordination/remote-authority-compose/state.sh @@ -0,0 +1,74 @@ +#!/bin/sh +set -eu + +usage() { + printf '%s\n' 'Usage: ./state.sh backup /absolute/new-directory' \ + ' COMPOSE_PROJECT_NAME=new-name ./state.sh restore /absolute/backup-directory' >&2 + exit 2 +} + +[ "$#" -eq 2 ] || usage +operation=$1 +archive=$2 +case "$archive" in /*) ;; *) usage ;; esac +directory=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$directory" +project=${COMPOSE_PROJECT_NAME:-loopx-authority} +case "$project" in ''|*[!a-z0-9_-]*) usage ;; esac +image=$(docker compose config --images | awk '/^loopx\/remote-authority:/ { print; exit }') +[ -n "$image" ] || { printf '%s\n' 'Cannot resolve the packaged gateway image' >&2; exit 1; } +volumes='pg-admin pg-auth pg-data gateway reader-gateway client-a client-b reader health' +umask 077 + +digest() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum < "$1" + else + shasum -a 256 < "$1" + fi +} + +case "$operation" in + backup) + for volume in $volumes; do + label=$(docker volume inspect --format '{{index .Labels "com.docker.compose.project"}}/{{index .Labels "com.docker.compose.volume"}}' "${project}_${volume}") + [ "$label" = "$project/$volume" ] || { printf '%s\n' 'Unexpected Compose volume ownership' >&2; exit 1; } + done + docker image inspect "$image" >/dev/null + mkdir -m 700 "$archive" + docker compose --project-name "$project" down + for volume in $volumes; do + docker run --rm --pull never --read-only --network none --user 0:0 \ + --cpus 0.25 --memory 256m --pids-limit 64 --log-driver none \ + --mount "type=volume,src=${project}_${volume},dst=/state,readonly" \ + --entrypoint tar "$image" --numeric-owner -C /state -cpf - . > "$archive/$volume.tar" + digest "$archive/$volume.tar" > "$archive/$volume.sha256" + done + printf '%s\n' "$image" > "$archive/COMPLETE" + printf '%s\n' 'Backup complete. Run ./start.sh to bring this project back online.' + ;; + restore) + [ -n "${COMPOSE_PROJECT_NAME:-}" ] || usage + [ "$(cat "$archive/COMPLETE")" = "$image" ] || + { printf '%s\n' 'Incomplete backup or bundle version mismatch' >&2; exit 1; } + for volume in $volumes; do + [ "$(digest "$archive/$volume.tar")" = "$(cat "$archive/$volume.sha256")" ] || + { printf 'Checksum mismatch for %s\n' "$volume" >&2; exit 1; } + if docker volume inspect "${project}_${volume}" >/dev/null 2>&1; then + printf 'Remove or choose a different restore project for %s\n' "${project}_${volume}" >&2 + exit 1 + fi + done + docker compose --project-name "$project" build secrets + for volume in $volumes; do + docker volume create --label "com.docker.compose.project=$project" \ + --label "com.docker.compose.volume=$volume" "${project}_${volume}" >/dev/null + docker run --rm --pull never --interactive --read-only --network none --user 0:0 \ + --cpus 0.25 --memory 256m --pids-limit 64 --log-driver none \ + --mount "type=volume,src=${project}_${volume},dst=/state" \ + --entrypoint tar "$image" --numeric-owner --same-owner --same-permissions -C /state -xpf - < "$archive/$volume.tar" + done + printf '%s\n' 'Restore complete. Run ./start.sh with the same COMPOSE_PROJECT_NAME.' + ;; + *) usage ;; +esac diff --git a/tests/test_remote_authority_compose_bundle.py b/tests/test_remote_authority_compose_bundle.py new file mode 100644 index 0000000000..1e7c1e42d8 --- /dev/null +++ b/tests/test_remote_authority_compose_bundle.py @@ -0,0 +1,55 @@ +"""The public self-hosting artifact preserves provenance without runtime secrets.""" + +import hashlib +import io +from pathlib import Path +import subprocess +import sys +import tarfile + + +ROOT = Path(__file__).resolve().parents[1] +BUILDER = ROOT / "examples/coordination/remote-authority-compose/build_bundle.py" + + +def build(output: Path, revision: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(BUILDER), "--source-revision", revision, "--output", str(output)], + cwd=ROOT, capture_output=True, text=True, check=False, + ) + + +def test_bundle_is_reproducible_and_preserves_exact_source_and_license(tmp_path: Path) -> None: + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + first, second = tmp_path / "first.tar.gz", tmp_path / "second.tar.gz" + assert build(first, revision).returncode == 0 + assert build(second, revision).returncode == 0 + assert first.read_bytes() == second.read_bytes() + with tarfile.open(first) as archive: + prefix = "loopx-remote-authority-compose/" + for name in ("LICENSE", "NOTICE"): + assert archive.extractfile(prefix + name).read() == subprocess.check_output( + ["git", "show", f"{revision}:{name}"], cwd=ROOT, + ) + metadata = dict(line.split("=", 1) for line in archive.extractfile(prefix + ".env").read().decode().splitlines()) + source = archive.extractfile(prefix + "loopx-source.tar.gz").read() + assert metadata["LOOPX_REVISION"] == revision + assert hashlib.sha256(source).hexdigest() == metadata["LOOPX_SOURCE_SHA256"] + with tarfile.open(fileobj=io.BytesIO(source)) as code: + path = "loopx/control_plane/coordination/remote_authority_policy.ts" + assert code.extractfile(path).read() == subprocess.check_output(["git", "show", f"{revision}:{path}"], cwd=ROOT) + assert archive.getmember(prefix + "start.sh").mode == 0o755 + assert all(member.isfile() and not member.name.startswith("/") and ".." not in Path(member.name).parts + for member in archive.getmembers()) + assert not any(Path(member.name).name in {"credential", "policy.json", "key.pem", "generation.json"} + for member in archive.getmembers()) + + +def test_builder_refuses_bad_revision_and_preserves_existing_output(tmp_path: Path) -> None: + output = tmp_path / "bundle.tar.gz" + assert build(output, "main").returncode != 0 + assert not output.exists() + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + output.write_bytes(b"existing artifact") + assert build(output, revision).returncode != 0 + assert output.read_bytes() == b"existing artifact" From 492937e302f0537781c213b6bcf20147f329f1fc Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:31:39 +0800 Subject: [PATCH 2/4] fix(coordination): recover historical task creation receipts Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../remote-authority-compose/README.en.md | 2 +- .../remote-authority-compose/README.md | 2 +- .../remote-authority-compose/rehearse.mjs | 12 +++- .../coordination/remote_authority_service.ts | 18 ++++-- .../control_plane/coordination/todo_create.ts | 14 ++++- .../remote_authority.integration.test.ts | 59 ++++++++++++------- .../todo_create_reviewed.test.ts | 19 +++++- 7 files changed, 90 insertions(+), 36 deletions(-) diff --git a/examples/coordination/remote-authority-compose/README.en.md b/examples/coordination/remote-authority-compose/README.en.md index 97a840944c..4b525ff52a 100644 --- a/examples/coordination/remote-authority-compose/README.en.md +++ b/examples/coordination/remote-authority-compose/README.en.md @@ -32,7 +32,7 @@ The same source commit and template contents produce the same archive. The bundl The source revision must include scoped v1 reads (#8861). Qualify from an unpacked bundle on a disposable Docker host with a unique project name and two free loopback ports. On amd64, verify the architecture of the host and running services. Time cold and cached starts through image pull, build, readiness and client reads. Check that PostgreSQL has no published port, clients have no route to the backend network and they have no database credential. -Run the rehearsal from the repository. It creates a temporary project, exercises writer claim, expiry, takeover and receipt recovery after restart, plus reader filtering, write denial, persistent revocation and missing/malformed policy refusal, then removes its own containers and volumes. +Run the rehearsal from the repository. It creates a temporary project, exercises writer claim, expiry, takeover, original receipt recovery and replay after restart for creation and completion, plus reader filtering, write denial, persistent revocation and missing/malformed policy refusal, then removes its own containers and volumes. ```sh node examples/coordination/remote-authority-compose/rehearse.mjs /path/to/unpacked-bundle diff --git a/examples/coordination/remote-authority-compose/README.md b/examples/coordination/remote-authority-compose/README.md index f96d36bd64..8ae2eb6dde 100644 --- a/examples/coordination/remote-authority-compose/README.md +++ b/examples/coordination/remote-authority-compose/README.md @@ -32,7 +32,7 @@ python3 examples/coordination/remote-authority-compose/build_bundle.py \ 源码版本须包含 scoped v1 读取实现(#8861)。在专用测试 Docker 环境中解压验证,使用唯一的 Compose 项目名和两个空闲回环端口。在 amd64 主机上验证本机架构和实际运行架构。冷启动与缓存启动分别记录镜像拉取、构建、就绪与客户端首次读回时间。确认数据库端口未发布,客户端无法访问 backend 网络,网关不向客户端提供数据库凭据。 -从仓库运行下列演练。它会创建临时项目,验证写客户端的认领、到期接管和重启后的回执,以及只读客户端的资源过滤、拒写、撤权重启和策略缺失/损坏拒绝,最后清理自己的容器和卷。 +从仓库运行下列演练。它会创建临时项目,验证写客户端的认领、到期接管、新增任务与完成操作的原回执恢复及重启重放,以及只读客户端的资源过滤、拒写、撤权重启和策略缺失/损坏拒绝,最后清理自己的容器和卷。 ```sh node examples/coordination/remote-authority-compose/rehearse.mjs /path/to/unpacked-bundle diff --git a/examples/coordination/remote-authority-compose/rehearse.mjs b/examples/coordination/remote-authority-compose/rehearse.mjs index 3b2727d3e4..a2c11d462a 100644 --- a/examples/coordination/remote-authority-compose/rehearse.mjs +++ b/examples/coordination/remote-authority-compose/rehearse.mjs @@ -67,9 +67,15 @@ try { assert.equal(client("b", "request", wrongStore).status, "rejected"); assert.equal(client("b", "request", {...b, actor_id: a.actor_id}).status, "rejected"); const hidden = "todo_outside_read_grant"; - assert.equal(client("a", "request", {...a, todo_id: hidden, operation: "todo_add", + const createHidden = {...a, todo_id: hidden, operation: "todo_add", request_id: "rehearsal-create-hidden", text: "Task outside the reader grant", - note: "Private writer-only note"}).result.status, "applied"); + note: "Private writer-only note"}; + assert.equal(client("a", "request", createHidden).result.status, "applied"); + const createdReceipt = client("a", "recover", createHidden); + assert.equal(createdReceipt.status, "found"); + assert.equal(createdReceipt.historical_only, true); + assert.equal(createdReceipt.result.changed, true); + assert.equal(createdReceipt.result.todo.todo_id, hidden); const beforeReadChecks = client("a", "read").provider_revision; assert.equal(client("reader", "request", {...reader, todo_id: hidden}).status, "rejected"); assert.equal(client("reader", "request", {...reader, operation: "claim", request_id: "reader-cannot-claim"}).status, "rejected"); @@ -116,6 +122,8 @@ try { run("docker", ["compose", "down", "--timeout", "5"]); run("./start.sh", []); + assert.equal(client("a", "recover", createHidden).receipt_sha256, createdReceipt.receipt_sha256); + assert.equal(client("a", "request", createHidden).result.status, "replayed"); assert.equal(client("b", "recover", complete).receipt_sha256, recover.receipt_sha256); assert.equal(client("b", "request", complete).result.status, "replayed"); const aEnd = client("a", "read"); diff --git a/loopx/control_plane/coordination/remote_authority_service.ts b/loopx/control_plane/coordination/remote_authority_service.ts index eff348b4d3..19fec31001 100644 --- a/loopx/control_plane/coordination/remote_authority_service.ts +++ b/loopx/control_plane/coordination/remote_authority_service.ts @@ -6,7 +6,7 @@ import {PostgreSqlAuthorityService} from "./postgresql_authority_service.ts"; import type {PostgreSqlAuthorityDatabase} from "./postgresql_authority_store.ts"; import {executeCoordinationTodoClaim} from "./todo_claim.ts"; import {executeCoordinationTodoUpdate} from "./todo_update.ts"; -import {executeCoordinationTodoCreate, COORDINATION_TODO_CREATE_RECEIPT_SCHEMA} from "./todo_create.ts"; +import {decodeCoordinationTodoCreateReceipt, executeCoordinationTodoCreate, COORDINATION_TODO_CREATE_RECEIPT_SCHEMA} from "./todo_create.ts"; import {executeCoordinationTodoTerminalLifecycle, type CoordinationTodoTerminalLifecycleInput} from "./todo_terminal_lifecycle.ts"; import {executeCanonicalTaskLeaseLifecycle} from "./task_lease_lifecycle.ts"; import {leaseVersion, leaseEpoch} from "../work_items/task_lease_acquire.ts"; @@ -243,11 +243,17 @@ export class RemoteAuthorityService { recorded.actor_id !== request.actor_id || recorded.request_sha256 !== binding.request_sha256) { return reply({status: "rejected", reason_code: "remote_authority_operation_mismatch"}); } - if (mode === "recover") return reply({status: "found", operation_id: operationId, - provider_revision: previous.provider_revision, cursor: previous.cursor, - receipt_sha256: canonicalAuthoritySha256(original), historical_only: true, - result: commandResult(original.result && typeof original.result === "object" - ? original.result as JsonObject : original)}); + if (mode === "recover") { + // Creation receipts store the original Todo, not command flags. Use + // their owner's decoder while retaining the original receipt digest. + const creation = request.operation === "todo_add" && original.schema_version === COORDINATION_TODO_CREATE_RECEIPT_SCHEMA + ? decodeCoordinationTodoCreateReceipt(original, request.todo_id) : null; + const historical = creation ? {...original, ...creation.fields, changed: creation.changed} + : original.result && typeof original.result === "object" ? original.result as JsonObject : original; + return reply({status: "found", operation_id: operationId, + provider_revision: previous.provider_revision, cursor: previous.cursor, + receipt_sha256: canonicalAuthoritySha256(original), historical_only: true, result: commandResult(historical)}); + } } else if (previous && previous.status !== "missing") { return reply({status: previous.reason_code === "remote_authority_operation_mismatch" ? "rejected" : "unavailable", reason_code: previous.reason_code}); diff --git a/loopx/control_plane/coordination/todo_create.ts b/loopx/control_plane/coordination/todo_create.ts index 64c9e4a4e9..028b3e3060 100644 --- a/loopx/control_plane/coordination/todo_create.ts +++ b/loopx/control_plane/coordination/todo_create.ts @@ -52,15 +52,23 @@ function failure(code: string, reason: string, detail: JsonObject = {}): Coordin }; } +/** Decode the creation owner's existing receipt without reading current state. */ +export function decodeCoordinationTodoCreateReceipt(original: JsonObject, todoId: string) { + const todo = canonicalAuthorityObject(original.todo, "created Todo receipt"); + if (original.schema_version !== COORDINATION_TODO_CREATE_RECEIPT_SCHEMA || + original.todo_id !== todoId || todo.todo_id !== todoId) { + throw new AuthorityStoreProtocolError("created Todo receipt identity mismatch"); + } + return {fields: {todo_id: original.todo_id, todo, original_receipt: original}, changed: true}; +} + function createReceipt(input: CoordinationTodoCreateInput, requestSha: string) { return new CoordinationCommandReceipt({result_schema: COORDINATION_TODO_CREATE_RESULT_SCHEMA, identity: {schema_version: COORDINATION_TODO_CREATE_RECEIPT_SCHEMA, operation_id: input.operation_id, goal_id: input.goal_id, todo_id: String(input.todo.todo_id), request_sha256: requestSha}, failure, decode(original) { - const todo = canonicalAuthorityObject(original.todo, "created Todo receipt"); - if (todo.todo_id !== input.todo.todo_id) throw new AuthorityStoreProtocolError("created Todo receipt identity mismatch"); - return {fields: {todo_id: original.todo_id, todo, original_receipt: original}, changed: true}; + return decodeCoordinationTodoCreateReceipt(original, String(input.todo.todo_id)); }}); } diff --git a/tests/control_plane_ts/remote_authority.integration.test.ts b/tests/control_plane_ts/remote_authority.integration.test.ts index 96436bc47c..0625560f9c 100644 --- a/tests/control_plane_ts/remote_authority.integration.test.ts +++ b/tests/control_plane_ts/remote_authority.integration.test.ts @@ -314,28 +314,43 @@ test("remote authority real TLS/PostgreSQL task journey and recovery", {skip: !c }); await t.test("lost response and server restart recover the exact receipt without repeating the write", async () => { await fixture.seed(["todo_disconnect"]); - const original = claimRequest("a", "todo_disconnect"); - // Relay the real TLS request, wait for the complete committed response, then - // deliberately discard it. No mocked database or fabricated receipt. - const proxy = createServer({key: fixture.key, cert: fixture.cert}, (incoming, outgoing) => { - const upstream = httpsRequest(new URL(incoming.url!, running.endpoint), {method: "POST", ca: fixture.cert, - headers: incoming.headers}, response => {response.resume(); response.on("end", () => outgoing.destroy());}); - upstream.on("error", () => outgoing.destroy()); incoming.pipe(upstream); - }); - await new Promise(resolve => proxy.listen(0, "127.0.0.1", resolve)); - let lost: JsonObject; - try {lost = await callRemoteAuthority(`https://127.0.0.1:${(proxy.address() as AddressInfo).port}`, credentials.a, original, {ca: fixture.cert});} - finally {await new Promise(resolve => proxy.close(() => resolve()));} - assert.equal(lost.status, "ambiguous"); - const receipt = await fixture.store.readReceipt(remoteAuthorityOperationId(original, "principal-a")); - assert.equal(receipt.status, "found"); if (receipt.status !== "found") return; - const before = await fixture.store.loadAuthority(); - const port = running.port; await running.stop(); running = await fixture.start(port); - const recovered = await fixture.client(running.endpoint, "a", original, true); - assert.equal(recovered.code, 0); assert.equal(recovered.value.status, "found"); - assert.equal(recovered.value.receipt_sha256, canonicalAuthoritySha256(receipt.receipts[0])); - assert.equal(result(await call(original)).status, "replayed"); - assert.deepEqual(await fixture.store.loadAuthority(), before); + const commands: RemoteAuthorityRequest[] = [claimRequest("a", "todo_disconnect"), + {...base("todo_created_after_disconnect"), operation: "todo_add", request_id: "create-disconnect", + text: "Synthetic task whose creation response is lost"}]; + for (const original of commands) { + // Relay the real TLS request, wait for the complete committed response, + // then discard it. Each command retains its owner's actual receipt shape. + const proxy = createServer({key: fixture.key, cert: fixture.cert}, (incoming, outgoing) => { + const upstream = httpsRequest(new URL(incoming.url!, running.endpoint), {method: "POST", ca: fixture.cert, + headers: incoming.headers}, response => {response.resume(); response.on("end", () => outgoing.destroy());}); + upstream.on("error", () => outgoing.destroy()); incoming.pipe(upstream); + }); + await new Promise(resolve => proxy.listen(0, "127.0.0.1", resolve)); + let lost: JsonObject; + try {lost = await callRemoteAuthority(`https://127.0.0.1:${(proxy.address() as AddressInfo).port}`, credentials.a, original, {ca: fixture.cert});} + finally {await new Promise(resolve => proxy.close(() => resolve()));} + assert.equal(lost.status, "ambiguous", original.operation); + const operationId = remoteAuthorityOperationId(original, "principal-a"); + const receipt = await fixture.store.readReceipt(operationId); + assert.equal(receipt.status, "found"); if (receipt.status !== "found") return; + const before = await fixture.store.loadAuthority(); + const port = running.port; await running.stop(); running = await fixture.start(port); + const recovered = await fixture.client(running.endpoint, "a", original, true); + assert.equal(recovered.code, 0, `${original.operation}: ${JSON.stringify(recovered.value)}`); + assert.equal(recovered.value.status, "found"); + assert.equal(recovered.value.historical_only, true); + assert.equal(recovered.value.receipt_sha256, canonicalAuthoritySha256(receipt.receipts[0])); + assert.equal((recovered.value.result as JsonObject).changed, true); + assert.equal(result(await call(original)).status, "replayed"); + if (original.operation === "todo_add") { + assert.equal(((recovered.value.result as JsonObject).todo as JsonObject).todo_id, original.todo_id); + const changed = await fixture.client(running.endpoint, "a", {...original, text: "Different creation intent"}, true); + assert.equal(changed.code, 1); assert.equal(changed.value.reason_code, "remote_authority_operation_mismatch"); + assert.equal((await fixture.client(running.endpoint, "b", original, true)).code, 1); + } + assert.deepEqual(await fixture.store.readReceipt(operationId), receipt, "recovery preserves the original receipt bytes"); + assert.deepEqual(await fixture.store.loadAuthority(), before, "recovery and replay commit no additional effect"); + } }); await t.test("unknown receipt query is read-only and does not turn into a claim", async () => { const before = await fixture.store.loadAuthority(); diff --git a/tests/control_plane_ts/todo_create_reviewed.test.ts b/tests/control_plane_ts/todo_create_reviewed.test.ts index d69128580a..73afeacc9e 100644 --- a/tests/control_plane_ts/todo_create_reviewed.test.ts +++ b/tests/control_plane_ts/todo_create_reviewed.test.ts @@ -11,9 +11,26 @@ import {PostgreSqlAuthorityStore, installPostgreSqlAuthorityStoreSchema, type PostgreSqlAuthorityDatabase} from "../../loopx/control_plane/coordination/postgresql_authority_store.ts"; import {coordinationTodoReadModel} from "../../loopx/control_plane/coordination/coordination_projection.ts"; import {TODO_DOMAIN_READ_RECORD_SCHEMA} from "../../loopx/control_plane/coordination/coordination_state_contract.ts"; -import {executeCoordinationTodoCreate} from "../../loopx/control_plane/coordination/todo_create.ts"; +import {executeCoordinationTodoCreate, decodeCoordinationTodoCreateReceipt, + COORDINATION_TODO_CREATE_RECEIPT_SCHEMA} from "../../loopx/control_plane/coordination/todo_create.ts"; import {createLocalCoordinationTodo} from "../../loopx/control_plane/coordination/local_authority_runtime.ts"; +test("historical creation decoding preserves the receipt and rejects mismatched identity", () => { + const original = {schema_version: COORDINATION_TODO_CREATE_RECEIPT_SCHEMA, + operation_id: "created-once", goal_id: "goal-a", todo_id: "todo-created", + todo: {todo_id: "todo-created", status: "open", role: "agent"}}; + const before = structuredClone(original); + const decoded = decodeCoordinationTodoCreateReceipt(original, "todo-created"); + assert.equal(decoded.changed, true); + assert.deepEqual(decoded.fields.todo, original.todo); + assert.deepEqual(original, before, "projection does not rewrite historical receipts"); + for (const malformed of [ + {...original, schema_version: "unrelated-receipt"}, + {...original, todo_id: "other"}, + {...original, todo: {...original.todo, todo_id: "other"}}, + ]) assert.throws(() => decodeCoordinationTodoCreateReceipt(malformed, "todo-created"), /identity mismatch/); +}); + for (const provider of ["file", "sqlite", "postgresql"] as const) { test(`${provider}: reviewed create fences concurrent commits and recovers its bound identity`, {skip: provider === "postgresql" && !process.env.LOOPX_TEST_POSTGRES_URL}, async t => { From 2b4b1f54e5b2d53c6c0a86bfc4a12ad73552b884 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Tue, 17 Nov 2026 07:36:11 +0800 Subject: [PATCH 3/4] Fix packaged rehearsal across host and authority clock skew Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../coordination/remote-authority-compose/rehearse.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/coordination/remote-authority-compose/rehearse.mjs b/examples/coordination/remote-authority-compose/rehearse.mjs index a2c11d462a..5cc85c59b2 100644 --- a/examples/coordination/remote-authority-compose/rehearse.mjs +++ b/examples/coordination/remote-authority-compose/rehearse.mjs @@ -103,9 +103,10 @@ try { assert.equal(renewed.status, "applied"); const beforeTakeover = client("b", "request", {...b, operation: "claim", request_id: "rehearsal-too-early"}); assert.equal(beforeTakeover.result.status, "failed"); - const remaining = Date.parse(renewed.lease.expires_at) - Date.now(); - assert(Number.isFinite(remaining) && remaining <= 21_000); - await sleep(Math.max(remaining, 0) + 250); + // Both timestamps belong to the authority; the host clock may differ. + const leaseDuration = Date.parse(renewed.lease.expires_at) - Date.parse(renewed.lease.updated_at); + assert(Number.isFinite(leaseDuration) && leaseDuration > 0 && leaseDuration <= 21_000); + await sleep(leaseDuration + 250); const claimB = {...b, operation: "claim", request_id: "rehearsal-claim-b"}; const takeover = client("b", "request", claimB).result; From 1286bcbf89ac35b4fdafeddb44b5988ebe523ae6 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:44:15 +0800 Subject: [PATCH 4/4] Document TLS renewal for both self-hosted gateways Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- examples/coordination/remote-authority-compose/README.en.md | 2 +- examples/coordination/remote-authority-compose/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/coordination/remote-authority-compose/README.en.md b/examples/coordination/remote-authority-compose/README.en.md index 4b525ff52a..abd42cbe9e 100644 --- a/examples/coordination/remote-authority-compose/README.en.md +++ b/examples/coordination/remote-authority-compose/README.en.md @@ -52,7 +52,7 @@ COMPOSE_PROJECT_NAME=loopx-restored LOOPX_PORT=8445 LOOPX_READ_PORT=8446 ./start Backup stops the project and captures its database and credential volumes together. Keep backups encrypted and access-controlled. Restore verifies the digests and bundle version, and requires a new project with no existing target volumes. Read back tasks and original operation receipts before switching clients, and stop admitting requests on the old project first. - Revoke access by atomically replacing `policy.json` in the relevant gateway volume: `reader-gateway` for scoped reads and `gateway` for trusted writes. Removing a resource entry or credential rejects the next request. Replacing a running v1 listener's policy with v0 cannot downgrade its permissions. Rotate high-entropy credentials together with their private policy digest and client file. Earlier backups still contain earlier credentials. -- The gateway certificate lasts 397 days. Reissue it with the CA material in `pg-admin` and the same SANs before expiry. Replace `cert.pem` and `key.pem` in the gateway volume, restart the gateway and check both clients. Distribute trust before rotating a CA. +- The generated leaf certificate lasts 397 days. Reissue it before expiry with the CA in `pg-admin` and the original SANs. Before switching, verify the new certificate with the existing CA on the clients and confirm their clocks have reached its `notBefore` time. Replace the matching `cert.pem` and `key.pem` in both the `gateway` and `reader-gateway` volumes. With the same Compose project settings, run `docker compose restart gateway reader-gateway`. Use `docker compose ps gateway reader-gateway` to confirm both services are healthy, then verify both TLS endpoints and read back the task with `./client.sh a read`, `./client.sh b read` and `./client.sh reader read`. Distribute trust before rotating a CA. - Back up before changing database credentials or versions. Rehearse migration and rollback in an isolated project. Update database roles and credential files together. Follow PostgreSQL's supported migration procedure across major versions. [简体中文](README.md) diff --git a/examples/coordination/remote-authority-compose/README.md b/examples/coordination/remote-authority-compose/README.md index 8ae2eb6dde..32965aead0 100644 --- a/examples/coordination/remote-authority-compose/README.md +++ b/examples/coordination/remote-authority-compose/README.md @@ -52,7 +52,7 @@ COMPOSE_PROJECT_NAME=loopx-restored LOOPX_PORT=8445 LOOPX_READ_PORT=8446 ./start 备份会停止当前 Compose 项目,保存同一时点的数据库和凭据卷。请将含私钥及数据库凭据的备份放在加密受限的位置。恢复前校验摘要和包版本,并要求新项目的目标卷尚未创建。启动后读回任务和原操作回执,确认旧项目已停止接收请求再切换入口。 - 撤权时原子替换对应网关卷中的 `policy.json`。只读策略位于 `reader-gateway` 卷,写策略位于 `gateway` 卷;撤销资源条目或凭据后下一次请求即拒绝。不能把 v0 策略放到已启动的 v1 监听器来降级权限。轮换密钥须生成新的高熵凭据,同步更新私有策略和客户端凭据。备份也含历史凭据。 -- 服务证书有效期 397 天。可在到期前用 `pg-admin` 中的 CA 签发保留原 SAN 的证书,更新网关卷中的 `cert.pem`、`key.pem`,重启网关并读回两个客户端。更换 CA 时需提前分发新信任。 +- 自动生成的服务叶证书有效期为 397 天。到期前用 `pg-admin` 中的 CA 续签,保留原 SAN。切换前在客户端用原 CA 校验新证书,确认各客户端时钟已到新证书的 `notBefore` 时间。将配对的 `cert.pem` 和 `key.pem` 更新到 `gateway` 与 `reader-gateway` 两个卷中。沿用同一 Compose 项目配置运行 `docker compose restart gateway reader-gateway`。用 `docker compose ps gateway reader-gateway` 确认两个服务均已就绪(`healthy`),再校验两个 TLS 入口,再用 `./client.sh a read`、`./client.sh b read` 和 `./client.sh reader read` 读回任务。更换 CA 时提前分发新信任。 - 更新 PostgreSQL 口令或版本前先备份,在隔离项目演练迁移和回退。同步修改数据库角色与对应凭据文件。PostgreSQL 跨大版本需使用其正式迁移流程。 [English](README.en.md)