From 81d117c7d259194b937fc0d5a6b13c64d1833cf2 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:05:37 +0800 Subject: [PATCH 1/2] feat(coordination): qualify real pg16/tls dual-actor rehearsal and volcengine templates (#8747) Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../volcengine-cn-beijing-pg-gateway.md | 160 +++ .../coordination/colima-pg16-e2e-rehearsal.sh | 169 ++++ .../coordination/colima-pg16-e2e-rehearsal.ts | 931 ++++++++++++++++++ .../volcengine/gateway-caddy-tls.Caddyfile | 51 + .../least-privilege-iam-policy.json | 19 + examples/coordination/volcengine/main.tf | 151 +++ .../remote-authority-systemd.service | 32 + .../coordination_colima_rehearsal.test.ts | 116 +++ tests/test_colima_pg_rehearsal.py | 165 ++++ tsconfig.control-plane.json | 2 + 10 files changed, 1796 insertions(+) create mode 100644 docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md create mode 100755 examples/coordination/colima-pg16-e2e-rehearsal.sh create mode 100644 examples/coordination/colima-pg16-e2e-rehearsal.ts create mode 100644 examples/coordination/volcengine/gateway-caddy-tls.Caddyfile create mode 100644 examples/coordination/volcengine/least-privilege-iam-policy.json create mode 100644 examples/coordination/volcengine/main.tf create mode 100644 examples/coordination/volcengine/remote-authority-systemd.service create mode 100644 tests/control_plane_ts/coordination_colima_rehearsal.test.ts create mode 100644 tests/test_colima_pg_rehearsal.py diff --git a/docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md b/docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md new file mode 100644 index 0000000000..153cb969c0 --- /dev/null +++ b/docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md @@ -0,0 +1,160 @@ +# 火山引擎 (cn-beijing) PostgreSQL 16 与远程协同网关部署参考 + +本文档规范 LoopX 在火山引擎北京地域(`cn-beijing`)的云端协同底座部署架构、参数化模板、最小权限开通清单及双人/多机分治联通流程。 + +> [!CAUTION] +> **严格红线:严禁未经用户确认自动购买云资源或改线上配置** +> 任何脚本、Agent 或自动化流绝不允许未经用户交互确认直接调用云厂商计费/下单 API(如自动开通包年包月或按量付费 RDS/ECS)或静默篡改生产环境线上防火墙/安全组/IAM 策略。所有模板与命令均为声明式、受审阅的参数化资产,仅在用户显式授权后由用户或受控发布流程执行。 + +--- + +## 1. 架构总览 (Multi-Machine Split Architecture) + +为了实现本地高性能交互与云端廉价长任务并行的双人/多机协同,LoopX 采用对齐 `postgresql-authority-service-v0` 与 `monorepo-distribution-split-v0` 的协同底座: + +``` ++-----------------------------+ +-----------------------------+ +| Local Operator Machine | | Cloud Worker Machine | +| operator.local | | worker.cloud | +| (macOS Apple Silicon dev) | | (Volcengine ECS / Ark LLM) | ++--------------+--------------+ +--------------+--------------+ + | | + | HTTPS / TLS 1.3 (Bearer Credential) | HTTPS / TLS 1.3 (Bearer Credential) + | `examples/coordination/ | `examples/coordination/ + | remote-authority-client.ts` | remote-authority-client.ts` + v v ++---------------------------------------------------------------------------+ +| Volcengine `cn-beijing` Remote Authority Gateway | +| | +| +---------------------------+ +---------------------------------+ | +| | Caddy TLS Reverse Proxy | ----> | RemoteAuthorityService (Node24) | | +| | (TLS termination, | :8443 | `loopx/control_plane/ | | +| | HSTS, IP allowlist) | | coordination/remote_authority` | | +| +---------------------------+ +----------------+----------------+ | +| | | +| VPC Internal SQL | | +| (Port 5432, No EIP) v | +| +---------------------------------+ | +| | Volcengine RDS for PostgreSQL | | +| | Version: 16 (loopx_app role) | | +| | Schema: loopx_control_plane | | +| +---------------------------------+ | +| | +| +---------------------------------------------------------------------+ | +| | Volcano Ark (`ark.cn-beijing.volces.com`) Video / LLM Model Service | | +| | - 同地域低延迟内网交互 | | +| | - 复用本机已就绪凭据 (`operations-ark-video.json`) 驱动廉价 Worker | | +| +---------------------------------------------------------------------+ | ++---------------------------------------------------------------------------+ +``` + +### 核心设计原则 +1. **网络零公网暴露数据库**:PostgreSQL 实例仅绑定 VPC 内网子网,安全组仅对网关 ECS 开放 5432 端口;严禁绑定公网弹性 IP (EIP)。 +2. **最小受限角色装配 (`loopx_app`)**:网关服务使用专用的 `loopx_app` 角色连接数据库,仅具备 `SELECT, INSERT, UPDATE` 必要权限,禁止 `DROP TABLE`, `TRUNCATE`, `ALTER ROLE` 等管理员特权。 +3. **端到端 TLS 传输认证**:外部客户端经由 TLS 1.3 连接网关,凭据在网关层做安全哈希对比(`timingSafeEqual`),绝不向数据库传递或持久化明文凭据。 +4. **同地域方舟模型协同**:利用同在 `cn-beijing` 的火山方舟大模型/视频 API 端点(`ark.cn-beijing.volces.com`),保证 Worker 节点在同一云内网络低延迟调用模型。 + +--- + +## 2. 火山引擎 cn-beijing 资源规格清单 + +| 资源类别 | 建议规格 / 配置 | 计费类型建议 | 用途 | +|---|---|---|---| +| **VPC** | CIDR: `172.16.0.0/16`, Region: `cn-beijing` | 免费 | 隔离协同云上内网 | +| **子网 (Subnet)** | 可用区: `cn-beijing-a`, CIDR: `172.16.1.0/24` | 免费 | 数据库与网关部署网段 | +| **安全组 (Gateway)** | 入方向: 仅放通 443 (TLS) 给授权 IP;出方向: 允许内网访问 RDS | 免费 | 网关流量防护 | +| **安全组 (RDS)** | 入方向: 仅放通 5432 来自网关安全组;严禁任何 `0.0.0.0/0` | 免费 | 数据库访问收敛 | +| **轻量网关 ECS** | 规格: `ecs.g3i.large` (2C4G) 或 `ecs.c3i.large` (2C4G) | 按量计费 / 抢占式 | 运行 Caddy + RemoteAuthorityService | +| **RDS for PG** | 引擎: PostgreSQL 16,规格: `rds.pg.s1.small` (1C2G) 或通用型 | 按量计费 (按需开启) | 承载协同权威状态与检查点 | +| **弹性公网 IP** | 带宽: 1-5 Mbps 按流量计费,绑定至网关 ECS | 按量计费 | 提供 HTTPS 客户端安全接入 | + +--- + +## 3. 最小权限开通清单 (Volcengine RAM Policy) + +远程 Worker 与 CI/CD 自动化部署账号仅授予最小只读查询权限。模板文件位于 `examples/coordination/volcengine/least-privilege-iam-policy.json`: + +```json +{ + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "rds_postgresql:DescribeDBInstances", + "rds_postgresql:DescribeDBInstanceDetail", + "rds_postgresql:DescribeDatabases", + "rds_postgresql:DescribeAccounts", + "vpc:DescribeVpcs", + "vpc:DescribeSubnets", + "vpc:DescribeSecurityGroups", + "ecs:DescribeInstances", + "ecs:DescribeInstanceStatus" + ], + "Resource": ["*"] + } + ] +} +``` + +> [!NOTE] +> 任何创建资源(`CreateDBInstance`、`RunInstances`)或修改安全组(`AuthorizeSecurityGroupIngress`)的操作必须由主账号管理员在控制台审批后执行,不向任何日常自动化代理开放。 + +--- + +## 4. 参数化部署模板 + +配套部署模板位于 `examples/coordination/volcengine/` 目录: +1. **Terraform 编排模板**:`main.tf` + - 定义 VPC、子网、安全组、RDS PostgreSQL 16 实例与网关 ECS 的参数化配置。 + - 所有变量均提供安全默认值(如北京地域 `cn-beijing`),并设置 `prevent_destroy = true` 避免误删。 +2. **TLS 网关配置**:`gateway-caddy-tls.Caddyfile` + - Caddy 自动化 TLS 反向代理至本地 Node 权威服务(`https://127.0.0.1:8443`)。 + - 包含请求大小限制(16KB)、连接超时约束与 HSTS 安全头。 +3. **网关守护进程服务**:`remote-authority-systemd.service` + - 以独立低特权系统用户运行 `examples/coordination/remote-authority-service.ts`。 + - 启用 `DynamicUser=yes`, `ProtectSystem=strict`, `PrivateTmp=true` 等 systemd 安全加固。 + +--- + +## 5. 多机分治与双人协同联调流程 + +### 步骤 1:本地零成本预演 (Colima / Docker) +在向火山引擎上线前,必须先在本机运行 Colima 端到端彩排套件验证: +```bash +# 模拟模式(无任何容器开销,内存数据库 + 本地 HTTPS/TLS 服务端快速预检 8 个核心切片) +./examples/coordination/colima-pg16-e2e-rehearsal.sh --simulate + +# 容器模式(启动隔离 postgres:16-alpine 容器 + 受限角色 + HTTPS/TLS 联调,缺少 Docker 直接失败退出) +./examples/coordination/colima-pg16-e2e-rehearsal.sh --rehearsal +``` + +### 步骤 2:创世种子脱敏与灌入 +对需要共享给云端的 Goal(如 `loopx-quality-review`),使用 Stage 2B 工具导出脱敏创世种子并灌入云端 RDS: +```bash +# 导出并校验隐私边界(自动剔除本地绝对路径、飞书私有 ID 与未过期租约) +node --no-warnings --experimental-sqlite --experimental-strip-types \ + -e ' + import {exportLocalToPgGenesisSeed} from "./loopx/control_plane/coordination/postgresql_authority_seed.ts"; + const result = await exportLocalToPgGenesisSeed({ + runtimeRoot: process.env.HOME + "/.loopx", + goalId: "loopx-quality-review", + tenantId: "tenant-prod", + outputPath: "/tmp/genesis-seed-canary.json" + }); + console.log("Genesis seed export result:", result.status, result.seed_digest); +' +``` + +### 步骤 3:多机客户端并发联调 +- 本机操作员 (`operator.local`): + ```bash + LOOPX_AUTHORITY_CREDENTIAL="" \ + node --no-warnings --experimental-sqlite --experimental-strip-types \ + examples/coordination/remote-authority-client.ts \ + --endpoint "https://gateway.loopx.bj.volces.com" \ + --ca /etc/ssl/certs/loopx-ca.pem + ``` +- 云端 Worker (`worker.cloud`): + 配合 `operations-ark-video.json` 中的火山方舟 API Key,在获取租约后执行测试/评审,随后向网关回传 `progress` 与带凭证证据的 `complete` 指令。 +- 租约冲突与超期接管: + 若云端 Worker 节点因网络断连或故障未续租,租约到达 TTL 后,本地操作员可无阻塞接管继续推进,彻底消除任务悬挂风险。 diff --git a/examples/coordination/colima-pg16-e2e-rehearsal.sh b/examples/coordination/colima-pg16-e2e-rehearsal.sh new file mode 100755 index 0000000000..6c77364dd9 --- /dev/null +++ b/examples/coordination/colima-pg16-e2e-rehearsal.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# ============================================================================== +# Colima / Docker PostgreSQL 16 Dual-Actor Coordination Rehearsal Suite +# +# Provides zero-cost local rehearsal for: +# - Schema initialization & least-privilege role assembly +# - Genesis seed ingestion (loopx-quality-review canary) +# - Dual-actor (operator.local / worker.cloud) lease competition & expiration takeover over HTTPS/TLS +# - Default-deny boundary rejection +# - Verified task completion & cross-machine snapshot readback +# +# Usage: +# ./colima-pg16-e2e-rehearsal.sh --simulate # Fast in-memory PG + real TLS simulation (preflight) +# ./colima-pg16-e2e-rehearsal.sh --database-url # Rehearse against existing PostgreSQL instance +# ./colima-pg16-e2e-rehearsal.sh --rehearsal # Fail-closed rehearsal against disposable postgres:16-alpine +# ./colima-pg16-e2e-rehearsal.sh --start-colima # Ensure Colima is started, then run containerized rehearsal +# ============================================================================== + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +MODE="simulate" +DATABASE_URL="${LOOPX_AUTHORITY_DATABASE_URL:-}" +START_COLIMA=false +CONTAINER_NAME="loopx-canary-pg16-rehearsal-$$-$(date +%s)" +PG_PORT="" +PORT_EXPLICIT=false +CONTAINER_STARTED=false + +cleanup() { + if [ "${CONTAINER_STARTED}" = true ]; then + echo "[colima-rehearsal] Cleaning up disposable PostgreSQL container ${CONTAINER_NAME}..." >&2 + docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT INT TERM + +show_help() { + cat < Run against an explicit PostgreSQL database URL. + --start-colima Start colima VM first if not running, then execute --rehearsal. + --port Local host port for disposable container (default: random ephemeral 127.0.0.1 port). + --help Show this help message. + +Environment Variables: + LOOPX_AUTHORITY_DATABASE_URL Default database connection URL if set. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --simulate) + MODE="simulate" + shift + ;; + --rehearsal) + MODE="container" + shift + ;; + --start-colima) + START_COLIMA=true + MODE="container" + shift + ;; + --database-url) + DATABASE_URL="$2" + MODE="real_url" + shift 2 + ;; + --port) + PG_PORT="$2" + PORT_EXPLICIT=true + shift 2 + ;; + -h|--help) + show_help + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + show_help + exit 1 + ;; + esac +done + +if [ "${START_COLIMA}" = true ]; then + if ! command -v colima >/dev/null 2>&1; then + echo "Error: colima CLI not found on PATH." >&2 + exit 1 + fi + if ! colima status >/dev/null 2>&1; then + echo "[colima-rehearsal] Starting Colima VM (2 CPU, 2GB memory)..." >&2 + colima start --cpu 2 --memory 2 + else + echo "[colima-rehearsal] Colima is already running." >&2 + fi +fi + +if [ "${MODE}" = "container" ]; then + if ! command -v docker >/dev/null 2>&1; then + echo "Error: docker CLI not found on PATH; --rehearsal requires Docker/Colima (use --simulate for preflight)." >&2 + exit 1 + fi + if ! docker info >/dev/null 2>&1; then + echo "Error: Docker daemon is unreachable; --rehearsal cannot fall back to simulation (use --simulate for preflight)." >&2 + exit 1 + fi + + PUBLISH_SPEC="127.0.0.1::5432" + if [ "${PORT_EXPLICIT}" = true ]; then + PUBLISH_SPEC="127.0.0.1:${PG_PORT}:5432" + fi + + echo "[colima-rehearsal] Starting ephemeral PostgreSQL 16 container: ${CONTAINER_NAME} (${PUBLISH_SPEC})..." >&2 + docker run -d \ + --name "${CONTAINER_NAME}" \ + -e POSTGRES_PASSWORD=loopx_canary_secret \ + -e POSTGRES_DB=loopx_canary \ + -p "${PUBLISH_SPEC}" \ + postgres:16-alpine >/dev/null + CONTAINER_STARTED=true + + PG_PORT="$(docker port "${CONTAINER_NAME}" 5432/tcp | head -n 1 | awk -F: '{print $NF}')" + if [ -z "${PG_PORT}" ]; then + echo "Error: Failed to resolve published host port for ${CONTAINER_NAME}." >&2 + exit 1 + fi + + # Await PostgreSQL readiness + echo "[colima-rehearsal] Waiting for PostgreSQL readiness on 127.0.0.1:${PG_PORT}..." >&2 + PG_READY=false + for _ in {1..40}; do + if docker exec "${CONTAINER_NAME}" pg_isready -U postgres >/dev/null 2>&1; then + echo "[colima-rehearsal] PostgreSQL 16 is ready." >&2 + PG_READY=true + break + fi + sleep 0.5 + done + if [ "${PG_READY}" != true ]; then + echo "Error: PostgreSQL 16 container ${CONTAINER_NAME} failed to become ready within timeout." >&2 + exit 1 + fi + + DATABASE_URL="postgresql://postgres:loopx_canary_secret@127.0.0.1:${PG_PORT}/loopx_canary" + MODE="real_url" +fi + +cd "${REPO_ROOT}" + +if [ "${MODE}" = "real_url" ]; then + echo "[colima-rehearsal] Running dual-actor coordination rehearsal against isolated real PostgreSQL + TLS..." >&2 + node --no-warnings --experimental-sqlite --experimental-strip-types \ + "${SCRIPT_DIR}/colima-pg16-e2e-rehearsal.ts" \ + --database-url "${DATABASE_URL}" +else + echo "[colima-rehearsal] Running dual-actor coordination rehearsal in in-memory simulation + TLS mode..." >&2 + node --no-warnings --experimental-sqlite --experimental-strip-types \ + "${SCRIPT_DIR}/colima-pg16-e2e-rehearsal.ts" \ + --simulate +fi diff --git a/examples/coordination/colima-pg16-e2e-rehearsal.ts b/examples/coordination/colima-pg16-e2e-rehearsal.ts new file mode 100644 index 0000000000..79de52ca31 --- /dev/null +++ b/examples/coordination/colima-pg16-e2e-rehearsal.ts @@ -0,0 +1,931 @@ +/** + * Colima / Docker PostgreSQL 16 Dual-Actor Coordination Rehearsal Suite + * + * Validates: + * 1. Schema DDL initialization and least-privilege restricted role assembly. + * 2. Genesis seed sanitization and ingestion via exportLocalToPgGenesisSeed. + * 3. Dual-actor lease competition between operator.local and worker.cloud over HTTPS/TLS. + * 4. Active lease conflict prevention (actor B rejected while actor A holds valid lease). + * 5. Default-deny security boundary (unauthorized goal and invalid bearer credential rejected). + * 6. Controlled-clock expired lease takeover + lease_epoch increment + stale-actor fencing. + * 7. Verified task progress, completion, and historical recover readback over HTTPS/TLS. + * 8. Cross-machine snapshot and direct restricted-role PostgreSQL authority readback. + */ + +import {execFile} from "node:child_process"; +import {createHash, randomBytes} from "node:crypto"; +import {mkdtemp, readFile, rm, writeFile} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import {parseArgs, promisify} from "node:util"; +import {Pool, type PoolClient, type PoolConfig} from "pg"; +import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; +import { + coordinationTodoReadModel, +} from "../../loopx/control_plane/coordination/coordination_projection.ts"; +import { + canonicalTodoDomainRecord, + TODO_DOMAIN_ITEM_SCHEMA, + TODO_DOMAIN_READ_RECORD_SCHEMA, +} from "../../loopx/control_plane/coordination/coordination_state_contract.ts"; +import { + openLocalAuthorityStore, +} from "../../loopx/control_plane/coordination/local_authority_provider.ts"; +import { + exportLocalToPgGenesisSeed, + setupPostgreSqlAuthoritySchemaAndRole, +} from "../../loopx/control_plane/coordination/postgresql_authority_seed.ts"; +import { + POSTGRESQL_SCHEMA_VERSION, + PostgreSqlAuthorityStore, + type PostgreSqlAuthorityConnection, + type PostgreSqlAuthorityDatabase, +} from "../../loopx/control_plane/coordination/postgresql_authority_store.ts"; +import { + decodeRemoteAuthorityRequest, + type RemoteAuthorityRequest, +} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; +import { + callRemoteAuthority, + createRemoteAuthorityServer, +} from "../../loopx/control_plane/coordination/remote_authority_http.ts"; +import { + fileRemoteAuthorityAdmission, + REMOTE_AUTHORITY_POLICY, +} from "../../loopx/control_plane/coordination/remote_authority_policy.ts"; +import { + RemoteAuthorityService, +} from "../../loopx/control_plane/coordination/remote_authority_service.ts"; +import { + materializeTodoRecordForSchema, +} from "../../loopx/control_plane/coordination/todo_presentation.ts"; + +export interface RehearsalStepResult { + step: string; + ok: boolean; + duration_ms: number; + detail: Record; +} + +export interface RehearsalReport { + schema_version: "colima_pg16_coordination_rehearsal_report_v0"; + ok: boolean; + mode: "real_pg" | "simulation"; + execution_boundary: "isolated_real_pg16_tls" | "in_memory_simulated_pg_tls"; + transport: "https_tls"; + clock_mode: "controlled_service_clock"; + runtime_role_verified: boolean; + tenant_id: string; + goal_id: string; + store_identity: string; + total_duration_ms: number; + actors: { + actor_a: string; + actor_b: string; + }; + steps: RehearsalStepResult[]; + error?: string; +} + +export interface RehearsalOptions { + databaseUrl?: string; + simulate?: boolean; + verbose?: boolean; + leaseTtlSeconds?: number; +} + +function asJsonObject(value: unknown): JsonObject | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonObject) + : undefined; +} + +function databaseFromPool(pool: Pool): PostgreSqlAuthorityDatabase { + return { + connect: async () => { + const client: PoolClient = await pool.connect(); + const connection: PostgreSqlAuthorityConnection = { + query: async (text, values) => + await client.query(text, values ? [...values] : undefined), + release: (error) => client.release(error), + }; + return connection; + }, + }; +} + +export class ColimaPgCoordinationRehearsal { + private readonly tenantId = "tenant-canary"; + private readonly goalId = "loopx-quality-review"; + private readonly actorA = "operator.local"; + private readonly actorB = "worker.cloud"; + private readonly tokenA = `rehearsal-local-${"a".repeat(32)}`; + private readonly tokenB = `rehearsal-cloud-${"b".repeat(32)}`; + private readonly invalidToken = `invalid-rehearsal-${"x".repeat(32)}`; + private readonly todoId = "todo_canary_001"; + private readonly options: RehearsalOptions; + private serviceClock: Date; + + constructor(options: RehearsalOptions = {}) { + this.options = options; + this.serviceClock = new Date(); + } + + async run(): Promise { + const startTime = Date.now(); + const steps: RehearsalStepResult[] = []; + const storeIdentity = `postgresql:${randomBytes(16).toString("hex").toLowerCase()}`; + const isRealPg = Boolean(this.options.databaseUrl && !this.options.simulate); + const mode: RehearsalReport["mode"] = isRealPg ? "real_pg" : "simulation"; + const executionBoundary: RehearsalReport["execution_boundary"] = isRealPg + ? "isolated_real_pg16_tls" + : "in_memory_simulated_pg_tls"; + + const disconnects = new Map>>(); + const trackedPool = (config: PoolConfig): Pool => { + const created = new Pool(config); + const pending = new Set>(); + disconnects.set(created, pending); + created.on("connect", client => { + const closed = new Promise(resolve => client.once("end", resolve)); + pending.add(closed); + void closed.then(() => pending.delete(closed)); + }); + return created; + }; + const closePool = async (target?: Pool | null): Promise => { + if (!target) return; + await target.end(); + await Promise.all(disconnects.get(target) ?? []); + }; + + const tempRoots: string[] = []; + let adminPool: Pool | null = null; + let setupPool: Pool | null = null; + let runtimePool: Pool | null = null; + let databaseCreated = false; + let roleCreated = false; + let dbName = ""; + let roleName = ""; + let runtimeRoleVerified = false; + let listener: ReturnType | null = null; + + try { + const suffix = randomBytes(8).toString("hex"); + dbName = `rehearsal_${suffix}`; + roleName = `loopx_app_${suffix}`; + const rolePassword = randomBytes(24).toString("hex"); + + let setupDatabase: PostgreSqlAuthorityDatabase; + let runtimeDatabase: PostgreSqlAuthorityDatabase; + + if (isRealPg) { + const baseUrl = this.options.databaseUrl!; + adminPool = trackedPool({ + connectionString: baseUrl, + max: 2, + connectionTimeoutMillis: 5000, + statement_timeout: 10000, + }); + await adminPool.query(`CREATE DATABASE ${dbName}`); + databaseCreated = true; + + const dbUrl = new URL(baseUrl); + dbUrl.pathname = `/${dbName}`; + setupPool = trackedPool({ + connectionString: dbUrl.href, + max: 8, + connectionTimeoutMillis: 5000, + statement_timeout: 10000, + }); + await setupPool.query( + `CREATE ROLE ${roleName} LOGIN PASSWORD '${rolePassword}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT`, + ); + roleCreated = true; + await setupPool.query(`GRANT CONNECT ON DATABASE ${dbName} TO ${roleName}`); + setupDatabase = databaseFromPool(setupPool); + + const runtimeUrl = new URL(dbUrl); + runtimeUrl.username = roleName; + runtimeUrl.password = rolePassword; + runtimePool = trackedPool({ + connectionString: runtimeUrl.href, + max: 8, + connectionTimeoutMillis: 5000, + statement_timeout: 10000, + }); + runtimeDatabase = databaseFromPool(runtimePool); + } else { + setupDatabase = this.createSimulatedDatabase(storeIdentity); + runtimeDatabase = setupDatabase; + } + + // Step 1: Schema initialization and restricted role assembly (admin/setup connection only) + const s1Start = Date.now(); + const setupResult = await setupPostgreSqlAuthoritySchemaAndRole({ + database: setupDatabase, + storeIdentity, + runtimeRole: roleName, + }); + const s1Ok = setupResult.status === "installed" && setupResult.runtime_role === roleName; + steps.push({ + step: "schema_initialization_and_role_assembly", + ok: s1Ok, + duration_ms: Date.now() - s1Start, + detail: { + status: setupResult.status, + store_identity: setupResult.store_identity, + runtime_role: setupResult.runtime_role, + mode, + }, + }); + if (!s1Ok) { + throw new Error(`Schema setup failed: ${setupResult.reason ?? "unknown"}`); + } + + // Step 2: Genesis seed sanitization & ingestion via real exportLocalToPgGenesisSeed + const s2Start = Date.now(); + const seedRuntimeRoot = await mkdtemp(join(tmpdir(), "loopx-pg-rehearsal-seed-")); + tempRoots.push(seedRuntimeRoot); + const localStore = await openLocalAuthorityStore(seedRuntimeRoot, this.goalId); + const todoDomain = canonicalTodoDomainRecord({ + schema_version: TODO_DOMAIN_ITEM_SCHEMA, + todo_id: this.todoId, + role: "agent", + status: "open", + done: false, + archive_state: "active", + text: "[Canary] Verify cross-machine PG lease coordination", + priority: "P1", + task_class: "advancement_task", + required_write_scopes: [`${this.todoId}/**`], + }, "canary rehearsal todo domain record"); + const canaryTodo = materializeTodoRecordForSchema( + todoDomain, + TODO_DOMAIN_READ_RECORD_SCHEMA, + "canary rehearsal todo", + ); + const localInit = await localStore.commitAuthority({ + operation_id: `local-seed-prep-${randomBytes(6).toString("hex")}`, + expected_provider_revision: null, + next_projection: { + goal_id: this.goalId, + handoff_mode: "hard_lease", + todos: [canaryTodo], + leases: [], + todo_read_model: coordinationTodoReadModel([canaryTodo], TODO_DOMAIN_READ_RECORD_SCHEMA), + }, + events: [], + receipts: [], + }); + if (localInit.status !== "applied") { + throw new Error(`Local synthetic seed store initialization failed: ${JSON.stringify(localInit)}`); + } + const seedResult = await exportLocalToPgGenesisSeed({ + runtimeRoot: seedRuntimeRoot, + goalId: this.goalId, + tenantId: this.tenantId, + targetDatabase: setupDatabase, + storeIdentity, + execute: true, + }); + const s2Ok = seedResult.status === "seeded" && seedResult.cursor === "1"; + steps.push({ + step: "genesis_seed_ingestion", + ok: s2Ok, + duration_ms: Date.now() - s2Start, + detail: { + status: seedResult.status, + provider_revision: seedResult.provider_revision, + cursor: seedResult.cursor, + seed_digest: seedResult.seed_digest, + todos_sanitized: seedResult.sanitization_report?.todos_sanitized ?? 0, + }, + }); + if (!s2Ok) { + throw new Error(`Genesis seed ingestion failed: ${JSON.stringify(seedResult)}`); + } + + // Close setup/admin provisioning pool before starting runtime service. + if (setupPool) { + await closePool(setupPool); + setupPool = null; + } + + if (isRealPg && runtimePool) { + const roleCheck = await runtimePool.query( + `SELECT current_user AS current_user, r.rolsuper, r.rolbypassrls + FROM pg_roles AS r + WHERE r.rolname = current_user`, + ); + const row = asJsonObject(roleCheck.rows[0]); + if (!row || row.current_user !== roleName || row.rolsuper !== false || row.rolbypassrls !== false) { + throw new Error(`Runtime connection is not using least-privilege role ${roleName}: ${JSON.stringify(row)}`); + } + let appendOnlyDenied = false; + try { + await runtimePool.query("DELETE FROM loopx_control_plane.authority_commits"); + } catch (err) { + const pgErr = err as {code?: string}; + if (pgErr.code === "42501") { + appendOnlyDenied = true; + } + } + if (!appendOnlyDenied) { + throw new Error("Restricted runtime role unexpectedly allowed DELETE on authority_commits"); + } + runtimeRoleVerified = true; + } + + // Prepare self-signed localhost TLS certificate & policy file for HTTPS client/server transport + const tlsRoot = await mkdtemp(join(tmpdir(), "loopx-pg-rehearsal-tls-")); + tempRoots.push(tlsRoot); + const keyPath = join(tlsRoot, "key.pem"); + const certPath = join(tlsRoot, "cert.pem"); + await promisify(execFile)("openssl", [ + "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=localhost", + "-addext", "subjectAltName=DNS:localhost,IP:127.0.0.1", + "-keyout", keyPath, + "-out", certPath, + ]); + const cert = await readFile(certPath); + const key = await readFile(keyPath); + + const policyPath = join(tlsRoot, "policy.json"); + await writeFile(policyPath, JSON.stringify({ + schema_version: REMOTE_AUTHORITY_POLICY, + scopes: [ + { + tenant_id: this.tenantId, + goal_id: this.goalId, + store_identity: storeIdentity, + registered_agents: [this.actorA, this.actorB], + }, + ], + principals: [ + { + principal_id: `principal-${this.actorA}`, + credential_sha256: createHash("sha256").update(this.tokenA).digest("hex"), + grants: [ + { + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorA, + operations: [ + "read", "claim", "progress", "renew", "release", + "complete", "block", "defer", "todo_add", "snapshot", + ], + }, + ], + }, + { + principal_id: `principal-${this.actorB}`, + credential_sha256: createHash("sha256").update(this.tokenB).digest("hex"), + grants: [ + { + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorB, + operations: [ + "read", "claim", "progress", "renew", "release", + "complete", "block", "defer", "todo_add", "snapshot", + ], + }, + ], + }, + ], + }), {mode: 0o600}); + + const service = new RemoteAuthorityService({ + database: runtimeDatabase, + admit: fileRemoteAuthorityAdmission(policyPath), + now: () => this.serviceClock, + }); + listener = createRemoteAuthorityServer(service, {cert, key}); + const port = await listener.listen("127.0.0.1", 0); + const endpoint = `https://127.0.0.1:${port}`; + const callTls = (credential: string, request: RemoteAuthorityRequest, callMode: "execute" | "recover" = "execute") => + callRemoteAuthority(endpoint, credential, request, {ca: cert, mode: callMode}); + + // Step 3: Actor A claims lease over HTTPS/TLS + const s3Start = Date.now(); + const claimAId = `claim-a-${randomBytes(6).toString("hex")}`; + const claimAReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorA, + store_identity: storeIdentity, + operation: "claim", + request_id: claimAId, + todo_id: this.todoId, + }); + const claimARes = await callTls(this.tokenA, claimAReq); + const claimAResult = asJsonObject(claimARes.result); + const claimALease = asJsonObject(claimAResult?.lease); + const expiresAt = typeof claimALease?.expires_at === "string" ? claimALease.expires_at : ""; + const epochA = typeof claimALease?.lease_epoch === "number" ? claimALease.lease_epoch : Number.NaN; + const s3Ok = claimARes.status === "result" && + claimAResult?.status === "applied" && + claimAResult?.claimed_by === this.actorA && + Boolean(expiresAt) && + Number.isFinite(epochA); + steps.push({ + step: "actor_a_lease_claim", + ok: s3Ok, + duration_ms: Date.now() - s3Start, + detail: { + status: claimARes.status, + result_status: claimAResult?.status, + claimed_by: claimAResult?.claimed_by, + lease_epoch: epochA, + expires_at: expiresAt, + transport: "https_tls", + }, + }); + if (!s3Ok) { + throw new Error(`Actor A claim failed: ${JSON.stringify(claimARes)}`); + } + + // Step 4: Actor B attempts to claim while lease is active (must reject conflict) + const s4Start = Date.now(); + const claimBConflictReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorB, + store_identity: storeIdentity, + operation: "claim", + request_id: `claim-b-conflict-${randomBytes(6).toString("hex")}`, + todo_id: this.todoId, + }); + const claimBConflictRes = await callTls(this.tokenB, claimBConflictReq); + const claimBResult = asJsonObject(claimBConflictRes.result); + const s4Ok = claimBConflictRes.status === "result" && + (claimBResult?.status === "rejected" || claimBResult?.status === "failed"); + steps.push({ + step: "active_lease_conflict_rejection", + ok: s4Ok, + duration_ms: Date.now() - s4Start, + detail: { + status: claimBConflictRes.status, + result_status: claimBResult?.status, + reason_code: claimBResult?.reason_code, + }, + }); + + // Step 5: Default-deny security boundary enforcement over HTTPS/TLS + const s5Start = Date.now(); + const unauthGoalReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: "ungranted-canary-goal", + actor_id: this.actorB, + store_identity: storeIdentity, + operation: "read", + todo_id: this.todoId, + }); + const unauthGoalRes = await callTls(this.tokenB, unauthGoalReq); + const unauthCredRes = await callTls(this.invalidToken, claimAReq); + + const s5Ok = unauthGoalRes.status === "rejected" && + unauthCredRes.status === "rejected" && + unauthCredRes.reason_code === "principal_unauthenticated"; + steps.push({ + step: "default_deny_boundary_enforcement", + ok: s5Ok, + duration_ms: Date.now() - s5Start, + detail: { + unauthorized_goal_status: unauthGoalRes.status, + unauthorized_goal_reason: unauthGoalRes.reason_code, + unauthorized_cred_status: unauthCredRes.status, + unauthorized_cred_reason: unauthCredRes.reason_code, + }, + }); + + // Step 6: Advance controlled service clock past observed expires_at -> Actor B takeover + stale Actor A fencing + const s6Start = Date.now(); + const expiresAtMs = Date.parse(expiresAt); + if (!Number.isFinite(expiresAtMs)) { + throw new Error(`Invalid observed expires_at timestamp: ${expiresAt}`); + } + this.serviceClock = new Date(expiresAtMs + 1_000); + const advancedClockIso = this.serviceClock.toISOString(); + + const takeoverId = `claim-b-takeover-${randomBytes(6).toString("hex")}`; + const takeoverReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorB, + store_identity: storeIdentity, + operation: "claim", + request_id: takeoverId, + todo_id: this.todoId, + }); + const takeoverRes = await callTls(this.tokenB, takeoverReq); + const takeoverResult = asJsonObject(takeoverRes.result); + const takeoverLease = asJsonObject(takeoverResult?.lease); + const newEpoch = typeof takeoverLease?.lease_epoch === "number" + ? takeoverLease.lease_epoch + : Number.NaN; + + const staleProgressReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorA, + store_identity: storeIdentity, + operation: "progress", + request_id: `stale-prog-a-${randomBytes(6).toString("hex")}`, + claim_request_id: claimAId, + todo_id: this.todoId, + note: "Stale actor A progress must be rejected after takeover", + }); + const staleProgressRes = await callTls(this.tokenA, staleProgressReq); + const staleProgressResult = asJsonObject(staleProgressRes.result); + const staleActorFenced = staleProgressRes.status === "result" && + (staleProgressResult?.status === "failed" || staleProgressResult?.status === "rejected"); + + const s6Ok = takeoverRes.status === "result" && + takeoverResult?.status === "applied" && + takeoverResult?.claimed_by === this.actorB && + newEpoch === epochA + 1 && + staleActorFenced; + steps.push({ + step: "lease_expiration_takeover", + ok: s6Ok, + duration_ms: Date.now() - s6Start, + detail: { + status: takeoverRes.status, + result_status: takeoverResult?.status, + claimed_by: takeoverResult?.claimed_by, + observed_expires_at: expiresAt, + advanced_service_clock: advancedClockIso, + previous_lease_epoch: epochA, + new_lease_epoch: newEpoch, + stale_actor_fenced: staleActorFenced, + stale_actor_result_status: staleProgressResult?.status, + stale_actor_reason_code: staleProgressResult?.reason_code, + }, + }); + + // Step 7: Actor B reports progress, completes task, and verifies recover receipt over HTTPS/TLS + const s7Start = Date.now(); + const progressReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorB, + store_identity: storeIdentity, + operation: "progress", + request_id: `prog-b-${randomBytes(6).toString("hex")}`, + claim_request_id: takeoverId, + todo_id: this.todoId, + note: "Colima PG16 dual-actor TLS rehearsal in progress", + }); + const progressRes = await callTls(this.tokenB, progressReq); + const progressResult = asJsonObject(progressRes.result); + + const completeReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorB, + store_identity: storeIdentity, + operation: "complete", + request_id: `comp-b-${randomBytes(6).toString("hex")}`, + claim_request_id: takeoverId, + todo_id: this.todoId, + evidence: "canary-rehearsal:pg16-tls-dual-actor-pass", + note: "Verified dual-actor TLS lease coordination and task completion", + no_followup: true, + }); + const completeRes = await callTls(this.tokenB, completeReq); + const compResult = asJsonObject(completeRes.result); + const recoverRes = await callTls(this.tokenB, completeReq, "recover"); + + const s7Ok = progressRes.status === "result" && + progressResult?.status === "applied" && + completeRes.status === "result" && + compResult?.status === "applied" && + compResult?.completed === true && + recoverRes.status === "found" && + recoverRes.historical_only === true; + steps.push({ + step: "actor_b_task_completion", + ok: s7Ok, + duration_ms: Date.now() - s7Start, + detail: { + progress_status: progressRes.status, + progress_result_status: progressResult?.status, + complete_status: completeRes.status, + result_status: compResult?.status, + completed: compResult?.completed, + recover_status: recoverRes.status, + historical_only: recoverRes.historical_only, + }, + }); + + // Step 8: Cross-machine snapshot, read, and direct restricted-role PostgreSQL authority readback + const s8Start = Date.now(); + const snapshotReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorA, + store_identity: storeIdentity, + operation: "snapshot", + }); + const snapshotRes = await callTls(this.tokenA, snapshotReq); + const readReq = decodeRemoteAuthorityRequest({ + schema_version: "loopx_remote_authority_request_v0", + tenant_id: this.tenantId, + goal_id: this.goalId, + actor_id: this.actorA, + store_identity: storeIdentity, + operation: "read", + todo_id: this.todoId, + }); + const readRes = await callTls(this.tokenA, readReq); + const readTodo = asJsonObject(readRes.todo); + const directHead = await new PostgreSqlAuthorityStore(runtimeDatabase, { + tenant_id: this.tenantId, + goal_id: this.goalId, + expected_store_identity: storeIdentity, + }).loadAuthority(); + const todos = Array.isArray(snapshotRes.todos) ? (snapshotRes.todos as JsonObject[]) : []; + const s8Ok = snapshotRes.status === "snapshot" && + typeof snapshotRes.receipt_sha256 === "string" && + snapshotRes.receipt_sha256.length === 64 && + todos.some(t => t.todo_id === this.todoId && t.status === "done") && + readRes.status === "found" && + readTodo?.status === "done" && + directHead.status === "loaded" && + directHead.cursor === "5"; + steps.push({ + step: "cross_machine_snapshot_readback", + ok: s8Ok, + duration_ms: Date.now() - s8Start, + detail: { + status: snapshotRes.status, + receipt_sha256: snapshotRes.receipt_sha256, + todo_count: todos.length, + read_status: readRes.status, + direct_cursor: directHead.status === "loaded" ? directHead.cursor : null, + }, + }); + + const allOk = steps.every(s => s.ok) && (!isRealPg || runtimeRoleVerified); + return { + schema_version: "colima_pg16_coordination_rehearsal_report_v0", + ok: allOk, + mode, + execution_boundary: executionBoundary, + transport: "https_tls", + clock_mode: "controlled_service_clock", + runtime_role_verified: runtimeRoleVerified, + tenant_id: this.tenantId, + goal_id: this.goalId, + store_identity: storeIdentity, + total_duration_ms: Date.now() - startTime, + actors: { + actor_a: this.actorA, + actor_b: this.actorB, + }, + steps, + }; + } catch (error) { + return { + schema_version: "colima_pg16_coordination_rehearsal_report_v0", + ok: false, + mode, + execution_boundary: executionBoundary, + transport: "https_tls", + clock_mode: "controlled_service_clock", + runtime_role_verified: runtimeRoleVerified, + tenant_id: this.tenantId, + goal_id: this.goalId, + store_identity: storeIdentity, + total_duration_ms: Date.now() - startTime, + actors: { + actor_a: this.actorA, + actor_b: this.actorB, + }, + steps, + error: error instanceof Error ? error.message : String(error), + }; + } finally { + if (listener) { + await listener.close(); + } + await closePool(runtimePool); + await closePool(setupPool); + if (adminPool) { + try { + if (databaseCreated) await adminPool.query(`DROP DATABASE IF EXISTS ${dbName}`); + if (roleCreated) await adminPool.query(`DROP ROLE IF EXISTS ${roleName}`); + } finally { + await closePool(adminPool); + } + } + for (const dir of tempRoots) { + await rm(dir, {recursive: true, force: true}); + } + } + } + + private createSimulatedDatabase(storeIdentity: string): PostgreSqlAuthorityDatabase { + interface SimulatedCommit { + tenant_id: string; + goal_id: string; + cursor: number; + provider_revision: string; + operation_id: string; + } + interface SimulatedReceipt { + tenant_id: string; + goal_id: string; + cursor: number; + receipt_index: number; + receipt: unknown; + } + + const heads = new Map(); + const commits: SimulatedCommit[] = []; + const receipts: SimulatedReceipt[] = []; + + return { + connect: async () => { + let activeTenantId = "default"; + return { + query: async (sql: string, args?: readonly unknown[]) => { + const s = sql.trim(); + if ( + s.startsWith("BEGIN") || s.startsWith("COMMIT") || s.startsWith("ROLLBACK") || + s.startsWith("CREATE") || s.startsWith("ALTER") || s.startsWith("GRANT") || + s.startsWith("DO") || s.startsWith("DROP POLICY") + ) { + return {rows: [], rowCount: 0}; + } + if (s.includes("set_config")) { + activeTenantId = String(args?.[0] ?? "default"); + return {rows: [{tenant_id: activeTenantId}], rowCount: 1}; + } + if (s.includes("pg_advisory_")) { + return {rows: [], rowCount: 1}; + } + if (s.includes("INSERT INTO loopx_control_plane.authority_store_metadata")) { + return {rows: [], rowCount: 1}; + } + if (s.includes("FROM loopx_control_plane.authority_store_metadata")) { + return { + rows: [{schema_version: POSTGRESQL_SCHEMA_VERSION, store_identity: storeIdentity}], + rowCount: 1, + }; + } + if (s.includes("INSERT INTO loopx_control_plane.authority_heads")) { + const tenant = String(args?.[0]); + const goal = String(args?.[1]); + const key = `${tenant}:${goal}`; + if (!heads.has(key)) { + heads.set(key, {tenant_id: tenant, goal_id: goal, provider_revision: 0, cursor: 0, head: null}); + } + return {rows: [], rowCount: 1}; + } + if (s.includes("UPDATE loopx_control_plane.authority_heads")) { + const tenant = String(args?.[0]); + const goal = String(args?.[1]); + const rev = Number(args?.[2]); + const cursor = Number(args?.[3]); + const headObj = args?.[4] + ? (typeof args[4] === "string" ? JSON.parse(args[4]) : args[4]) as JsonObject + : null; + heads.set(`${tenant}:${goal}`, { + tenant_id: tenant, + goal_id: goal, + provider_revision: rev, + cursor, + head: headObj, + }); + return {rows: [], rowCount: 1}; + } + if (s.includes("FROM loopx_control_plane.authority_heads")) { + const tenant = String(args?.[0]); + const goal = String(args?.[1]); + const head = heads.get(`${tenant}:${goal}`); + if (!head || head.head === null) { + return { + rows: head ? [{provider_revision: "0", cursor: "0", head: null}] : [], + rowCount: head ? 1 : 0, + }; + } + return { + rows: [{ + provider_revision: String(head.provider_revision), + cursor: String(head.cursor), + head: head.head, + }], + rowCount: 1, + }; + } + if (s.includes("INSERT INTO loopx_control_plane.authority_checkpoints")) { + return {rows: [], rowCount: 1}; + } + if (s.includes("INSERT INTO loopx_control_plane.authority_commits")) { + commits.push({ + tenant_id: String(args?.[0]), + goal_id: String(args?.[1]), + cursor: Number(args?.[2]), + provider_revision: String(args?.[3]), + operation_id: String(args?.[4]), + }); + return {rows: [], rowCount: 1}; + } + if (s.includes("INSERT INTO loopx_control_plane.authority_receipts")) { + receipts.push({ + tenant_id: String(args?.[0]), + goal_id: String(args?.[1]), + cursor: Number(args?.[2]), + receipt_index: Number(args?.[3]), + receipt: args?.[4] ? (typeof args[4] === "string" ? JSON.parse(args[4]) : args[4]) : {}, + }); + return {rows: [], rowCount: 1}; + } + if (s.includes("INSERT INTO loopx_control_plane.authority_events")) { + return {rows: [], rowCount: 1}; + } + if (s.includes("FROM loopx_control_plane.authority_commits AS commit") && s.includes("commit.operation_id =")) { + const tenant = String(args?.[0]); + const goal = String(args?.[1]); + const opId = String(args?.[2]); + const match = commits.find(c => c.tenant_id === tenant && c.goal_id === goal && c.operation_id === opId); + if (!match) return {rows: [], rowCount: 0}; + const matchedReceipts = receipts + .filter(r => r.tenant_id === tenant && r.goal_id === goal && r.cursor === match.cursor) + .sort((a, b) => a.receipt_index - b.receipt_index) + .map(r => r.receipt); + return { + rows: [{ + cursor: String(match.cursor), + provider_revision: match.provider_revision, + operation_id: match.operation_id, + receipts: matchedReceipts, + }], + rowCount: 1, + }; + } + if (s.includes("FROM loopx_control_plane.authority_commits") && s.includes("operation_id =")) { + const tenant = String(args?.[0]); + const goal = String(args?.[1]); + const opId = String(args?.[2]); + const match = commits.find(c => c.tenant_id === tenant && c.goal_id === goal && c.operation_id === opId); + if (!match) return {rows: [], rowCount: 0}; + return { + rows: [{ + cursor: String(match.cursor), + provider_revision: match.provider_revision, + }], + rowCount: 1, + }; + } + return {rows: [], rowCount: 0}; + }, + release: () => {}, + }; + }, + }; + } +} + +// CLI entrypoint +if (import.meta.url === `file://${process.argv[1]}`) { + const {values} = parseArgs({ + options: { + "database-url": {type: "string"}, + simulate: {type: "boolean", default: false}, + verbose: {type: "boolean", default: false}, + "lease-ttl": {type: "string", default: "2"}, + }, + }); + + const rehearsal = new ColimaPgCoordinationRehearsal({ + databaseUrl: values["database-url"] || process.env.LOOPX_AUTHORITY_DATABASE_URL, + simulate: values.simulate || !(values["database-url"] || process.env.LOOPX_AUTHORITY_DATABASE_URL), + verbose: values.verbose, + leaseTtlSeconds: Number(values["lease-ttl"] || "2"), + }); + + const report = await rehearsal.run(); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + if (!report.ok) { + process.exit(1); + } +} diff --git a/examples/coordination/volcengine/gateway-caddy-tls.Caddyfile b/examples/coordination/volcengine/gateway-caddy-tls.Caddyfile new file mode 100644 index 0000000000..04ed8f0afa --- /dev/null +++ b/examples/coordination/volcengine/gateway-caddy-tls.Caddyfile @@ -0,0 +1,51 @@ +# ============================================================================== +# LoopX Remote Authority TLS Gateway - Caddy Configuration +# Reverse proxy terminating HTTPS and forwarding to local Node authority service +# ============================================================================== + +{ + admin off + auto_https disable_redirects + log { + output file /var/log/caddy/loopx_authority_access.log { + roll_size 50MiB + roll_keep 5 + } + format json + } +} + +:443, gateway.loopx.internal { + tls /etc/ssl/loopx/cert.pem /etc/ssl/loopx/key.pem { + protocols tls1.3 + } + + # Security headers + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + Cache-Control "no-store, no-cache, must-revalidate" + } + + # Limit request body to prevent memory pressure on authority service (16KB) + request_body { + max_size 16384 + } + + # Reverse proxy to loopx remote-authority-service + reverse_proxy 127.0.0.1:8443 { + transport http { + tls_insecure_skip_verify + dial_timeout 3s + response_header_timeout 15s + } + + # Pass client identification + header_up Host {host} + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} + header_up X-Forwarded-Proto https + } +} diff --git a/examples/coordination/volcengine/least-privilege-iam-policy.json b/examples/coordination/volcengine/least-privilege-iam-policy.json new file mode 100644 index 0000000000..d54622be83 --- /dev/null +++ b/examples/coordination/volcengine/least-privilege-iam-policy.json @@ -0,0 +1,19 @@ +{ + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "rds_postgresql:DescribeDBInstances", + "rds_postgresql:DescribeDBInstanceDetail", + "rds_postgresql:DescribeDatabases", + "rds_postgresql:DescribeAccounts", + "vpc:DescribeVpcs", + "vpc:DescribeSubnets", + "vpc:DescribeSecurityGroups", + "ecs:DescribeInstances", + "ecs:DescribeInstanceStatus" + ], + "Resource": ["*"] + } + ] +} diff --git a/examples/coordination/volcengine/main.tf b/examples/coordination/volcengine/main.tf new file mode 100644 index 0000000000..6691ec9047 --- /dev/null +++ b/examples/coordination/volcengine/main.tf @@ -0,0 +1,151 @@ +# ============================================================================== +# LoopX Volcengine cn-beijing Coordination & Gateway Infrastructure Template +# +# WARNING: STRICT HUMAN CONFIRMATION REQUIRED +# Do NOT run `terraform apply` automatically without explicit user authorization. +# ============================================================================== + +terraform { + required_version = ">= 1.5.0" + required_providers { + volcengine = { + source = "volcengine/volcengine" + version = "~> 0.1.20" + } + } +} + +variable "region" { + type = string + default = "cn-beijing" + description = "Volcengine region for coordination infrastructure" +} + +variable "availability_zone" { + type = string + default = "cn-beijing-a" + description = "Volcengine availability zone" +} + +variable "vpc_cidr" { + type = string + default = "172.16.0.0/16" + description = "VPC CIDR block" +} + +variable "subnet_cidr" { + type = string + default = "172.16.1.0/24" + description = "Subnet CIDR block for coordination services" +} + +variable "db_instance_type" { + type = string + default = "rds.pg.s1.small" + description = "RDS for PostgreSQL 16 instance specification (1C 2GB)" +} + +variable "db_storage_gb" { + type = number + default = 20 + description = "Allocated storage size in GB" +} + +variable "operator_allowlist_ips" { + type = list(string) + default = [] + description = "List of authorized operator IP CIDRs allowed to connect to Gateway port 443" +} + +provider "volcengine" { + region = var.region +} + +# 1. Virtual Private Cloud (VPC) +resource "volcengine_vpc" "coordination_vpc" { + vpc_name = "loopx-coordination-vpc-beijing" + cidr_block = var.vpc_cidr + lifecycle { + prevent_destroy = true + } +} + +# 2. Isolated Subnet +resource "volcengine_subnet" "coordination_subnet" { + subnet_name = "loopx-coordination-subnet-a" + vpc_id = volcengine_vpc.coordination_vpc.id + cidr_block = var.subnet_cidr + zone_id = var.availability_zone + lifecycle { + prevent_destroy = true + } +} + +# 3. Security Group for Remote Authority Gateway (ECS) +resource "volcengine_security_group" "gateway_sg" { + security_group_name = "loopx-gateway-sg" + vpc_id = volcengine_vpc.coordination_vpc.id + description = "Security group for LoopX Remote Authority TLS Gateway" +} + +# Ingress: Allow TLS 443 from authorized operator CIDRs only +resource "volcengine_security_group_rule" "gateway_ingress_https" { + count = length(var.operator_allowlist_ips) > 0 ? length(var.operator_allowlist_ips) : 1 + security_group_id = volcengine_security_group.gateway_sg.id + direction = "ingress" + protocol = "tcp" + port_start = 443 + port_end = 443 + cidr_ip = length(var.operator_allowlist_ips) > 0 ? var.operator_allowlist_ips[count.index] : var.vpc_cidr + description = "Allow inbound TLS 443" +} + +# 4. Security Group for RDS for PostgreSQL (Strict VPC internal only) +resource "volcengine_security_group" "rds_sg" { + security_group_name = "loopx-rds-internal-sg" + vpc_id = volcengine_vpc.coordination_vpc.id + description = "Strict internal security group for RDS PostgreSQL (No public ingress)" +} + +# Ingress: Port 5432 strictly from gateway security group +resource "volcengine_security_group_rule" "rds_ingress_from_gateway" { + security_group_id = volcengine_security_group.rds_sg.id + direction = "ingress" + protocol = "tcp" + port_start = 5432 + port_end = 5432 + source_group_id = volcengine_security_group.gateway_sg.id + description = "Allow PostgreSQL access from gateway only" +} + +# 5. RDS for PostgreSQL 16 Instance +resource "volcengine_rds_postgresql_instance" "coordination_db" { + db_engine_version = "PostgreSQL_16" + instance_name = "loopx-coordination-rds-pg16" + instance_type = var.db_instance_type + storage_space = var.db_storage_gb + storage_type = "LocalSSD" + vpc_id = volcengine_vpc.coordination_vpc.id + subnet_id = volcengine_subnet.coordination_subnet.id + charge_type = "PostPaid" # Pay-as-you-go + + lifecycle { + prevent_destroy = true + ignore_changes = [charge_type] + } +} + +output "vpc_id" { + value = volcengine_vpc.coordination_vpc.id + description = "Coordination VPC ID" +} + +output "subnet_id" { + value = volcengine_subnet.coordination_subnet.id + description = "Coordination Subnet ID" +} + +output "rds_endpoint" { + value = volcengine_rds_postgresql_instance.coordination_db.endpoint + description = "Internal PostgreSQL endpoint for RemoteAuthorityService connection" +} diff --git a/examples/coordination/volcengine/remote-authority-systemd.service b/examples/coordination/volcengine/remote-authority-systemd.service new file mode 100644 index 0000000000..8f35e74e18 --- /dev/null +++ b/examples/coordination/volcengine/remote-authority-systemd.service @@ -0,0 +1,32 @@ +[Unit] +Description=LoopX Remote Authority Coordination Service (Volcengine cn-beijing) +Documentation=https://github.com/loopx-project/loopx +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/opt/loopx +Environment=NODE_ENV=production +Environment=LOOPX_AUTHORITY_DATABASE_URL=postgresql://loopx_app:REPLACE_WITH_RESTRICTED_APP_PASSWORD@rds-pg-internal.cn-beijing.volces.com:5432/loopx_coordination +ExecStart=/usr/bin/node --no-warnings --experimental-strip-types examples/coordination/remote-authority-service.ts --policy /etc/loopx/remote-policy.json --cert /etc/ssl/loopx/cert.pem --key /etc/ssl/loopx/key.pem --port 8443 --host 127.0.0.1 +Restart=on-failure +RestartSec=5s + +# Security Hardening +DynamicUser=yes +User=loopx-authority +Group=loopx-authority +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/tests/control_plane_ts/coordination_colima_rehearsal.test.ts b/tests/control_plane_ts/coordination_colima_rehearsal.test.ts new file mode 100644 index 0000000000..a73650213e --- /dev/null +++ b/tests/control_plane_ts/coordination_colima_rehearsal.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + ColimaPgCoordinationRehearsal, + type RehearsalReport, +} from "../../examples/coordination/colima-pg16-e2e-rehearsal.ts"; + +function assertAllRehearsalSteps( + report: RehearsalReport, + expectedMode: RehearsalReport["mode"], + expectedBoundary: RehearsalReport["execution_boundary"], +): void { + assert.equal(report.schema_version, "colima_pg16_coordination_rehearsal_report_v0"); + assert.equal(report.ok, true, `Rehearsal failed with error: ${report.error ?? JSON.stringify(report.steps)}`); + assert.equal(report.mode, expectedMode); + assert.equal(report.execution_boundary, expectedBoundary); + assert.equal(report.transport, "https_tls"); + assert.equal(report.clock_mode, "controlled_service_clock"); + assert.equal(report.runtime_role_verified, expectedMode === "real_pg"); + assert.equal(report.tenant_id, "tenant-canary"); + assert.equal(report.goal_id, "loopx-quality-review"); + assert.equal(report.actors.actor_a, "operator.local"); + assert.equal(report.actors.actor_b, "worker.cloud"); + assert.equal(report.steps.length, 8); + + const stepsByName = new Map(report.steps.map(s => [s.step, s])); + + // Step 1: Schema initialization + const step1 = stepsByName.get("schema_initialization_and_role_assembly"); + assert.ok(step1); + assert.equal(step1.ok, true); + assert.equal(step1.detail.status, "installed"); + assert.match(String(step1.detail.runtime_role), /^loopx_app_[0-9a-f]+$/); + + // Step 2: Genesis seed ingestion + const step2 = stepsByName.get("genesis_seed_ingestion"); + assert.ok(step2); + assert.equal(step2.ok, true); + assert.equal(step2.detail.status, "seeded"); + assert.equal(step2.detail.cursor, "1"); + + // Step 3: Actor A lease claim over HTTPS/TLS + const step3 = stepsByName.get("actor_a_lease_claim"); + assert.ok(step3); + assert.equal(step3.ok, true); + assert.equal(step3.detail.result_status, "applied"); + assert.equal(step3.detail.claimed_by, "operator.local"); + assert.equal(step3.detail.transport, "https_tls"); + + // Step 4: Active lease conflict rejection + const step4 = stepsByName.get("active_lease_conflict_rejection"); + assert.ok(step4); + assert.equal(step4.ok, true); + assert.equal(step4.detail.status, "result"); + assert.ok(step4.detail.result_status === "rejected" || step4.detail.result_status === "failed"); + + // Step 5: Default-deny boundary enforcement + const step5 = stepsByName.get("default_deny_boundary_enforcement"); + assert.ok(step5); + assert.equal(step5.ok, true); + assert.equal(step5.detail.unauthorized_goal_status, "rejected"); + assert.equal(step5.detail.unauthorized_cred_status, "rejected"); + assert.equal(step5.detail.unauthorized_cred_reason, "principal_unauthenticated"); + + // Step 6: Lease expiration takeover & stale-actor fencing + const step6 = stepsByName.get("lease_expiration_takeover"); + assert.ok(step6); + assert.equal(step6.ok, true); + assert.equal(step6.detail.result_status, "applied"); + assert.equal(step6.detail.claimed_by, "worker.cloud"); + assert.equal(step6.detail.previous_lease_epoch, 1); + assert.equal(step6.detail.new_lease_epoch, 2); + assert.equal(step6.detail.stale_actor_fenced, true); + + // Step 7: Actor B task completion + historical recovery readback + const step7 = stepsByName.get("actor_b_task_completion"); + assert.ok(step7); + assert.equal(step7.ok, true); + assert.equal(step7.detail.result_status, "applied"); + assert.equal(step7.detail.completed, true); + assert.equal(step7.detail.recover_status, "found"); + assert.equal(step7.detail.historical_only, true); + + // Step 8: Cross-machine snapshot & direct authority readback + const step8 = stepsByName.get("cross_machine_snapshot_readback"); + assert.ok(step8); + assert.equal(step8.ok, true); + assert.equal(step8.detail.status, "snapshot"); + assert.equal(typeof step8.detail.receipt_sha256, "string"); + assert.equal((step8.detail.receipt_sha256 as string).length, 64); + assert.equal(step8.detail.todo_count, 1); + assert.equal(step8.detail.read_status, "found"); + assert.equal(step8.detail.direct_cursor, "5"); +} + +test("Colima/PG16 coordination rehearsal completes all 8 HTTPS/TLS steps in simulation mode", async () => { + const rehearsal = new ColimaPgCoordinationRehearsal({ + simulate: true, + leaseTtlSeconds: 1, + }); + const report = await rehearsal.run(); + assertAllRehearsalSteps(report, "simulation", "in_memory_simulated_pg_tls"); +}); + +test( + "Colima/PG16 coordination rehearsal qualifies all 8 steps against real PostgreSQL with restricted role and TLS", + {skip: !process.env.LOOPX_TEST_POSTGRES_URL}, + async () => { + const rehearsal = new ColimaPgCoordinationRehearsal({ + databaseUrl: process.env.LOOPX_TEST_POSTGRES_URL, + simulate: false, + }); + const report = await rehearsal.run(); + assertAllRehearsalSteps(report, "real_pg", "isolated_real_pg16_tls"); + }, +); diff --git a/tests/test_colima_pg_rehearsal.py b/tests/test_colima_pg_rehearsal.py new file mode 100644 index 0000000000..4f809486fb --- /dev/null +++ b/tests/test_colima_pg_rehearsal.py @@ -0,0 +1,165 @@ +"""Verification test suite for Colima / Docker PostgreSQL 16 dual-actor rehearsal + +and Volcengine cn-beijing deployment templates. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def test_rehearsal_shell_script_help() -> None: + """Verify that the shell wrapper outputs help information cleanly.""" + script_path = REPO_ROOT / "examples" / "coordination" / "colima-pg16-e2e-rehearsal.sh" + assert script_path.exists() + assert script_path.is_file() + + result = subprocess.run( + [str(script_path), "--help"], + capture_output=True, + text=True, + check=True, + ) + assert "--simulate" in result.stdout + assert "--rehearsal" in result.stdout + assert "--database-url" in result.stdout + assert "--start-colima" in result.stdout + + +def test_rehearsal_explicit_mode_fails_closed_without_docker() -> None: + """Explicit --rehearsal must fail closed rather than silently downgrading to --simulate.""" + script_path = REPO_ROOT / "examples" / "coordination" / "colima-pg16-e2e-rehearsal.sh" + env = {**os.environ, "DOCKER_HOST": "unix:///nonexistent/loopx-no-docker.sock"} + result = subprocess.run( + [str(script_path), "--rehearsal"], + capture_output=True, + text=True, + env=env, + cwd=REPO_ROOT, + ) + assert result.returncode != 0 + assert "simulation" not in result.stdout.lower() + + +def test_rehearsal_simulation_e2e() -> None: + """Verify that rehearsal runs end-to-end in simulation mode with ok: true.""" + script_path = REPO_ROOT / "examples" / "coordination" / "colima-pg16-e2e-rehearsal.sh" + result = subprocess.run( + [str(script_path), "--simulate"], + capture_output=True, + text=True, + check=True, + cwd=REPO_ROOT, + ) + data = json.loads(result.stdout) + + assert data["schema_version"] == "colima_pg16_coordination_rehearsal_report_v0" + assert data["ok"] is True + assert data["mode"] == "simulation" + assert data["execution_boundary"] == "in_memory_simulated_pg_tls" + assert data["transport"] == "https_tls" + assert data["clock_mode"] == "controlled_service_clock" + assert data["runtime_role_verified"] is False + assert data["tenant_id"] == "tenant-canary" + assert data["goal_id"] == "loopx-quality-review" + assert data["actors"]["actor_a"] == "operator.local" + assert data["actors"]["actor_b"] == "worker.cloud" + + expected_steps = [ + "schema_initialization_and_role_assembly", + "genesis_seed_ingestion", + "actor_a_lease_claim", + "active_lease_conflict_rejection", + "default_deny_boundary_enforcement", + "lease_expiration_takeover", + "actor_b_task_completion", + "cross_machine_snapshot_readback", + ] + step_names = [s["step"] for s in data["steps"]] + assert step_names == expected_steps + + for step in data["steps"]: + assert step["ok"] is True, f"Step {step['step']} failed: {step['detail']}" + + +def test_rehearsal_ts_cli_direct() -> None: + """Verify that the TypeScript harness can be directly invoked via node.""" + ts_path = REPO_ROOT / "examples" / "coordination" / "colima-pg16-e2e-rehearsal.ts" + result = subprocess.run( + [ + "node", + "--no-warnings", + "--experimental-sqlite", + "--experimental-strip-types", + str(ts_path), + "--simulate", + ], + capture_output=True, + text=True, + check=True, + cwd=REPO_ROOT, + ) + data = json.loads(result.stdout) + assert data["ok"] is True + assert data["transport"] == "https_tls" + assert len(data["steps"]) == 8 + + +def test_volcengine_deployment_artifacts() -> None: + """Verify presence and validity of all Volcengine cn-beijing deployment assets.""" + doc_path = ( + REPO_ROOT + / "docs" + / "reference" + / "deployment" + / "volcengine-cn-beijing-pg-gateway.md" + ) + assert doc_path.exists(), "Deployment reference document missing" + doc_content = doc_path.read_text(encoding="utf-8") + assert "严禁未经用户确认自动购买云资源或改线上配置" in doc_content + assert "cn-beijing" in doc_content + assert "loopx_app" in doc_content + assert "ark.cn-beijing.volces.com" in doc_content + assert "operator.local" in doc_content + assert "worker.cloud" in doc_content + + volc_dir = REPO_ROOT / "examples" / "coordination" / "volcengine" + assert volc_dir.is_dir() + + # Terraform template + tf_path = volc_dir / "main.tf" + assert tf_path.exists() + tf_content = tf_path.read_text(encoding="utf-8") + assert 'variable "region"' in tf_content + assert "volcengine_rds_postgresql_instance" in tf_content + assert "prevent_destroy = true" in tf_content + + # Caddyfile + caddy_path = volc_dir / "gateway-caddy-tls.Caddyfile" + assert caddy_path.exists() + caddy_content = caddy_path.read_text(encoding="utf-8") + assert "reverse_proxy 127.0.0.1:8443" in caddy_content + + # Systemd service + svc_path = volc_dir / "remote-authority-systemd.service" + assert svc_path.exists() + svc_content = svc_path.read_text(encoding="utf-8") + assert "DynamicUser=yes" in svc_content + assert "remote-authority-service.ts" in svc_content + + # Least privilege IAM policy + iam_path = volc_dir / "least-privilege-iam-policy.json" + assert iam_path.exists() + iam_json = json.loads(iam_path.read_text(encoding="utf-8")) + assert "Statement" in iam_json + actions = iam_json["Statement"][0]["Action"] + assert "rds_postgresql:DescribeDBInstances" in actions + # Guardrail: Mutating actions must NOT be present in worker policy + for action in actions: + assert not action.startswith("rds_postgresql:Create"), f"Mutating action {action} in policy" + assert not action.startswith("rds_postgresql:Delete"), f"Destructive action {action} in policy" diff --git a/tsconfig.control-plane.json b/tsconfig.control-plane.json index febde3aafc..b5d2277d86 100644 --- a/tsconfig.control-plane.json +++ b/tsconfig.control-plane.json @@ -31,6 +31,8 @@ "examples/coordination/sqlite-authority-migration.ts", "examples/coordination/remote-authority-service.ts", "examples/coordination/remote-authority-client.ts", + "examples/coordination/colima-pg16-e2e-rehearsal.ts", + "tests/control_plane_ts/coordination_colima_rehearsal.test.ts", "tests/control_plane_ts/remote_authority.test.ts", "tests/control_plane_ts/remote_authority.integration.test.ts", "tests/control_plane_ts/remote_authority_fixture.ts", From 9023c1c2b76235e0098cb1d5c32524d9a1a893d7 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:07:57 +0800 Subject: [PATCH 2/2] fix(coordination): align volcengine tf schema and scrub local deployment assumptions Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../volcengine-cn-beijing-pg-gateway.md | 54 +++++++++++-------- examples/coordination/volcengine/main.tf | 44 ++++++++++----- .../remote-authority-systemd.service | 4 +- .../coordination_colima_rehearsal.test.ts | 8 +++ tests/test_colima_pg_rehearsal.py | 32 ++++++++++- 5 files changed, 106 insertions(+), 36 deletions(-) diff --git a/docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md b/docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md index 153cb969c0..504c0acac9 100644 --- a/docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md +++ b/docs/reference/deployment/volcengine-cn-beijing-pg-gateway.md @@ -16,7 +16,7 @@ +-----------------------------+ +-----------------------------+ | Local Operator Machine | | Cloud Worker Machine | | operator.local | | worker.cloud | -| (macOS Apple Silicon dev) | | (Volcengine ECS / Ark LLM) | +| (Operator Workstation) | | (Volcengine ECS / Ark LLM) | +--------------+--------------+ +--------------+--------------+ | | | HTTPS / TLS 1.3 (Bearer Credential) | HTTPS / TLS 1.3 (Bearer Credential) @@ -42,8 +42,8 @@ | | | +---------------------------------------------------------------------+ | | | Volcano Ark (`ark.cn-beijing.volces.com`) Video / LLM Model Service | | -| | - 同地域低延迟内网交互 | | -| | - 复用本机已就绪凭据 (`operations-ark-video.json`) 驱动廉价 Worker | | +| | - 同地域低延迟交互(由部署者按环境参数配置端点与模型 ID) | | +| | - 通过独立受限环境变量注入凭据(不依赖开发机本地配置文件) | | | +---------------------------------------------------------------------+ | +---------------------------------------------------------------------------+ ``` @@ -52,7 +52,7 @@ 1. **网络零公网暴露数据库**:PostgreSQL 实例仅绑定 VPC 内网子网,安全组仅对网关 ECS 开放 5432 端口;严禁绑定公网弹性 IP (EIP)。 2. **最小受限角色装配 (`loopx_app`)**:网关服务使用专用的 `loopx_app` 角色连接数据库,仅具备 `SELECT, INSERT, UPDATE` 必要权限,禁止 `DROP TABLE`, `TRUNCATE`, `ALTER ROLE` 等管理员特权。 3. **端到端 TLS 传输认证**:外部客户端经由 TLS 1.3 连接网关,凭据在网关层做安全哈希对比(`timingSafeEqual`),绝不向数据库传递或持久化明文凭据。 -4. **同地域方舟模型协同**:利用同在 `cn-beijing` 的火山方舟大模型/视频 API 端点(`ark.cn-beijing.volces.com`),保证 Worker 节点在同一云内网络低延迟调用模型。 +4. **同地域方舟模型与受限凭据注入**:利用同在 `cn-beijing` 的火山方舟大模型/视频 API 端点(`ark.cn-beijing.volces.com`),由部署者在云端通过环境变量或密钥管理服务为 Worker 单独注入受限模型 API Key 与网关 Bearer 凭据,严禁复用或上传开发机本地私有配置文件。 --- @@ -65,7 +65,7 @@ | **安全组 (Gateway)** | 入方向: 仅放通 443 (TLS) 给授权 IP;出方向: 允许内网访问 RDS | 免费 | 网关流量防护 | | **安全组 (RDS)** | 入方向: 仅放通 5432 来自网关安全组;严禁任何 `0.0.0.0/0` | 免费 | 数据库访问收敛 | | **轻量网关 ECS** | 规格: `ecs.g3i.large` (2C4G) 或 `ecs.c3i.large` (2C4G) | 按量计费 / 抢占式 | 运行 Caddy + RemoteAuthorityService | -| **RDS for PG** | 引擎: PostgreSQL 16,规格: `rds.pg.s1.small` (1C2G) 或通用型 | 按量计费 (按需开启) | 承载协同权威状态与检查点 | +| **RDS for PG** | 引擎: PostgreSQL 16,规格: `rds.postgres.1c2g` (1C2G) 或通用型 | 按量计费 (按需开启) | 承载协同权威状态与检查点 | | **弹性公网 IP** | 带宽: 1-5 Mbps 按流量计费,绑定至网关 ECS | 按量计费 | 提供 HTTPS 客户端安全接入 | --- @@ -105,14 +105,15 @@ 配套部署模板位于 `examples/coordination/volcengine/` 目录: 1. **Terraform 编排模板**:`main.tf` - - 定义 VPC、子网、安全组、RDS PostgreSQL 16 实例与网关 ECS 的参数化配置。 - - 所有变量均提供安全默认值(如北京地域 `cn-beijing`),并设置 `prevent_destroy = true` 避免误删。 + - 锁定 `volcengine/volcengine` provider 已发布约束(`>= 0.0.186, < 0.1.0`,支持 `PostgreSQL_16`)。 + - 定义 VPC、子网、安全组、`volcengine_rds_postgresql_instance`(使用 `node_spec`、`primary_zone_id`、`secondary_zone_id` 与 `charge_info` 块)及 `rds_endpoints` / `rds_endpoint` 输出。 + - 所有变量均由部署者显式传入或使用安全默认值(如北京地域 `cn-beijing`、默认仅放通 VPC 内网至 443 端口),并设置 `prevent_destroy = true` 避免误删。 2. **TLS 网关配置**:`gateway-caddy-tls.Caddyfile` - Caddy 自动化 TLS 反向代理至本地 Node 权威服务(`https://127.0.0.1:8443`)。 - 包含请求大小限制(16KB)、连接超时约束与 HSTS 安全头。 3. **网关守护进程服务**:`remote-authority-systemd.service` - 以独立低特权系统用户运行 `examples/coordination/remote-authority-service.ts`。 - - 启用 `DynamicUser=yes`, `ProtectSystem=strict`, `PrivateTmp=true` 等 systemd 安全加固。 + - 通过仅 root 可读的 `EnvironmentFile=/etc/loopx/authority.env` 注入 `LOOPX_AUTHORITY_DATABASE_URL`(使用受限 `loopx_app` 角色),并启用 `DynamicUser=yes`, `ProtectSystem=strict`, `PrivateTmp=true` 等 systemd 安全加固。 --- @@ -129,32 +130,43 @@ ``` ### 步骤 2:创世种子脱敏与灌入 -对需要共享给云端的 Goal(如 `loopx-quality-review`),使用 Stage 2B 工具导出脱敏创世种子并灌入云端 RDS: +部署者按目标环境设置参数变量后,使用 Stage 2B 工具导出脱敏创世种子,并通过只读权限隔离后的初始化连接灌入目标 PostgreSQL 实例: ```bash -# 导出并校验隐私边界(自动剔除本地绝对路径、飞书私有 ID 与未过期租约) +# 由部署者提供参数与输出路径(自动剔除本地绝对路径、飞书私有 ID 与未过期租约) +export LOOPX_RUNTIME_ROOT="/path/to/operator-runtime-root" +export LOOPX_GOAL_ID="" +export LOOPX_TENANT_ID="" +export LOOPX_SEED_OUTPUT="/tmp/genesis-seed.json" + node --no-warnings --experimental-sqlite --experimental-strip-types \ -e ' import {exportLocalToPgGenesisSeed} from "./loopx/control_plane/coordination/postgresql_authority_seed.ts"; const result = await exportLocalToPgGenesisSeed({ - runtimeRoot: process.env.HOME + "/.loopx", - goalId: "loopx-quality-review", - tenantId: "tenant-prod", - outputPath: "/tmp/genesis-seed-canary.json" + runtimeRoot: process.env.LOOPX_RUNTIME_ROOT!, + goalId: process.env.LOOPX_GOAL_ID!, + tenantId: process.env.LOOPX_TENANT_ID!, + outputPath: process.env.LOOPX_SEED_OUTPUT!, }); console.log("Genesis seed export result:", result.status, result.seed_digest); ' ``` ### 步骤 3:多机客户端并发联调 -- 本机操作员 (`operator.local`): +每个客户端仅持有所属 `principal` 的受限网关 Bearer 凭据与目标 `store_identity` / `tenant_id` / `goal_id` 参数,不接触数据库管理员凭据或彼此私钥: +- **操作员节点 (`operator.local`)**: + 通过环境变量或密钥代理注入网关凭据,经 TLS CA 校验将命令 JSON 通过标准输入提交给网关: ```bash LOOPX_AUTHORITY_CREDENTIAL="" \ node --no-warnings --experimental-sqlite --experimental-strip-types \ examples/coordination/remote-authority-client.ts \ - --endpoint "https://gateway.loopx.bj.volces.com" \ - --ca /etc/ssl/certs/loopx-ca.pem + --endpoint "https://" \ + --ca "" < command.json ``` -- 云端 Worker (`worker.cloud`): - 配合 `operations-ark-video.json` 中的火山方舟 API Key,在获取租约后执行测试/评审,随后向网关回传 `progress` 与带凭证证据的 `complete` 指令。 -- 租约冲突与超期接管: - 若云端 Worker 节点因网络断连或故障未续租,租约到达 TTL 后,本地操作员可无阻塞接管继续推进,彻底消除任务悬挂风险。 +- **云端 Worker (`worker.cloud`)**: + 由部署者在云端环境独立注入以下受限凭据与参数(可通过 ECS 实例环境文件、systemd `EnvironmentFile` 或云厂商密钥托管服务挂载,权限收紧为 `0600`,不依赖任何开发机本地文件): + - `LOOPX_AUTHORITY_CREDENTIAL`:仅授权 `worker.cloud` 对应 `principal` 与目标 scope 的网关 Bearer 凭据。 + - 模型调用参数与密钥(如 `ARK_API_KEY`、`ARK_ENDPOINT=https://ark.cn-beijing.volces.com/api/v3` 与目标 `MODEL_ID`):独立签发的最小额度模型凭据,仅用于 Worker 执行分配任务。 + - `LOOPX_GATEWAY_ENDPOINT` 与 `LOOPX_GATEWAY_CA`:部署者配置的网关域名及可信 CA 路径。 + Worker 通过 `examples/coordination/remote-authority-client.ts` 获取租约并执行分配的任务,随后向网关回传 `progress` 与带验证证据的 `complete` 指令;若遇到网络超时或未知响应,须保留原请求 JSON 与 `request_id`,改用 `--recover` 只读查询回执,严禁自动重试或替换 `request_id`。 +- **租约冲突与超期接管**: + 若云端 Worker 节点因网络断连或故障未续租,租约到达 TTL 后,其他已授权节点可按服务端时钟合法接管继续推进,并在完成后由任一授权节点经 `snapshot` / `read` 回读一致终态。 diff --git a/examples/coordination/volcengine/main.tf b/examples/coordination/volcengine/main.tf index 6691ec9047..47a86cfeaa 100644 --- a/examples/coordination/volcengine/main.tf +++ b/examples/coordination/volcengine/main.tf @@ -10,7 +10,7 @@ terraform { required_providers { volcengine = { source = "volcengine/volcengine" - version = "~> 0.1.20" + version = ">= 0.0.186, < 0.1.0" } } } @@ -24,7 +24,13 @@ variable "region" { variable "availability_zone" { type = string default = "cn-beijing-a" - description = "Volcengine availability zone" + description = "Primary Volcengine availability zone" +} + +variable "secondary_availability_zone" { + type = string + default = "" + description = "Secondary Volcengine availability zone for RDS PostgreSQL (defaults to availability_zone)" } variable "vpc_cidr" { @@ -39,16 +45,20 @@ variable "subnet_cidr" { description = "Subnet CIDR block for coordination services" } -variable "db_instance_type" { +variable "db_node_spec" { type = string - default = "rds.pg.s1.small" - description = "RDS for PostgreSQL 16 instance specification (1C 2GB)" + default = "rds.postgres.1c2g" + description = "RDS for PostgreSQL 16 node specification (1C 2GB)" } variable "db_storage_gb" { type = number default = 20 - description = "Allocated storage size in GB" + description = "Allocated storage size in GB (range [20, 3000], step 10)" + validation { + condition = var.db_storage_gb >= 20 && var.db_storage_gb <= 3000 && var.db_storage_gb % 10 == 0 + error_message = "db_storage_gb must be between 20 and 3000 GB in steps of 10." + } } variable "operator_allowlist_ips" { @@ -122,16 +132,19 @@ resource "volcengine_security_group_rule" "rds_ingress_from_gateway" { resource "volcengine_rds_postgresql_instance" "coordination_db" { db_engine_version = "PostgreSQL_16" instance_name = "loopx-coordination-rds-pg16" - instance_type = var.db_instance_type + node_spec = var.db_node_spec + primary_zone_id = var.availability_zone + secondary_zone_id = var.secondary_availability_zone != "" ? var.secondary_availability_zone : var.availability_zone storage_space = var.db_storage_gb - storage_type = "LocalSSD" - vpc_id = volcengine_vpc.coordination_vpc.id subnet_id = volcengine_subnet.coordination_subnet.id - charge_type = "PostPaid" # Pay-as-you-go + + charge_info { + charge_type = "PostPaid" # Pay-as-you-go + } lifecycle { prevent_destroy = true - ignore_changes = [charge_type] + ignore_changes = [charge_info] } } @@ -145,7 +158,12 @@ output "subnet_id" { description = "Coordination Subnet ID" } +output "rds_endpoints" { + value = volcengine_rds_postgresql_instance.coordination_db.endpoints + description = "PostgreSQL endpoint records exported by the Volcengine RDS instance" +} + output "rds_endpoint" { - value = volcengine_rds_postgresql_instance.coordination_db.endpoint - description = "Internal PostgreSQL endpoint for RemoteAuthorityService connection" + value = try(volcengine_rds_postgresql_instance.coordination_db.endpoints[0].address[0].domain, "") + description = "Primary internal PostgreSQL domain endpoint for RemoteAuthorityService connection" } diff --git a/examples/coordination/volcengine/remote-authority-systemd.service b/examples/coordination/volcengine/remote-authority-systemd.service index 8f35e74e18..0f3ce3e886 100644 --- a/examples/coordination/volcengine/remote-authority-systemd.service +++ b/examples/coordination/volcengine/remote-authority-systemd.service @@ -8,7 +8,9 @@ Wants=network-online.target Type=simple WorkingDirectory=/opt/loopx Environment=NODE_ENV=production -Environment=LOOPX_AUTHORITY_DATABASE_URL=postgresql://loopx_app:REPLACE_WITH_RESTRICTED_APP_PASSWORD@rds-pg-internal.cn-beijing.volces.com:5432/loopx_coordination +# Supply LOOPX_AUTHORITY_DATABASE_URL via a 0600 root-owned env file: +# LOOPX_AUTHORITY_DATABASE_URL=postgresql://loopx_app:@:5432/ +EnvironmentFile=/etc/loopx/authority.env ExecStart=/usr/bin/node --no-warnings --experimental-strip-types examples/coordination/remote-authority-service.ts --policy /etc/loopx/remote-policy.json --cert /etc/ssl/loopx/cert.pem --key /etc/ssl/loopx/key.pem --port 8443 --host 127.0.0.1 Restart=on-failure RestartSec=5s diff --git a/tests/control_plane_ts/coordination_colima_rehearsal.test.ts b/tests/control_plane_ts/coordination_colima_rehearsal.test.ts index a73650213e..f3041745e7 100644 --- a/tests/control_plane_ts/coordination_colima_rehearsal.test.ts +++ b/tests/control_plane_ts/coordination_colima_rehearsal.test.ts @@ -71,6 +71,14 @@ function assertAllRehearsalSteps( assert.equal(step6.detail.previous_lease_epoch, 1); assert.equal(step6.detail.new_lease_epoch, 2); assert.equal(step6.detail.stale_actor_fenced, true); + // Prove takeover is driven from observed expires_at via controlled service clock rather than wall-clock sleep. + const observedExpiresMs = Date.parse(String(step3.detail.expires_at)); + const advancedClockMs = Date.parse(String(step6.detail.advanced_service_clock)); + assert.ok(Number.isFinite(observedExpiresMs)); + assert.ok(Number.isFinite(advancedClockMs)); + assert.ok(observedExpiresMs - Date.now() > 60_000, "canonical lease expires ~2700s in the future"); + assert.equal(step6.detail.observed_expires_at, step3.detail.expires_at); + assert.equal(advancedClockMs, observedExpiresMs + 1_000); // Step 7: Actor B task completion + historical recovery readback const step7 = stepsByName.get("actor_b_task_completion"); diff --git a/tests/test_colima_pg_rehearsal.py b/tests/test_colima_pg_rehearsal.py index 4f809486fb..9e7a88e769 100644 --- a/tests/test_colima_pg_rehearsal.py +++ b/tests/test_colima_pg_rehearsal.py @@ -128,15 +128,43 @@ def test_volcengine_deployment_artifacts() -> None: assert "operator.local" in doc_content assert "worker.cloud" in doc_content + # Verify public-boundary neutrality: no machine-specific or pre-existing local credential files + for forbidden in ( + "operations-ark-video.json", + "复用本机", + "macOS Apple Silicon dev", + "gateway.loopx.bj.volces.com", + ): + assert forbidden not in doc_content, f"Non-public local assumption {forbidden!r} in deployment doc" + assert "EnvironmentFile" in doc_content or "环境变量" in doc_content + assert "LOOPX_AUTHORITY_CREDENTIAL" in doc_content + assert "ARK_API_KEY" in doc_content + volc_dir = REPO_ROOT / "examples" / "coordination" / "volcengine" assert volc_dir.is_dir() - # Terraform template + # Terraform template: check valid published provider series (< 0.1.0) and real resource fields tf_path = volc_dir / "main.tf" assert tf_path.exists() tf_content = tf_path.read_text(encoding="utf-8") assert 'variable "region"' in tf_content + assert 'source = "volcengine/volcengine"' in tf_content + assert "~> 0.1.20" not in tf_content + assert ">= 0.0.186, < 0.1.0" in tf_content assert "volcengine_rds_postgresql_instance" in tf_content + assert "node_spec" in tf_content + assert "primary_zone_id" in tf_content + assert "secondary_zone_id" in tf_content + assert "charge_info" in tf_content + assert ".endpoints" in tf_content + # Guardrail: computed-only or non-schema root attributes must not appear as instance arguments + rds_block = tf_content.split('resource "volcengine_rds_postgresql_instance"')[1].split("output ")[0] + for invalid_arg in ("instance_type ", "storage_type ", "vpc_id ", "charge_type ="): + # charge_type inside charge_info block is valid, so check root-level indentation (2 spaces) + if invalid_arg == "charge_type =": + assert "\n charge_type " not in rds_block + else: + assert f"\n {invalid_arg}" not in rds_block assert "prevent_destroy = true" in tf_content # Caddyfile @@ -150,6 +178,8 @@ def test_volcengine_deployment_artifacts() -> None: assert svc_path.exists() svc_content = svc_path.read_text(encoding="utf-8") assert "DynamicUser=yes" in svc_content + assert "EnvironmentFile=/etc/loopx/authority.env" in svc_content + assert "REPLACE_WITH_RESTRICTED_APP_PASSWORD" not in svc_content assert "remote-authority-service.ts" in svc_content # Least privilege IAM policy