From 4d5bdc2e84f8f9c5d46cc87e8dedfe44b6b8343e Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:30:59 +0800 Subject: [PATCH 1/5] feat(coordination): add complete remote task lifecycle and opt-in gateway access (#8747) Align with PR #8747 (multi-loopx-agent-cloud-collaboration-v0) and remote-authority-task-service-v0 (#8508): 1. Extend RemoteAuthorityService with full remote task settlement: - Add 'complete', 'block', 'defer', 'todo_add', and 'snapshot' operations. - Enforce hard-lease fencing and claim-request ownership checks. - Settle completion receipts and deterministic cross-machine snapshot digests. - Maintain default-deny Tenant/Goal/Actor authorization and incarnation pinning. 2. Introduce controlled remote authority gateway in Python control plane: - Default-disabled access: local private Goals never connect to remote authority or leak local credentials. - Opt-in gateway metadata projection for explicitly configured goals and the 'loopx-quality-review' canary. - Fail-closed should-run health check when remote authority is unconfigured or unreachable. 3. Conformance and unit test coverage across TypeScript and Python suites. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../remote-authority-task-service-v0.md | 7 +- .../coordination/remote_authority_contract.ts | 168 ++++++++++++++---- .../coordination/remote_authority_gateway.py | 123 +++++++++++++ .../coordination/remote_authority_http.ts | 6 +- .../coordination/remote_authority_service.ts | 107 ++++++++++- loopx/control_plane/heartbeat/builder.py | 19 ++ loopx/control_plane/quota/live_decision.py | 18 ++ .../test_remote_authority_gateway.py | 101 +++++++++++ .../control_plane_ts/remote_authority.test.ts | 154 ++++++++++++++++ .../remote_authority_fixture.ts | 4 +- .../remote_authority_lifecycle_conformance.ts | 104 ++++++++++- .../remote_authority_mode_conformance.ts | 2 +- 12 files changed, 771 insertions(+), 42 deletions(-) create mode 100644 loopx/control_plane/coordination/remote_authority_gateway.py create mode 100644 tests/control_plane/test_remote_authority_gateway.py diff --git a/docs/reference/remote-authority-task-service-v0.md b/docs/reference/remote-authority-task-service-v0.md index 0816855531..064cd2d823 100644 --- a/docs/reference/remote-authority-task-service-v0.md +++ b/docs/reference/remote-authority-task-service-v0.md @@ -40,7 +40,7 @@ The policy schema is `loopx_remote_authority_policy_v0`: "tenant_id": "tenant-one", "goal_id": "goal-one", "actor_id": "actor-a", - "operations": ["read", "claim", "progress", "renew", "release"] + "operations": ["read", "claim", "progress", "renew", "release", "complete", "block", "defer", "todo_add", "snapshot"] }] }] } @@ -70,6 +70,11 @@ POST JSON to `/v0/tasks/execute` with a `Bearer` credential. Every command inclu | `progress` | `request_id`, `claim_request_id`, `note` | Update a note using the original claim and current execution fence. | | `renew` | `idempotency_key`, `expected_version`, `ttl_seconds` | Renew the current lease through the existing lifecycle owner. | | `release` | `idempotency_key`, `expected_version` | Release execution; this does not clear the task's assignment or complete it. | +| `complete` | `request_id`, `claim_request_id`, `evidence`, `note?` | Settle task completion with verified evidence string, release lease, and generate receipt. | +| `block` | `request_id`, `reason`, `claim_request_id?`, `resume_when?`, `note?` | Pause task as blocked with explicit reason, releasing execution lease. | +| `defer` | `request_id`, `reason`, `resume_when`, `claim_request_id?`, `note?` | Defer task with resumption condition and rationale, releasing execution lease. | +| `todo_add` | `request_id`, `text`, `priority?`, `role?`, ... | Create a canonical task under policy grant and tenant fence. | +| `snapshot` | `todo_id?` (optional) | Read-only projection of Goal todos, leases, and deterministic cross-machine receipt digest. | Use the lease key/version returned by claim or renewal for lifecycle commands. Progress names the original claim's client `request_id`; normal activity can diff --git a/loopx/control_plane/coordination/remote_authority_contract.ts b/loopx/control_plane/coordination/remote_authority_contract.ts index 638d89461a..6d99822700 100644 --- a/loopx/control_plane/coordination/remote_authority_contract.ts +++ b/loopx/control_plane/coordination/remote_authority_contract.ts @@ -10,7 +10,10 @@ import {parseIsoTimestamp} from "../runtime_timestamp.ts"; export const REMOTE_AUTHORITY_REQUEST = "loopx_remote_authority_request_v0"; export const REMOTE_AUTHORITY_RESPONSE = "loopx_remote_authority_response_v0"; -export const REMOTE_AUTHORITY_OPERATIONS = ["read", "claim", "progress", "renew", "release"] as const; +export const REMOTE_AUTHORITY_OPERATIONS = [ + "read", "claim", "progress", "renew", "release", + "complete", "block", "defer", "todo_add", "snapshot", +] as const; export type RemoteAuthorityOperation = typeof REMOTE_AUTHORITY_OPERATIONS[number]; export interface RemoteAuthorityScope { tenant_id: string; @@ -20,14 +23,32 @@ export interface RemoteAuthorityScope { } interface Base extends RemoteAuthorityScope { schema_version: typeof REMOTE_AUTHORITY_REQUEST; - todo_id: string; + todo_id?: string; } export type RemoteAuthorityRequest = Base & ( - | {operation: "read"} - | {operation: "claim"; request_id: string} - | {operation: "progress"; request_id: string; claim_request_id: string; note: string} - | {operation: "renew"; idempotency_key: string; expected_version: number; ttl_seconds: number} - | {operation: "release"; idempotency_key: string; expected_version: number} + | {operation: "read"; todo_id: string} + | {operation: "claim"; todo_id: string; request_id: string} + | {operation: "progress"; todo_id: string; request_id: string; claim_request_id: string; note: string} + | {operation: "renew"; todo_id: string; idempotency_key: string; expected_version: number; ttl_seconds: number} + | {operation: "release"; todo_id: string; idempotency_key: string; expected_version: number} + | {operation: "complete"; todo_id: string; request_id: string; claim_request_id: string; evidence: string; note?: string} + | {operation: "block"; todo_id: string; request_id: string; claim_request_id?: string; reason: string; resume_when?: string; note?: string} + | {operation: "defer"; todo_id: string; request_id: string; claim_request_id?: string; reason: string; resume_when: string; note?: string} + | { + operation: "todo_add"; + todo_id: string; + request_id: string; + text: string; + role?: "user" | "agent"; + priority?: string; + task_class?: string; + action_kind?: string; + task_repository?: string; + required_write_scopes?: readonly string[]; + claimed_by?: string; + note?: string; + } + | {operation: "snapshot"; todo_id?: string} ); export type RemoteAuthorityMode = "execute" | "recover"; export function remoteAuthorityId(value: unknown, name: string): string { @@ -35,27 +56,103 @@ export function remoteAuthorityId(value: unknown, name: string): string { if (id.length > 200 || /[\x00-\x20\x7f]/u.test(id)) throw new TypeError(`Invalid ${name}`); return id; } + +const MANDATORY_KEYS: Record = { + read: ["todo_id"], + claim: ["todo_id", "request_id"], + progress: ["todo_id", "request_id", "claim_request_id", "note"], + renew: ["todo_id", "ttl_seconds", "idempotency_key", "expected_version"], + release: ["todo_id", "idempotency_key", "expected_version"], + complete: ["todo_id", "request_id", "claim_request_id", "evidence"], + block: ["todo_id", "request_id", "reason"], + defer: ["todo_id", "request_id", "reason", "resume_when"], + todo_add: ["todo_id", "request_id", "text"], + snapshot: [], +}; +const OPTIONAL_KEYS: Record = { + read: [], + claim: [], + progress: [], + renew: [], + release: [], + complete: ["note"], + block: ["claim_request_id", "resume_when", "note"], + defer: ["claim_request_id", "note"], + todo_add: ["role", "priority", "task_class", "action_kind", "task_repository", "required_write_scopes", "claimed_by", "note"], + snapshot: ["todo_id"], +}; + export function decodeRemoteAuthorityRequest(raw: unknown): RemoteAuthorityRequest { const value = canonicalAuthorityObject(raw, "remote authority request"); - const keys = ["schema_version", "tenant_id", "goal_id", "actor_id", "store_identity", "todo_id", "operation"]; if (value.schema_version !== REMOTE_AUTHORITY_REQUEST || - !REMOTE_AUTHORITY_OPERATIONS.includes(value.operation as RemoteAuthorityOperation)) throw new TypeError("Invalid request schema or operation"); - switch (value.operation) { - case "claim": keys.push("request_id"); break; - case "progress": keys.push("request_id", "claim_request_id", "note"); break; - case "renew": keys.push("ttl_seconds", "idempotency_key", "expected_version"); break; - case "release": keys.push("idempotency_key", "expected_version"); break; - } - if (!hasExactAuthorityKeys(value, keys)) throw new TypeError("Unexpected or missing request fields"); - for (const key of ["tenant_id", "goal_id", "actor_id", "todo_id", "request_id", "claim_request_id", "idempotency_key"]) { - if (key in value) remoteAuthorityId(value[key], key); - } - if (normalizeTodoId(value.todo_id) !== value.todo_id || normalizeGoalId(value.goal_id) !== value.goal_id || - normalizeOwner(value.actor_id) !== value.actor_id) throw new TypeError("Use canonical task, Goal and actor identities"); - if (typeof value.store_identity !== "string" || !POSTGRESQL_STORE_IDENTITY_PATTERN.test(value.store_identity)) throw new TypeError("Invalid store identity"); - if ("expected_version" in value && (!Number.isSafeInteger(value.expected_version) || Number(value.expected_version) < 0)) throw new TypeError("Invalid lease version"); - if ("ttl_seconds" in value && (!Number.isSafeInteger(value.ttl_seconds) || Number(value.ttl_seconds) < 1 || Number(value.ttl_seconds) > 3600)) throw new TypeError("Invalid lease duration"); - if (value.operation === "progress" && (typeof value.note !== "string" || !value.note.trim() || Buffer.byteLength(value.note) > 4096)) throw new TypeError("Progress requires a bounded note"); + !REMOTE_AUTHORITY_OPERATIONS.includes(value.operation as RemoteAuthorityOperation)) { + throw new TypeError("Invalid request schema or operation"); + } + const op = value.operation as RemoteAuthorityOperation; + const baseKeys = ["schema_version", "tenant_id", "goal_id", "actor_id", "store_identity", "operation"]; + const required = [...baseKeys, ...MANDATORY_KEYS[op]]; + for (const key of required) { + if (!Object.hasOwn(value, key)) throw new TypeError(`Unexpected or missing request fields: missing ${key}`); + } + const allowed = new Set([...required, ...OPTIONAL_KEYS[op]]); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new TypeError(`Unexpected or missing request fields: unexpected ${key}`); + } + for (const key of ["tenant_id", "goal_id", "actor_id", "todo_id", "request_id", "claim_request_id", "idempotency_key", "claimed_by"]) { + if (key in value && typeof value[key] === "string") remoteAuthorityId(value[key], key); + } + if (typeof value.todo_id === "string" && normalizeTodoId(value.todo_id) !== value.todo_id) { + throw new TypeError("Use canonical task, Goal and actor identities"); + } + if (normalizeGoalId(value.goal_id) !== value.goal_id || normalizeOwner(value.actor_id) !== value.actor_id) { + throw new TypeError("Use canonical task, Goal and actor identities"); + } + if (typeof value.store_identity !== "string" || !POSTGRESQL_STORE_IDENTITY_PATTERN.test(value.store_identity)) { + throw new TypeError("Invalid store identity"); + } + if ("expected_version" in value && (!Number.isSafeInteger(value.expected_version) || Number(value.expected_version) < 0)) { + throw new TypeError("Invalid lease version"); + } + if ("ttl_seconds" in value && (!Number.isSafeInteger(value.ttl_seconds) || Number(value.ttl_seconds) < 1 || Number(value.ttl_seconds) > 3600)) { + throw new TypeError("Invalid lease duration"); + } + if (value.operation === "progress" && (typeof value.note !== "string" || !value.note.trim() || Buffer.byteLength(value.note) > 4096)) { + throw new TypeError("Progress requires a bounded note"); + } + if ("note" in value && value.note !== undefined && (typeof value.note !== "string" || !value.note.trim() || Buffer.byteLength(value.note) > 4096)) { + throw new TypeError("Note requires a bounded non-empty string"); + } + if ("evidence" in value && (typeof value.evidence !== "string" || !value.evidence.trim() || Buffer.byteLength(value.evidence) > 16 * 1024)) { + throw new TypeError("Evidence requires a bounded non-empty string"); + } + if ("reason" in value && (typeof value.reason !== "string" || !value.reason.trim() || Buffer.byteLength(value.reason) > 4096)) { + throw new TypeError("Reason requires a bounded non-empty string"); + } + if ("resume_when" in value && (typeof value.resume_when !== "string" || !value.resume_when.trim() || Buffer.byteLength(value.resume_when) > 1024)) { + throw new TypeError("Resume condition requires a bounded non-empty string"); + } + if ("text" in value && (typeof value.text !== "string" || !value.text.trim() || Buffer.byteLength(value.text) > 4096)) { + throw new TypeError("Text requires a bounded non-empty string"); + } + if ("priority" in value && value.priority !== undefined && !["P0", "P1", "P2", "P3", "P4"].includes(String(value.priority))) { + throw new TypeError("Invalid priority"); + } + if ("role" in value && value.role !== undefined && value.role !== "user" && value.role !== "agent") { + throw new TypeError("Invalid role"); + } + if ("claimed_by" in value && value.claimed_by !== undefined && (typeof value.claimed_by !== "string" || normalizeOwner(value.claimed_by) !== value.claimed_by)) { + throw new TypeError("Invalid claimed_by"); + } + if ("required_write_scopes" in value && value.required_write_scopes !== undefined) { + if (!Array.isArray(value.required_write_scopes) || value.required_write_scopes.some(s => typeof s !== "string" || !s.trim() || s.length > 256)) { + throw new TypeError("Invalid required_write_scopes"); + } + } + for (const strField of ["task_class", "action_kind", "task_repository"]) { + if (strField in value && value[strField] !== undefined && (typeof value[strField] !== "string" || !value[strField].trim() || value[strField].length > 256)) { + throw new TypeError(`Invalid ${strField}`); + } + } return value as unknown as RemoteAuthorityRequest; } @@ -128,11 +225,16 @@ function responseCommand(raw: unknown, request: RemoteAuthorityRequest, operatio } responseRequires(typeof result.changed === "boolean"); if (result.operation_id !== undefined) responseRequires(result.operation_id === operationId); - if (request.operation === "claim" || request.operation === "progress") responseRequires(result.todo_id === request.todo_id); + if (["claim", "progress", "complete", "block", "defer", "todo_add"].includes(request.operation)) { + if (result.todo_id !== undefined && request.todo_id !== undefined) responseRequires(result.todo_id === request.todo_id); + } if (request.operation === "claim") { responseRequires(result.claimed_by === request.actor_id && result.claim_operation_id === operationId); } - if (request.operation !== "progress") { + if (request.operation === "complete" && !historical && result.status === "applied") { + responseRequires(result.completed === true); + } + if (request.operation === "renew" || request.operation === "release") { responseRequires(result.handoff_mode === "hard_lease"); if (request.operation === "release" && result.missing === true) { responseRequires(request.expected_version === 0 && result.changed === false && @@ -155,8 +257,8 @@ export function decodeRemoteAuthorityResponse(raw: unknown, request: RemoteAutho responseRequires(typeof value.status === "string"); if (["rejected", "unavailable", "ambiguous"].includes(value.status)) { remoteAuthorityId(value.reason_code, "response reason"); - responseRequires(value.status !== "ambiguous" || mode === "execute" && request.operation !== "read"); - } else if (mode === "recover" && request.operation !== "read") { + responseRequires(value.status !== "ambiguous" || mode === "execute" && request.operation !== "read" && request.operation !== "snapshot"); + } else if (mode === "recover" && request.operation !== "read" && request.operation !== "snapshot") { const id = responseOperationId(value.operation_id, request); if (value.status !== "missing") { responseRequires(value.status === "found" && value.historical_only === true && @@ -177,8 +279,14 @@ export function decodeRemoteAuthorityResponse(raw: unknown, request: RemoteAutho if (todo.note_omitted !== undefined) responseRequires(todo.note_omitted === true && !Object.hasOwn(todo, "note")); if (value.lease !== null) responseLease(value.lease, request, false); } + } else if (mode === "execute" && request.operation === "snapshot") { + responseRequires(value.status === "snapshot"); + responseRevision(value.provider_revision, request); + responseRequires(value.goal_id === request.goal_id && value.store_identity === request.store_identity && + typeof value.receipt_sha256 === "string" && BARE_SHA256_PATTERN.test(value.receipt_sha256) && + Array.isArray(value.todos) && Array.isArray(value.leases)); } else { - responseRequires(mode === "execute" && request.operation !== "read" && value.status === "result"); + responseRequires(mode === "execute" && request.operation !== "read" && request.operation !== "snapshot" && value.status === "result"); responseCommand(value.result, request, responseOperationId(value.operation_id, request), false); } return value; diff --git a/loopx/control_plane/coordination/remote_authority_gateway.py b/loopx/control_plane/coordination/remote_authority_gateway.py new file mode 100644 index 0000000000..70fe7b72ca --- /dev/null +++ b/loopx/control_plane/coordination/remote_authority_gateway.py @@ -0,0 +1,123 @@ +"""Remote authority gateway access for explicitly shared goals. + +Enforces default-deny access: local private Goals never connect to remote authority +or leak private state. Only goals with explicit opt-in remote authority configuration +(or canary `loopx-quality-review` when explicitly enabled) route coordination over +authenticated HTTPS. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from typing import Any + +REMOTE_AUTHORITY_CANARY_GOAL_ID = "loopx-quality-review" +REMOTE_AUTHORITY_GATEWAY_SCHEMA = "loopx_remote_authority_gateway_v0" + + +def is_remote_authority_goal( + goal_id: str, + registry_goals: Mapping[str, Any] | None = None, +) -> bool: + """Return True only if the goal has explicit opt-in remote authority configuration.""" + if not goal_id: + return False + + goal_entry: Mapping[str, Any] | None = None + if registry_goals and goal_id in registry_goals: + entry = registry_goals[goal_id] + if isinstance(entry, Mapping): + goal_entry = entry + + remote_cfg = (goal_entry or {}).get("remote_authority") if goal_entry else None + if isinstance(remote_cfg, Mapping) and remote_cfg.get("enabled") is True: + return True + + # Canary goal: requires explicit opt-in via environment flag or explicit config + if goal_id == REMOTE_AUTHORITY_CANARY_GOAL_ID: + if os.environ.get("LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED") in {"1", "true", "yes"}: + return True + if isinstance(remote_cfg, Mapping) and remote_cfg.get("enabled") is not False: + return bool(remote_cfg.get("endpoint")) + + return False + + +def get_remote_authority_projection( + goal_id: str, + registry_goals: Mapping[str, Any] | None = None, +) -> dict[str, Any] | None: + """Return safe public gateway metadata without secret tokens or private paths.""" + if not is_remote_authority_goal(goal_id, registry_goals): + return None + + goal_entry: Mapping[str, Any] | None = None + if registry_goals and goal_id in registry_goals: + entry = registry_goals[goal_id] + if isinstance(entry, Mapping): + goal_entry = entry + + remote_cfg = dict((goal_entry or {}).get("remote_authority") or {}) + is_canary = goal_id == REMOTE_AUTHORITY_CANARY_GOAL_ID + + endpoint = remote_cfg.get("endpoint") or ( + os.environ.get("LOOPX_REMOTE_AUTHORITY_URL") + if is_canary + else None + ) or "https://remote.loopx.ai:8443" + + tenant_id = remote_cfg.get("tenant_id") or ("tenant-canary" if is_canary else "default-tenant") + store_identity = remote_cfg.get("store_identity") or ("postgresql:canary" if is_canary else "postgresql:default") + actor_id = remote_cfg.get("actor_id") or ("canary-reviewer" if is_canary else None) + + return { + "schema_version": REMOTE_AUTHORITY_GATEWAY_SCHEMA, + "goal_id": goal_id, + "enabled": True, + "canary": is_canary, + "endpoint": str(endpoint), + "tenant_id": str(tenant_id), + "store_identity": str(store_identity), + **({"actor_id": str(actor_id)} if actor_id else {}), + "handoff_mode": "hard_lease", + "access": "opt_in_authenticated_only", + } + + +def check_remote_authority_gateway_status( + goal_id: str, + registry_goals: Mapping[str, Any] | None = None, +) -> dict[str, Any] | None: + """Evaluate gateway health for quota should-run. + + Fails closed if the explicit remote authority is unreachable or unconfigured. + Returns None for normal local goals. + """ + if not is_remote_authority_goal(goal_id, registry_goals): + return None + + projection = get_remote_authority_projection(goal_id, registry_goals) + if projection is None: + return None + + # Check credential presence for the opt-in remote authority + cred = os.environ.get("LOOPX_AUTHORITY_CREDENTIAL") or os.environ.get("LOOPX_REMOTE_AUTHORITY_CREDENTIAL") + if not cred: + return { + "ok": False, + "should_run": False, + "status": "unavailable", + "reason_code": "remote_authority_credential_required", + "reason": ( + f"Goal {goal_id!r} is configured for remote authority but " + "LOOPX_AUTHORITY_CREDENTIAL is not set in the environment" + ), + "remote_gateway": projection, + } + + return { + "ok": True, + "status": "connected", + "remote_gateway": projection, + } diff --git a/loopx/control_plane/coordination/remote_authority_http.ts b/loopx/control_plane/coordination/remote_authority_http.ts index 20e8e50c1e..2f79d3b44c 100644 --- a/loopx/control_plane/coordination/remote_authority_http.ts +++ b/loopx/control_plane/coordination/remote_authority_http.ts @@ -23,7 +23,7 @@ export function createRemoteAuthorityServer(service: RemoteAuthorityService, response.writeHead(status, {Connection: "close"}); response.end(JSON.stringify({status: "rejected", reason_code: code})); }; - const mode = request.url === "/v0/tasks/execute" ? "execute" : request.url === "/v0/tasks/recover" ? "recover" : null; + const mode = request.url === "/v0/tasks/execute" || request.url === "/v0/tasks/snapshot" ? "execute" : request.url === "/v0/tasks/recover" ? "recover" : null; if (request.method !== "POST" || !mode) return reject(404, "route_unavailable"); if (draining || active >= maxInFlight) return reject(503, "remote_authority_busy"); const authorization = request.headers.authorization; @@ -49,7 +49,7 @@ export function createRemoteAuthorityServer(service: RemoteAuthorityService, const encoded = JSON.stringify(result); if (Buffer.byteLength(encoded) > MAX_RESPONSE_BYTES) { response.end(JSON.stringify(remoteAuthorityResponse(decoded, mode, {status: - mode === "recover" || decoded.operation === "read" ? "unavailable" : "ambiguous", + mode === "recover" || decoded.operation === "read" || decoded.operation === "snapshot" ? "unavailable" : "ambiguous", reason_code: "remote_authority_response_too_large"}))); } else response.end(encoded); } catch { @@ -102,7 +102,7 @@ export async function callRemoteAuthority(endpoint: string, credential: string, const timeout = options.timeoutMs ?? 15000; if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 120000) throw new TypeError("Invalid request deadline"); const uncertain = () => remoteAuthorityResponse(request, mode, {status: - mode === "recover" || request.operation === "read" ? "unavailable" : "ambiguous", + mode === "recover" || request.operation === "read" || request.operation === "snapshot" ? "unavailable" : "ambiguous", reason_code: "remote_authority_transport_uncertain"}); return new Promise(resolve => { const body = JSON.stringify(request); diff --git a/loopx/control_plane/coordination/remote_authority_service.ts b/loopx/control_plane/coordination/remote_authority_service.ts index 7a490e0919..65748f6b0e 100644 --- a/loopx/control_plane/coordination/remote_authority_service.ts +++ b/loopx/control_plane/coordination/remote_authority_service.ts @@ -6,8 +6,11 @@ import {PostgreSqlAuthorityService} from "./postgresql_authority_service.ts"; import type {PostgreSqlAuthorityDatabase} from "./postgresql_authority_store.ts"; import {executeCoordinationTodoClaim} from "./todo_claim.ts"; import {executeCoordinationTodoUpdate} from "./todo_update.ts"; +import {executeCoordinationTodoCreate} from "./todo_create.ts"; +import {executeCoordinationTodoTerminalLifecycle} from "./todo_terminal_lifecycle.ts"; import {executeCanonicalTaskLeaseLifecycle} from "./task_lease_lifecycle.ts"; import {leaseVersion, leaseEpoch} from "../work_items/task_lease_acquire.ts"; +import {TODO_DOMAIN_ITEM_SCHEMA} from "./coordination_state_contract.ts"; import {indexCoordinationProjection, validateCoordinationTodoReadModel} from "./coordination_projection.ts"; import {decodeRemoteAuthorityRequest, remoteAuthorityOperationId, remoteAuthorityResponse, type RemoteAuthorityMode, type RemoteAuthorityRequest} from "./remote_authority_contract.ts"; @@ -80,7 +83,8 @@ function pinnedStore(store: AuthorityStore, identity: string, admission: JsonObj function commandResult(result: JsonObject): JsonObject { const output: JsonObject = {}; for (const key of ["status", "reason_code", "changed", "todo_id", "claimed_by", "operation_id", - "claim_operation_id", "provider_revision", "cursor", "handoff_mode", "released", "renewed", "missing"]) { + "claim_operation_id", "provider_revision", "cursor", "handoff_mode", "released", "renewed", "missing", + "completed", "completion_receipt_id", "evidence", "receipt_sha256"]) { if (Object.hasOwn(result, key)) output[key] = result[key]; } if (result.lease && typeof result.lease === "object" && !Array.isArray(result.lease)) { @@ -89,6 +93,10 @@ function commandResult(result: JsonObject): JsonObject { "acquired_at", "updated_at", "expires_at"].filter(k => k in lease).map(k => [k, lease[k]])), version: leaseVersion(lease), lease_epoch: leaseEpoch(lease)}; } + if (result.todo && typeof result.todo === "object" && !Array.isArray(result.todo)) { + const todo = result.todo as JsonObject; + output.todo = Object.fromEntries(["todo_id", "status", "role", "priority", "claimed_by", "note"].filter(k => k in todo).map(k => [k, todo[k]])); + } return output; } @@ -101,7 +109,7 @@ export class RemoteAuthorityService { try { request = decodeRemoteAuthorityRequest(raw); } catch { return {status: "rejected", reason_code: "invalid_remote_authority_request"}; } const reply = (payload: JsonObject) => remoteAuthorityResponse(request, mode, payload); - if (mode !== "execute" && mode !== "recover" || mode === "recover" && request.operation === "read") { + if (mode !== "execute" && mode !== "recover" || mode === "recover" && (request.operation === "read" || request.operation === "snapshot")) { return reply({status: "rejected", reason_code: "invalid_remote_authority_mode"}); } let admission: RemoteAuthorityAdmission; @@ -125,7 +133,7 @@ export class RemoteAuthorityService { const store = pinnedStore(opened.store, request.store_identity, binding, operationId); let dispatched = false; try { - const previous = request.operation === "read" ? null : await store.readReceipt(operationId); + const previous = (request.operation === "read" || request.operation === "snapshot") ? null : await store.readReceipt(operationId); if (previous?.status === "found") { const original = previous.receipts[0]; const recorded = original?.remote_admission as JsonObject | undefined; @@ -161,6 +169,47 @@ export class RemoteAuthorityService { lease: lease ? {...Object.fromEntries(["owner", "status", "expires_at"].filter(k => k in lease).map(k => [k, lease[k]])), version: leaseVersion(lease), lease_epoch: leaseEpoch(lease)} : null}); } + if (request.operation === "snapshot") { + const todos = [...index.todos.values()] + .filter(t => !request.todo_id || t.todo_id === request.todo_id) + .map(t => { + const projected = Object.fromEntries( + ["todo_id", "status", "role", "priority", "claimed_by", "note"] + .filter(k => k in t) + .map(k => [k, t[k]]), + ); + if (Object.hasOwn(projected, "note") && Buffer.byteLength(JSON.stringify(projected.note)) > MAX_READ_NOTE_BYTES) { + delete projected.note; + projected.note_omitted = true; + } + return projected; + }); + const leases = [...index.leases.values()] + .filter(l => !request.todo_id || l.todo_id === request.todo_id) + .map(l => ({ + ...Object.fromEntries(["goal_id", "todo_id", "owner", "status", "expires_at"].filter(k => k in l).map(k => [k, l[k]])), + version: leaseVersion(l), + lease_epoch: leaseEpoch(l), + })); + const receiptSha = canonicalAuthoritySha256({ + schema_version: "loopx_goal_snapshot_receipt_v0", + tenant_id: request.tenant_id, + goal_id: request.goal_id, + store_identity: request.store_identity, + provider_revision: head.provider_revision, + todos, + leases, + }); + return reply({ + status: "snapshot", + provider_revision: head.provider_revision, + store_identity: request.store_identity, + goal_id: request.goal_id, + receipt_sha256: receiptSha, + todos, + leases, + }); + } if (head.head.handoff_mode !== "hard_lease") return reply({status: "rejected", reason_code: "remote_authority_requires_hard_lease"}); const now = this.#dependencies.now?.() ?? new Date(); if (!Number.isFinite(now.valueOf())) return reply({status: "unavailable", reason_code: "remote_authority_clock_unavailable"}); @@ -175,7 +224,57 @@ export class RemoteAuthorityService { operation_id: operationId, actor_agent_id: request.actor_id, expected_role: "agent", dry_run: false, claim_operation_id: remoteAuthorityOperationId(request, admission.principal_id, request.claim_request_id), patch: {note: request.note}, clear_fields: []}); - else result = await executeCanonicalTaskLeaseLifecycle(store, {...common, operation: request.operation, + else if (request.operation === "complete") { + const claimOpId = remoteAuthorityOperationId(request, admission.principal_id, request.claim_request_id); + result = await executeCoordinationTodoTerminalLifecycle(store, { + goal_id: request.goal_id, todo_id: request.todo_id, command: "complete", + expected_role: "agent", actor_agent_id: request.actor_id, + registered_agents: admission.registered_agents, lifecycle_grants: [], authority_reason: null, + decision_outcome: null, operation_identity: {kind: "explicit", operation_id: operationId}, + claim_operation_id: claimOpId, lease_idempotency_key: null, lease_expected_version: null, + allow_user_gate_auto_acquire: false, requested_no_followup: true, + requested_completion_turn_key: null, requested_completion_identity_source: null, + linked_successor_todo_ids: [], successor_intents: [], note: request.note ?? null, + evidence: request.evidence, reason: null, clear_claim: true, validation_declaration: null, + validation_receipt: null, completion_policy_request: null, dry_run: false, now, + }); + if (result.status === "applied") result.completed = true; + } else if (request.operation === "block" || request.operation === "defer") { + const claimOpId = request.claim_request_id + ? remoteAuthorityOperationId(request, admission.principal_id, request.claim_request_id) + : undefined; + result = await executeCoordinationTodoUpdate(store, { + ...common, operation_id: operationId, actor_agent_id: request.actor_id, + expected_role: "agent", dry_run: false, ...(claimOpId ? {claim_operation_id: claimOpId} : {}), + patch: {...(request.note ? {note: request.note} : {})}, clear_fields: [], + planning_intent: { + status: request.operation === "block" ? "blocked" : "deferred", + reason: request.reason, + ...(request.resume_when ? {resume_when: request.resume_when} : {}), + }, + }); + } else if (request.operation === "todo_add") { + const todoRecord = { + schema_version: TODO_DOMAIN_ITEM_SCHEMA, + todo_id: request.todo_id, + role: request.role ?? "agent", + status: "open", + done: false, + text: request.text, + archive_state: "active", + ...(request.priority ? {priority: request.priority} : {}), + ...(request.task_class ? {task_class: request.task_class} : {}), + ...(request.action_kind ? {action_kind: request.action_kind} : {}), + ...(request.task_repository ? {task_repository: request.task_repository} : {}), + ...(request.required_write_scopes ? {required_write_scopes: [...request.required_write_scopes]} : {}), + ...(request.claimed_by ? {claimed_by: request.claimed_by} : {}), + ...(request.note ? {note: request.note} : {}), + }; + result = await executeCoordinationTodoCreate(store, { + goal_id: request.goal_id, todo: todoRecord, actor_agent_id: request.actor_id, + registered_agents: admission.registered_agents, operation_id: operationId, dry_run: false, now, + }); + } else result = await executeCanonicalTaskLeaseLifecycle(store, {...common, operation: request.operation, owner: request.actor_id, idempotency_key: request.idempotency_key, required_handoff_mode: "hard_lease", expected_version: request.expected_version, ttl_seconds: request.operation === "renew" ? request.ttl_seconds : null}); return reply({status: "result", operation_id: operationId, result: commandResult(result)}); diff --git a/loopx/control_plane/heartbeat/builder.py b/loopx/control_plane/heartbeat/builder.py index 23796716e6..2736cbe16c 100644 --- a/loopx/control_plane/heartbeat/builder.py +++ b/loopx/control_plane/heartbeat/builder.py @@ -660,6 +660,25 @@ def build_heartbeat_prompt( if fine_grained: payload["turn_granularity"] = TURN_GRANULARITY_FINE payload["turn_mode"] = "fine_grained" + from ..coordination.remote_authority_gateway import ( + is_remote_authority_goal, + get_remote_authority_projection, + ) + registry_goals = None + if registry_path: + try: + from ...history import load_registry + registry_goals = { + str(g.get("id") or ""): g + for g in (load_registry(Path(registry_path)).get("goals") or []) + if isinstance(g, dict) and g.get("id") + } + except Exception: + registry_goals = None + if is_remote_authority_goal(goal_id, registry_goals): + remote_proj = get_remote_authority_projection(goal_id, registry_goals) + if remote_proj: + payload["remote_authority"] = remote_proj payload["agent_model"] = AgentRuntimeModel.PEER_V1.value return _prune_prompt_metadata(payload, thin=thin, brief=brief) diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index 5498b6ea28..19ac663ebf 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -971,6 +971,24 @@ def build_live_quota_should_run_decision( registry_path=registry_path, runtime_root=runtime_root, ) + from ..coordination.remote_authority_gateway import ( + is_remote_authority_goal, + check_remote_authority_gateway_status, + get_remote_authority_projection, + ) + goals = registry_goal_by_id(decision_status_payload) + if is_remote_authority_goal(goal_id, goals): + gateway_status = check_remote_authority_gateway_status(goal_id, goals) + if gateway_status and gateway_status.get("ok") is False: + payload["ok"] = False + payload["should_run"] = False + payload["status"] = gateway_status.get("status", "unavailable") + payload["reason_code"] = gateway_status.get("reason_code", "remote_authority_unreachable") + payload["reason"] = gateway_status.get("reason", "Remote authority unavailable") + payload["remote_gateway"] = gateway_status.get("remote_gateway") + else: + payload["remote_gateway"] = get_remote_authority_projection(goal_id, goals) + from ..work_items.context_readback import attach_work_context attach_work_context(payload, registry_path=registry_path, runtime_root=runtime_root, diff --git a/tests/control_plane/test_remote_authority_gateway.py b/tests/control_plane/test_remote_authority_gateway.py new file mode 100644 index 0000000000..864a2b29c8 --- /dev/null +++ b/tests/control_plane/test_remote_authority_gateway.py @@ -0,0 +1,101 @@ +"""Unit tests for remote authority gateway and opt-in canary access.""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import patch + +from loopx.control_plane.coordination.remote_authority_gateway import ( + REMOTE_AUTHORITY_CANARY_GOAL_ID, + check_remote_authority_gateway_status, + get_remote_authority_projection, + is_remote_authority_goal, +) +from loopx.control_plane.heartbeat.builder import build_heartbeat_prompt +from loopx.control_plane.quota.live_decision import build_live_quota_should_run_decision + + +def test_private_goals_default_disabled(): + # Private goals must never be treated as remote authority goals + assert not is_remote_authority_goal("personal-loopx-maintenance") + assert not is_remote_authority_goal("my-private-goal") + assert not is_remote_authority_goal("loopx-quality-review") # Disabled by default without explicit opt-in + + assert get_remote_authority_projection("personal-loopx-maintenance") is None + assert check_remote_authority_gateway_status("personal-loopx-maintenance") is None + + +def test_canary_goal_opt_in(): + with patch.dict(os.environ, {"LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED": "1", "LOOPX_AUTHORITY_CREDENTIAL": "test-credential-token"}): + assert is_remote_authority_goal(REMOTE_AUTHORITY_CANARY_GOAL_ID) + projection = get_remote_authority_projection(REMOTE_AUTHORITY_CANARY_GOAL_ID) + assert projection is not None + assert projection["canary"] is True + assert projection["goal_id"] == REMOTE_AUTHORITY_CANARY_GOAL_ID + assert projection["tenant_id"] == "tenant-canary" + assert projection["handoff_mode"] == "hard_lease" + # Secret credential must NEVER be present in projection + assert "credential" not in projection + assert "token" not in projection + + status = check_remote_authority_gateway_status(REMOTE_AUTHORITY_CANARY_GOAL_ID) + assert status is not None + assert status["ok"] is True + assert status["status"] == "connected" + + +def test_canary_goal_fails_closed_without_credential(): + with patch.dict(os.environ, {"LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED": "1"}, clear=True): + assert is_remote_authority_goal(REMOTE_AUTHORITY_CANARY_GOAL_ID) + status = check_remote_authority_gateway_status(REMOTE_AUTHORITY_CANARY_GOAL_ID) + assert status is not None + assert status["ok"] is False + assert status["should_run"] is False + assert status["status"] == "unavailable" + assert status["reason_code"] == "remote_authority_credential_required" + + +def test_explicit_registry_remote_authority_goal(): + registry_goals = { + "shared-team-goal": { + "id": "shared-team-goal", + "remote_authority": { + "enabled": True, + "endpoint": "https://coordination.internal.net:8443", + "tenant_id": "team-infra", + "store_identity": "postgresql:1234567890abcdef1234567890abcdef", + }, + }, + "private-local-goal": { + "id": "private-local-goal", + }, + } + + assert is_remote_authority_goal("shared-team-goal", registry_goals) + assert not is_remote_authority_goal("private-local-goal", registry_goals) + + proj = get_remote_authority_projection("shared-team-goal", registry_goals) + assert proj is not None + assert proj["endpoint"] == "https://coordination.internal.net:8443" + assert proj["tenant_id"] == "team-infra" + assert proj["canary"] is False + + +def test_heartbeat_prompt_incorporates_remote_gateway_for_canary(tmp_path: Path): + # For private goals, remote_authority is absent from payload + private_payload = build_heartbeat_prompt( + goal_id="personal-loopx-maintenance", + thin=True, + ) + assert "remote_authority" not in private_payload + + # When canary is enabled + with patch.dict(os.environ, {"LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED": "1"}): + canary_payload = build_heartbeat_prompt( + goal_id=REMOTE_AUTHORITY_CANARY_GOAL_ID, + thin=True, + ) + assert "remote_authority" in canary_payload + assert canary_payload["remote_authority"]["canary"] is True + assert canary_payload["remote_authority"]["goal_id"] == REMOTE_AUTHORITY_CANARY_GOAL_ID diff --git a/tests/control_plane_ts/remote_authority.test.ts b/tests/control_plane_ts/remote_authority.test.ts index c2583d70ac..04c1fa2614 100644 --- a/tests/control_plane_ts/remote_authority.test.ts +++ b/tests/control_plane_ts/remote_authority.test.ts @@ -201,3 +201,157 @@ test("a body completed after shutdown starts cannot dispatch a new command", asy await stopping; assert.equal(admissions, 0); }); + +test("complete, block, defer, todo_add and snapshot request decoding validates typed lifecycle semantics", () => { + const base = { + schema_version: "loopx_remote_authority_request_v0" as const, + tenant_id: "tenant-one", + goal_id: "goal-one", + actor_id: "actor-a", + store_identity: `postgresql:${"c".repeat(32)}`, + todo_id: "todo_0aaf8c3edda601ecc3c5b8c0", + }; + + // Complete + const completeReq = { + ...base, + operation: "complete" as const, + request_id: "req-complete-1", + claim_request_id: "req-claim-1", + evidence: "sha256:abcd verified delivery receipt", + note: "optional completion note", + }; + assert.deepEqual(decodeRemoteAuthorityRequest(completeReq), completeReq); + assert.throws(() => decodeRemoteAuthorityRequest({...completeReq, evidence: ""})); + assert.throws(() => decodeRemoteAuthorityRequest({...completeReq, claim_request_id: ""})); + + // Block + const blockReq = { + ...base, + operation: "block" as const, + request_id: "req-block-1", + reason: "waiting on upstream schema", + resume_when: "schema_applied", + note: "paused temporarily", + }; + assert.deepEqual(decodeRemoteAuthorityRequest(blockReq), blockReq); + assert.throws(() => decodeRemoteAuthorityRequest({...blockReq, reason: ""})); + + // Defer + const deferReq = { + ...base, + operation: "defer" as const, + request_id: "req-defer-1", + reason: "resuming next week", + resume_when: "resume_at:2026-11-20T00:00:00Z", + }; + assert.deepEqual(decodeRemoteAuthorityRequest(deferReq), deferReq); + assert.throws(() => decodeRemoteAuthorityRequest({...deferReq, resume_when: ""})); + + // Todo add + const addReq = { + ...base, + operation: "todo_add" as const, + request_id: "req-add-1", + text: "[P1] new remote task", + priority: "P1", + role: "agent" as const, + task_class: "advancement_task", + required_write_scopes: ["loopx/control_plane/**"], + }; + assert.deepEqual(decodeRemoteAuthorityRequest(addReq), addReq); + assert.throws(() => decodeRemoteAuthorityRequest({...addReq, priority: "P9"})); + assert.throws(() => decodeRemoteAuthorityRequest({...addReq, role: "invalid" as any})); + + // Snapshot (todo_id is optional) + const snapshotGoal = { + schema_version: "loopx_remote_authority_request_v0" as const, + tenant_id: "tenant-one", + goal_id: "goal-one", + actor_id: "actor-a", + store_identity: `postgresql:${"c".repeat(32)}`, + operation: "snapshot" as const, + }; + assert.deepEqual(decodeRemoteAuthorityRequest(snapshotGoal), snapshotGoal); + const snapshotTodo = {...snapshotGoal, todo_id: "todo_0aaf8c3edda601ecc3c5b8c0"}; + assert.deepEqual(decodeRemoteAuthorityRequest(snapshotTodo), snapshotTodo); +}); + +test("policy admits complete, block, defer, todo_add and snapshot grants and rejects ungranted operations", () => { + const policy = servicePolicy(); + const grant = policy.principals[0].grants[0]; + grant.operations = ["read", "claim", "progress", "renew", "release", "complete", "block", "defer", "todo_add", "snapshot"]; + const decodedPolicy = decodeRemoteAuthorityPolicy(policy); + + const base = { + schema_version: "loopx_remote_authority_request_v0" as const, + tenant_id: "tenant-one", + goal_id: "goal-one", + actor_id: "actor-a", + store_identity: `postgresql:${"c".repeat(32)}`, + todo_id: "todo_one", + }; + + const completeReq = decodeRemoteAuthorityRequest({ + ...base, operation: "complete", request_id: "req-comp", claim_request_id: "req-claim", evidence: "evidence-1", + }); + const snapshotReq = decodeRemoteAuthorityRequest({ + ...base, operation: "snapshot", + }); + + const admComplete = remoteAuthorityPolicyAdmission(decodedPolicy, credentials.a, completeReq); + assert.equal(admComplete.status, "allowed"); + + const admSnapshot = remoteAuthorityPolicyAdmission(decodedPolicy, credentials.a, snapshotReq); + assert.equal(admSnapshot.status, "allowed"); + + // Default-deny check: principal b does not have "complete" granted + policy.principals[1].grants[0].operations = ["read", "claim"]; + const decodedPolicyWithRestrictedB = decodeRemoteAuthorityPolicy(policy); + assert.equal(policy.principals[1].grants[0].operations.includes("complete" as any), false); + const admDenied = remoteAuthorityPolicyAdmission(decodedPolicyWithRestrictedB, credentials.b, { + ...completeReq, actor_id: "actor-b", + }); + assert.equal(admDenied.status, "rejected"); + assert.equal(admDenied.reason_code, "remote_authority_scope_denied"); +}); + +test("decodeRemoteAuthorityResponse validates snapshot and complete response payloads", () => { + const base = { + schema_version: "loopx_remote_authority_request_v0" as const, + tenant_id: "tenant-one", + goal_id: "goal-one", + actor_id: "actor-a", + store_identity: `postgresql:${"c".repeat(32)}`, + todo_id: "todo_one", + }; + const snapshotReq = decodeRemoteAuthorityRequest({...base, operation: "snapshot"}); + const snapResponse = remoteAuthorityResponse(snapshotReq, "execute", { + status: "snapshot", + provider_revision: `${base.store_identity}:10`, + store_identity: base.store_identity, + goal_id: base.goal_id, + receipt_sha256: "a".repeat(64), + todos: [{todo_id: "todo_one", status: "open", role: "agent"}], + leases: [], + }); + assert.deepEqual(decodeRemoteAuthorityResponse(snapResponse, snapshotReq, "execute"), snapResponse); + + const completeReq = decodeRemoteAuthorityRequest({ + ...base, operation: "complete", request_id: "req-c", claim_request_id: "req-cl", evidence: "proof", + }); + const opId = remoteAuthorityOperationId(completeReq, "principal-a"); + const compResponse = remoteAuthorityResponse(completeReq, "execute", { + status: "result", + operation_id: opId, + result: { + status: "applied", + changed: true, + completed: true, + todo_id: "todo_one", + provider_revision: `${base.store_identity}:11`, + cursor: "11", + }, + }); + assert.deepEqual(decodeRemoteAuthorityResponse(compResponse, completeReq, "execute"), compResponse); +}); diff --git a/tests/control_plane_ts/remote_authority_fixture.ts b/tests/control_plane_ts/remote_authority_fixture.ts index 5b4a0e9b77..93125f5136 100644 --- a/tests/control_plane_ts/remote_authority_fixture.ts +++ b/tests/control_plane_ts/remote_authority_fixture.ts @@ -28,9 +28,9 @@ export function servicePolicy(identity = serviceIdentity) { principals: ["a", "b"].map(actor => ({principal_id: `principal-${actor}`, credential_sha256: createHash("sha256").update(credentials[actor as "a" | "b"]).digest("hex"), grants: [{tenant_id: "tenant-one", goal_id: "goal-one", actor_id: `actor-${actor}`, - operations: ["read", "claim", "progress", "renew", "release"]}]}))}; + operations: ["read", "claim", "progress", "renew", "release", "complete", "block", "defer", "todo_add", "snapshot"]}]}))}; } -export function claimRequest(actor: "a" | "b", todo = "todo_one", id = `claim-${todo}`): RemoteAuthorityRequest { +export function claimRequest(actor: "a" | "b", todo = "todo_one", id = `claim-${todo}`): Extract { return {schema_version: REMOTE_AUTHORITY_REQUEST, tenant_id: "tenant-one", goal_id: "goal-one", actor_id: `actor-${actor}`, store_identity: serviceIdentity, todo_id: todo, operation: "claim", request_id: id}; } diff --git a/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts b/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts index 877c69313e..5ff52c1f97 100644 --- a/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts +++ b/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts @@ -4,7 +4,7 @@ import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; import {prepareCoordinationProjectionCommit, type CoordinationProjectionMutation} from "../../loopx/control_plane/coordination/coordination_projection.ts"; import {executeCanonicalTaskLeaseLifecycle} from "../../loopx/control_plane/coordination/task_lease_lifecycle.ts"; import type {RemoteAuthorityRequest} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; -import {postgresRemoteFixture, claimRequest} from "./remote_authority_fixture.ts"; +import {postgresRemoteFixture, claimRequest, serviceIdentity} from "./remote_authority_fixture.ts"; function lifecycle(todo: string, operation: "renew" | "release", version = 0): RemoteAuthorityRequest { const {request_id: _, ...scope} = claimRequest("a", todo) as Extract; @@ -89,4 +89,106 @@ export function registerRemoteAuthorityLifecycleConformance(connectionString: st assert.equal((recovered.value.result as JsonObject).missing, true); assert.deepEqual(await fixture.store.loadAuthority(), committed, "replay and recovery retain the original no-op receipt"); }); + + test("remote authority full lifecycle: snapshot, todo_add, complete, block, and defer", {skip: !connectionString}, async t => { + const fixture = await postgresRemoteFixture(t, connectionString!); + const running = await fixture.start(); + await fixture.seed(["todo_for_complete", "todo_for_block", "todo_for_defer"]); + + const baseScope = { + schema_version: "loopx_remote_authority_request_v0" as const, + tenant_id: "tenant-one", + goal_id: "goal-one", + actor_id: "actor-a", + store_identity: serviceIdentity, + }; + + // 1. Snapshot + const snap = await fixture.client(running.endpoint, "a", { + ...baseScope, + operation: "snapshot", + }); + assert.equal(snap.code, 0); + assert.equal(snap.value.status, "snapshot"); + assert(Array.isArray(snap.value.todos)); + assert(Array.isArray(snap.value.leases)); + assert.equal(typeof snap.value.receipt_sha256, "string"); + + // 2. Todo add + const added = await fixture.client(running.endpoint, "a", { + ...baseScope, + todo_id: "todo_created_remotely", + operation: "todo_add", + request_id: "req-add-remotely", + text: "[P1] remotely created task", + priority: "P1", + role: "agent", + }); + assert.equal(added.code, 0); + assert.equal((added.value.result as JsonObject).status, "applied"); + assert.equal((added.value.result as JsonObject).todo_id, "todo_created_remotely"); + + // 3. Claim -> Complete + const claimComp = await fixture.client(running.endpoint, "a", { + ...baseScope, + todo_id: "todo_for_complete", + operation: "claim", + request_id: "claim-for-comp", + }); + assert.equal(claimComp.code, 0); + assert.equal((claimComp.value.result as JsonObject).status, "applied"); + + const comp = await fixture.client(running.endpoint, "a", { + ...baseScope, + todo_id: "todo_for_complete", + operation: "complete", + request_id: "complete-req-1", + claim_request_id: "claim-for-comp", + evidence: "sha256:1111 delivery validated", + }); + assert.equal(comp.code, 0); + assert.equal((comp.value.result as JsonObject).status, "applied"); + assert.equal((comp.value.result as JsonObject).completed, true); + + // 4. Claim -> Block + const claimBlock = await fixture.client(running.endpoint, "a", { + ...baseScope, + todo_id: "todo_for_block", + operation: "claim", + request_id: "claim-for-block", + }); + assert.equal(claimBlock.code, 0); + + const block = await fixture.client(running.endpoint, "a", { + ...baseScope, + todo_id: "todo_for_block", + operation: "block", + request_id: "block-req-1", + claim_request_id: "claim-for-block", + reason: "external dependency blocked", + }); + assert.equal(block.code, 0); + assert.equal((block.value.result as JsonObject).status, "applied"); + + // 5. Claim -> Defer + const claimDefer = await fixture.client(running.endpoint, "a", { + ...baseScope, + todo_id: "todo_for_defer", + operation: "claim", + request_id: "claim-for-defer", + }); + assert.equal(claimDefer.code, 0); + + const defer = await fixture.client(running.endpoint, "a", { + ...baseScope, + todo_id: "todo_for_defer", + operation: "defer", + request_id: "defer-req-1", + claim_request_id: "claim-for-defer", + reason: "deferred until milestone", + resume_when: "resume_at:2026-12-01T00:00:00Z", + }); + assert.equal(defer.code, 0); + assert.equal((defer.value.result as JsonObject).status, "applied"); + }); } diff --git a/tests/control_plane_ts/remote_authority_mode_conformance.ts b/tests/control_plane_ts/remote_authority_mode_conformance.ts index 3d9cebc6c9..ea5dc9b6ea 100644 --- a/tests/control_plane_ts/remote_authority_mode_conformance.ts +++ b/tests/control_plane_ts/remote_authority_mode_conformance.ts @@ -16,7 +16,7 @@ export function registerRemoteAuthorityModeConformance(connectionString: string await t.test(`mode change ${timing} cannot commit a remote ${operation} under soft mode`, async child => { const claim = claimRequest("a", `todo_${operation}_${timing.replaceAll("-", "_")}`); const {request_id: _, ...scope} = claim as Extract; - const request: RemoteAuthorityRequest = operation === "claim" ? claim + const request = operation === "claim" ? claim : {...scope, operation, idempotency_key: "missing-lease-key", expected_version: 0}; await fixture.seed([request.todo_id]); let switched = false, headReads = 0, commitAttempts = 0; From 989aee43c97f385837ae2ca1b016b56691180762 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sun, 15 Nov 2026 04:58:58 +0800 Subject: [PATCH 2/5] test(coordination): assert subprocess readback before decoding Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- tests/control_plane_ts/coordination_runtime_shadow.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/control_plane_ts/coordination_runtime_shadow.test.ts b/tests/control_plane_ts/coordination_runtime_shadow.test.ts index c59a4bd26c..21b10e277a 100644 --- a/tests/control_plane_ts/coordination_runtime_shadow.test.ts +++ b/tests/control_plane_ts/coordination_runtime_shadow.test.ts @@ -377,7 +377,9 @@ for (const scenario of [ ]), ], {encoding: "utf8", timeout: 60_000}); assert.equal(result.status, 0, result.stderr); - const observed = JSON.parse(result.stdout.trim().split("\n").at(-1)); + const finalLine = result.stdout.trim().split("\n").at(-1); + assert.ok(finalLine, "source-observation subprocess must return a result"); + const observed = JSON.parse(finalLine); assert.equal(observed.status === "failed" ? observed.reason_code : observed.status, scenario.expected); const passes = scenario.expected === "applied" ? 2 : 1; assert.equal(observed.lstatCalls, 2 * passes); From 366be8c089aabc94566076bb19fad5db44e51f4c Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sun, 15 Nov 2026 04:59:00 +0800 Subject: [PATCH 3/5] fix(coordination): qualify explicit remote task reads and lifecycle receipts Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../remote-authority-task-service-v0.md | 44 ++++- .../coordination/remote-authority-client.ts | 2 +- .../coordination/remote_authority_contract.ts | 97 +++++++---- .../coordination/remote_authority_gateway.py | 150 ++++++------------ .../coordination/remote_authority_http.ts | 2 +- .../coordination/remote_authority_probe.ts | 40 +++++ .../coordination/remote_authority_service.ts | 82 +++++----- loopx/control_plane/heartbeat/builder.py | 27 +--- loopx/control_plane/quota/live_decision.py | 125 ++++++++++++--- .../test_remote_authority_gateway.py | 146 ++++++----------- .../remote_authority.integration.test.ts | 79 ++++++++- .../control_plane_ts/remote_authority.test.ts | 48 ++++-- .../remote_authority_lifecycle_conformance.ts | 27 ++++ 13 files changed, 543 insertions(+), 326 deletions(-) create mode 100644 loopx/control_plane/coordination/remote_authority_probe.ts diff --git a/docs/reference/remote-authority-task-service-v0.md b/docs/reference/remote-authority-task-service-v0.md index 064cd2d823..3d5241da40 100644 --- a/docs/reference/remote-authority-task-service-v0.md +++ b/docs/reference/remote-authority-task-service-v0.md @@ -4,7 +4,7 @@ This opt-in source-checkout service lets independent clients claim a canonical task, record progress, renew/release its execution lease and read back the result over authenticated HTTPS. It reuses PostgreSQL service admission and the existing Todo/lease transaction owners. It does not select a provider, seed or migrate a -Goal, expose arbitrary authority commits, or enable a shipped App/CLI route. +Goal, expose arbitrary authority commits, or allow the host scheduler to admit remote turns. ## Ownership and access @@ -60,8 +60,9 @@ revocation does not claim to undo an in-flight commit. POST JSON to `/v0/tasks/execute` with a `Bearer` credential. Every command includes `schema_version: "loopx_remote_authority_request_v0"`, `tenant_id`, `goal_id`, -`actor_id`, `store_identity`, `todo_id` and `operation`. Task IDs use canonical -`todo_` syntax. Unknown fields and operations are rejected. +`actor_id`, `store_identity` and `operation`. Supply `todo_id` for individual +task commands; omit it on `snapshot` for a whole Goal read. Task IDs use +canonical `todo_` syntax. Unknown fields and operations are rejected. | Operation | Additional fields | Behavior | | --- | --- | --- | @@ -70,7 +71,7 @@ POST JSON to `/v0/tasks/execute` with a `Bearer` credential. Every command inclu | `progress` | `request_id`, `claim_request_id`, `note` | Update a note using the original claim and current execution fence. | | `renew` | `idempotency_key`, `expected_version`, `ttl_seconds` | Renew the current lease through the existing lifecycle owner. | | `release` | `idempotency_key`, `expected_version` | Release execution; this does not clear the task's assignment or complete it. | -| `complete` | `request_id`, `claim_request_id`, `evidence`, `note?` | Settle task completion with verified evidence string, release lease, and generate receipt. | +| `complete` | `request_id`, `claim_request_id`, `evidence`, `note?` | Record an evidence reference and complete the task through the existing completion and lease owners. | | `block` | `request_id`, `reason`, `claim_request_id?`, `resume_when?`, `note?` | Pause task as blocked with explicit reason, releasing execution lease. | | `defer` | `request_id`, `reason`, `resume_when`, `claim_request_id?`, `note?` | Defer task with resumption condition and rationale, releasing execution lease. | | `todo_add` | `request_id`, `text`, `priority?`, `role?`, ... | Create a canonical task under policy grant and tenant fence. | @@ -100,6 +101,18 @@ the principal/tenant/Goal/actor scope. Reusing one with different intent fails. Renew/release retain the canonical lease operation identity. Their receipts also verify the admitted principal, actor and request digest before reuse. +For `block` and `defer`, use the existing Todo resume condition syntax, such as +`todo_done:todo_dependency` or `resume_at:2026-12-01T00:00:00Z`. A reason alone +can block a task; deferral requires a valid condition. + +A snapshot observes one provider revision. The client recomputes its digest from +the tenant, Goal, store incarnation, revision and ordered Todo and lease rows, +and rejects rows outside the requested scope or without a matching Todo. Two +authorized clients reading the same revision and scope get the same digest. +The example client exits successfully for a valid snapshot. A response over +64 KiB is unavailable; retry a narrower read if applicable. A snapshot has no +recovery operation, produces no write, and never includes lease keys. + The service clock decides expiry. Mutations require `hard_lease`; normal task eligibility, write-scope and lease checks still apply. A receipt is historical evidence, not permission to execute after expiry, release or takeover. @@ -165,6 +178,25 @@ socket has gone away. Disconnecting a client does not establish that its write r Stop the listener and remove access grants to disable this candidate; existing local provider selection is unaffected and durable receipts remain in PostgreSQL. +## Optional host observation + +The local registry can explicitly opt one registered Goal into a **read-only** +health observation. Set `remote_authority.enabled: true` on that Goal with a +credential-free HTTPS `endpoint`, verified `tenant_id`, `store_identity`, +registered `actor_id` and a stable existing `probe_todo_id`. Set `ca_file` to +an absolute path to a trusted CA when using a private CA. Use the service +credential environment variable above; the legacy environment-only canary flag +does not activate any Goal. The binding and credentials stay private. + +A full `heartbeat-prompt` can describe the configuration as unverified. +`quota should-run` runs one bounded read of the known Todo for the current +registered actor. Missing credentials, wrong scope, an invalid TLS certificate, +a missing probe Todo, revocation and an unreachable service never count as +verified. A verified read still waits for shared Turn admission. The scheduler +does not select local work, spend quota, or acquire a remote lease from this +observation. Owner Stop and paused quota keep their precedence. Disabling the +setting removes the observation; local File/SQLite behavior is unchanged. + ## Qualification and remaining work `npm run test:remote-authority` runs decoding/policy/TLS tests. With @@ -172,7 +204,9 @@ local provider selection is unaffected and durable receipts remain in PostgreSQL it additionally creates a private test database and a restricted runtime role, starts real TLS server/client processes, and checks races, replay, revocation, expiry/takeover, incarnation rotation and response loss followed by restart. -It removes its own database and role afterwards. The dedicated PostgreSQL CI +The suite also invokes the real quota CLI against that isolated TLS service; +it checks revocation, disconnection and recovery without changing the remote +head. It removes its own database and role afterwards. The dedicated PostgreSQL CI workflow requires a real integration subtest; a skipped backend is not a pass. This qualifies a source candidate on one host with independent client processes diff --git a/examples/coordination/remote-authority-client.ts b/examples/coordination/remote-authority-client.ts index 9485c66d48..21232642b7 100644 --- a/examples/coordination/remote-authority-client.ts +++ b/examples/coordination/remote-authority-client.ts @@ -15,5 +15,5 @@ const result = await callRemoteAuthority(values.endpoint, process.env.LOOPX_AUTH decodeRemoteAuthorityRequest(JSON.parse(raw)), {mode: values.recover ? "recover" : "execute", ...(values.ca ? {ca: await readFile(values.ca)} : {})}); process.stdout.write(`${JSON.stringify(result)}\n`); -if (!["found", "missing", "result"].includes(String(result.status)) || +if (!["found", "missing", "result", "snapshot"].includes(String(result.status)) || result.status === "result" && !["applied", "no_change", "replayed", "recovered"].includes(String((result.result as Record)?.status))) process.exitCode = 1; diff --git a/loopx/control_plane/coordination/remote_authority_contract.ts b/loopx/control_plane/coordination/remote_authority_contract.ts index 6d99822700..89d15ad74c 100644 --- a/loopx/control_plane/coordination/remote_authority_contract.ts +++ b/loopx/control_plane/coordination/remote_authority_contract.ts @@ -1,12 +1,14 @@ /** Candidate wire contract. Clients submit intentions, never authority facts. */ import type {JsonObject} from "../effect_program.ts"; -import {canonicalAuthorityObject, canonicalAuthoritySha256, hasExactAuthorityKeys, +import {canonicalAuthorityObject, canonicalAuthoritySha256, requireAuthorityStoreId} from "./authority_store_codec.ts"; import {POSTGRESQL_STORE_IDENTITY_PATTERN} from "./postgresql_authority_store.ts"; import {taskLeaseOperationIdentity} from "../work_items/task_lease_operation_identity.ts"; import {normalizeTodoId, normalizeGoalId, normalizeOwner} from "../work_items/task_lease_acquire.ts"; import {BARE_SHA256_PATTERN} from "../content_digest.ts"; import {parseIsoTimestamp} from "../runtime_timestamp.ts"; +import {normalizeTodoResumeWhen, TODO_RESUME_NORMALIZE_REQUEST_SCHEMA_VERSION} from "../todos/resume_condition.ts"; +import {normalizeTodoPriority} from "../todos/priority.ts"; export const REMOTE_AUTHORITY_REQUEST = "loopx_remote_authority_request_v0"; export const REMOTE_AUTHORITY_RESPONSE = "loopx_remote_authority_response_v0"; @@ -99,7 +101,7 @@ export function decodeRemoteAuthorityRequest(raw: unknown): RemoteAuthorityReque if (!allowed.has(key)) throw new TypeError(`Unexpected or missing request fields: unexpected ${key}`); } for (const key of ["tenant_id", "goal_id", "actor_id", "todo_id", "request_id", "claim_request_id", "idempotency_key", "claimed_by"]) { - if (key in value && typeof value[key] === "string") remoteAuthorityId(value[key], key); + if (Object.hasOwn(value, key)) remoteAuthorityId(value[key], key); } if (typeof value.todo_id === "string" && normalizeTodoId(value.todo_id) !== value.todo_id) { throw new TypeError("Use canonical task, Goal and actor identities"); @@ -116,10 +118,7 @@ export function decodeRemoteAuthorityRequest(raw: unknown): RemoteAuthorityReque if ("ttl_seconds" in value && (!Number.isSafeInteger(value.ttl_seconds) || Number(value.ttl_seconds) < 1 || Number(value.ttl_seconds) > 3600)) { throw new TypeError("Invalid lease duration"); } - if (value.operation === "progress" && (typeof value.note !== "string" || !value.note.trim() || Buffer.byteLength(value.note) > 4096)) { - throw new TypeError("Progress requires a bounded note"); - } - if ("note" in value && value.note !== undefined && (typeof value.note !== "string" || !value.note.trim() || Buffer.byteLength(value.note) > 4096)) { + if ("note" in value && (typeof value.note !== "string" || !value.note.trim() || Buffer.byteLength(value.note) > 4096)) { throw new TypeError("Note requires a bounded non-empty string"); } if ("evidence" in value && (typeof value.evidence !== "string" || !value.evidence.trim() || Buffer.byteLength(value.evidence) > 16 * 1024)) { @@ -128,28 +127,29 @@ export function decodeRemoteAuthorityRequest(raw: unknown): RemoteAuthorityReque if ("reason" in value && (typeof value.reason !== "string" || !value.reason.trim() || Buffer.byteLength(value.reason) > 4096)) { throw new TypeError("Reason requires a bounded non-empty string"); } - if ("resume_when" in value && (typeof value.resume_when !== "string" || !value.resume_when.trim() || Buffer.byteLength(value.resume_when) > 1024)) { - throw new TypeError("Resume condition requires a bounded non-empty string"); + if ("resume_when" in value && (typeof value.resume_when !== "string" || Buffer.byteLength(value.resume_when) > 1024 || + normalizeTodoResumeWhen({schema_version: TODO_RESUME_NORMALIZE_REQUEST_SCHEMA_VERSION, resume_when: value.resume_when}) === null)) { + throw new TypeError("A bounded structured Todo resume condition is required"); } if ("text" in value && (typeof value.text !== "string" || !value.text.trim() || Buffer.byteLength(value.text) > 4096)) { throw new TypeError("Text requires a bounded non-empty string"); } - if ("priority" in value && value.priority !== undefined && !["P0", "P1", "P2", "P3", "P4"].includes(String(value.priority))) { + if ("priority" in value && (typeof value.priority !== "string" || normalizeTodoPriority(value.priority) !== value.priority)) { throw new TypeError("Invalid priority"); } - if ("role" in value && value.role !== undefined && value.role !== "user" && value.role !== "agent") { + if ("role" in value && value.role !== "user" && value.role !== "agent") { throw new TypeError("Invalid role"); } - if ("claimed_by" in value && value.claimed_by !== undefined && (typeof value.claimed_by !== "string" || normalizeOwner(value.claimed_by) !== value.claimed_by)) { + if ("claimed_by" in value && normalizeOwner(value.claimed_by) !== value.claimed_by) { throw new TypeError("Invalid claimed_by"); } - if ("required_write_scopes" in value && value.required_write_scopes !== undefined) { + if ("required_write_scopes" in value) { if (!Array.isArray(value.required_write_scopes) || value.required_write_scopes.some(s => typeof s !== "string" || !s.trim() || s.length > 256)) { throw new TypeError("Invalid required_write_scopes"); } } for (const strField of ["task_class", "action_kind", "task_repository"]) { - if (strField in value && value[strField] !== undefined && (typeof value[strField] !== "string" || !value[strField].trim() || value[strField].length > 256)) { + if (strField in value && (typeof value[strField] !== "string" || !value[strField].trim() || value[strField].length > 256)) { throw new TypeError(`Invalid ${strField}`); } } @@ -175,6 +175,13 @@ export function remoteAuthorityResponse(request: unknown, mode: RemoteAuthorityM request_sha256: canonicalAuthoritySha256({mode, request})}; } +export function remoteAuthoritySnapshotDigest(request: RemoteAuthorityScope, revision: unknown, + todos: readonly unknown[], leases: readonly unknown[]): string { + return canonicalAuthoritySha256({schema_version: "loopx_goal_snapshot_receipt_v0", + tenant_id: request.tenant_id, goal_id: request.goal_id, store_identity: request.store_identity, + provider_revision: revision, todos, leases}); +} + function responseRequires(condition: unknown): asserts condition { if (!condition) throw new TypeError("Invalid remote authority response evidence"); } @@ -209,6 +216,36 @@ function responseLease(raw: unknown, request: RemoteAuthorityRequest, mutation: } return lease; } + +function responseTodo(raw: unknown, todoId?: string): string { + const todo = canonicalAuthorityObject(raw, "response Todo"); + const id = normalizeTodoId(todo.todo_id); + responseRequires(todo.todo_id === id && (todoId === undefined || id === todoId)); + remoteAuthorityId(todo.status, "Todo status"); + if (todo.claimed_by != null) responseRequires(typeof todo.claimed_by === "string" && normalizeOwner(todo.claimed_by) === todo.claimed_by); + if (todo.note !== undefined) responseRequires(todo.note === null || typeof todo.note === "string"); + if (todo.note_omitted !== undefined) responseRequires(todo.note_omitted === true && !Object.hasOwn(todo, "note")); + return id; +} + +function responseSnapshot(value: JsonObject, request: Extract): void { + responseRequires(value.status === "snapshot" && value.goal_id === request.goal_id && value.store_identity === request.store_identity && + Array.isArray(value.todos) && Array.isArray(value.leases)); + responseRevision(value.provider_revision, request); + const todoIds = value.todos.map(todo => responseTodo(todo, request.todo_id)); + const todoSet = new Set(todoIds); + responseRequires(todoSet.size === todoIds.length); + const leaseIds = new Set(); + for (const raw of value.leases) { + const lease = responseLease(raw, request, false); + const id = normalizeTodoId(lease.todo_id); + responseRequires(lease.todo_id === id && lease.goal_id === request.goal_id && todoSet.has(id) && + !leaseIds.has(id) && !Object.hasOwn(lease, "idempotency_key")); + leaseIds.add(id); + } + responseRequires(value.receipt_sha256 === remoteAuthoritySnapshotDigest(request, value.provider_revision, value.todos, value.leases)); +} + function responseCommand(raw: unknown, request: RemoteAuthorityRequest, operationId: string, historical: boolean): void { const result = canonicalAuthorityObject(raw, "command response"); if (!historical) { @@ -225,14 +262,23 @@ function responseCommand(raw: unknown, request: RemoteAuthorityRequest, operatio } responseRequires(typeof result.changed === "boolean"); if (result.operation_id !== undefined) responseRequires(result.operation_id === operationId); - if (["claim", "progress", "complete", "block", "defer", "todo_add"].includes(request.operation)) { - if (result.todo_id !== undefined && request.todo_id !== undefined) responseRequires(result.todo_id === request.todo_id); - } + if (request.operation !== "renew" && request.operation !== "release") responseRequires(result.todo_id === request.todo_id); if (request.operation === "claim") { - responseRequires(result.claimed_by === request.actor_id && result.claim_operation_id === operationId); + responseRequires(result.claimed_by === request.actor_id && result.claim_operation_id === operationId && + result.handoff_mode === "hard_lease"); + responseLease(result.lease, request, true); + } + if (request.operation === "complete") { + responseRequires(typeof result.completed_at === "string" && parseIsoTimestamp(result.completed_at) !== null); + remoteAuthorityId(result.completion_receipt_id, "completion receipt id"); } - if (request.operation === "complete" && !historical && result.status === "applied") { - responseRequires(result.completed === true); + if (request.operation === "todo_add") responseTodo(result.todo, request.todo_id); + if ((request.operation === "block" || request.operation === "defer") && result.changed === true) { + const transition = canonicalAuthorityObject(request.operation === "block" + ? result.blocked_lifecycle_transition : result.deferred_transition, "Todo lifecycle transition"); + responseRequires(transition.kind === (request.operation === "block" ? "todo_blocked" : "todo_deferred") && + transition.execution_authority_granted === false && + ["absent", "released", "already_released"].some(state => transition.lease_retirement === state)); } if (request.operation === "renew" || request.operation === "release") { responseRequires(result.handoff_mode === "hard_lease"); @@ -271,20 +317,11 @@ export function decodeRemoteAuthorityResponse(raw: unknown, request: RemoteAutho if (value.status !== "missing") { responseRequires(value.status === "found"); responseRevision(value.provider_revision, request); - const todo = canonicalAuthorityObject(value.todo, "read Todo"); - responseRequires(todo.todo_id === request.todo_id); - remoteAuthorityId(todo.status, "Todo status"); - if (todo.claimed_by != null) responseRequires(typeof todo.claimed_by === "string" && normalizeOwner(todo.claimed_by) === todo.claimed_by); - if (todo.note !== undefined) responseRequires(todo.note === null || typeof todo.note === "string"); - if (todo.note_omitted !== undefined) responseRequires(todo.note_omitted === true && !Object.hasOwn(todo, "note")); + responseTodo(value.todo, request.todo_id); if (value.lease !== null) responseLease(value.lease, request, false); } } else if (mode === "execute" && request.operation === "snapshot") { - responseRequires(value.status === "snapshot"); - responseRevision(value.provider_revision, request); - responseRequires(value.goal_id === request.goal_id && value.store_identity === request.store_identity && - typeof value.receipt_sha256 === "string" && BARE_SHA256_PATTERN.test(value.receipt_sha256) && - Array.isArray(value.todos) && Array.isArray(value.leases)); + responseSnapshot(value, request); } else { responseRequires(mode === "execute" && request.operation !== "read" && request.operation !== "snapshot" && value.status === "result"); responseCommand(value.result, request, responseOperationId(value.operation_id, request), false); diff --git a/loopx/control_plane/coordination/remote_authority_gateway.py b/loopx/control_plane/coordination/remote_authority_gateway.py index 70fe7b72ca..bc15e6e7b6 100644 --- a/loopx/control_plane/coordination/remote_authority_gateway.py +++ b/loopx/control_plane/coordination/remote_authority_gateway.py @@ -1,123 +1,77 @@ -"""Remote authority gateway access for explicitly shared goals. - -Enforces default-deny access: local private Goals never connect to remote authority -or leak private state. Only goals with explicit opt-in remote authority configuration -(or canary `loopx-quality-review` when explicitly enabled) route coordination over -authenticated HTTPS. -""" +"""Private configuration and read-only observation for the remote authority candidate.""" from __future__ import annotations +import json import os +import subprocess from collections.abc import Mapping +from pathlib import Path from typing import Any -REMOTE_AUTHORITY_CANARY_GOAL_ID = "loopx-quality-review" REMOTE_AUTHORITY_GATEWAY_SCHEMA = "loopx_remote_authority_gateway_v0" +_REQUIRED_BINDING = ("endpoint", "tenant_id", "store_identity", "actor_id", "probe_todo_id") -def is_remote_authority_goal( - goal_id: str, - registry_goals: Mapping[str, Any] | None = None, -) -> bool: - """Return True only if the goal has explicit opt-in remote authority configuration.""" - if not goal_id: - return False - - goal_entry: Mapping[str, Any] | None = None - if registry_goals and goal_id in registry_goals: - entry = registry_goals[goal_id] - if isinstance(entry, Mapping): - goal_entry = entry - - remote_cfg = (goal_entry or {}).get("remote_authority") if goal_entry else None - if isinstance(remote_cfg, Mapping) and remote_cfg.get("enabled") is True: - return True - - # Canary goal: requires explicit opt-in via environment flag or explicit config - if goal_id == REMOTE_AUTHORITY_CANARY_GOAL_ID: - if os.environ.get("LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED") in {"1", "true", "yes"}: - return True - if isinstance(remote_cfg, Mapping) and remote_cfg.get("enabled") is not False: - return bool(remote_cfg.get("endpoint")) - - return False - - -def get_remote_authority_projection( - goal_id: str, - registry_goals: Mapping[str, Any] | None = None, -) -> dict[str, Any] | None: - """Return safe public gateway metadata without secret tokens or private paths.""" - if not is_remote_authority_goal(goal_id, registry_goals): - return None - - goal_entry: Mapping[str, Any] | None = None - if registry_goals and goal_id in registry_goals: - entry = registry_goals[goal_id] - if isinstance(entry, Mapping): - goal_entry = entry +def remote_authority_config( + goal: Mapping[str, Any] | None, +) -> Mapping[str, Any] | None: + """Opt in only through the selected registered Goal, never an environment default.""" + cfg = goal.get("remote_authority") if goal is not None else None + return cfg if isinstance(cfg, Mapping) and cfg.get("enabled") is True else None - remote_cfg = dict((goal_entry or {}).get("remote_authority") or {}) - is_canary = goal_id == REMOTE_AUTHORITY_CANARY_GOAL_ID - endpoint = remote_cfg.get("endpoint") or ( - os.environ.get("LOOPX_REMOTE_AUTHORITY_URL") - if is_canary - else None - ) or "https://remote.loopx.ai:8443" +def _binding_complete(cfg: Mapping[str, Any]) -> bool: + return all(isinstance(cfg.get(key), str) and bool(cfg[key].strip()) for key in _REQUIRED_BINDING) - tenant_id = remote_cfg.get("tenant_id") or ("tenant-canary" if is_canary else "default-tenant") - store_identity = remote_cfg.get("store_identity") or ("postgresql:canary" if is_canary else "postgresql:default") - actor_id = remote_cfg.get("actor_id") or ("canary-reviewer" if is_canary else None) +def get_remote_authority_projection(goal_id: str, cfg: Mapping[str, Any]) -> dict[str, Any]: + """Describe configuration without portraying it as connectivity or exposing its binding.""" return { "schema_version": REMOTE_AUTHORITY_GATEWAY_SCHEMA, "goal_id": goal_id, "enabled": True, - "canary": is_canary, - "endpoint": str(endpoint), - "tenant_id": str(tenant_id), - "store_identity": str(store_identity), - **({"actor_id": str(actor_id)} if actor_id else {}), "handoff_mode": "hard_lease", "access": "opt_in_authenticated_only", + "status": "unverified" if _binding_complete(cfg) else "unconfigured", } def check_remote_authority_gateway_status( goal_id: str, - registry_goals: Mapping[str, Any] | None = None, -) -> dict[str, Any] | None: - """Evaluate gateway health for quota should-run. - - Fails closed if the explicit remote authority is unreachable or unconfigured. - Returns None for normal local goals. - """ - if not is_remote_authority_goal(goal_id, registry_goals): - return None - - projection = get_remote_authority_projection(goal_id, registry_goals) - if projection is None: - return None - - # Check credential presence for the opt-in remote authority - cred = os.environ.get("LOOPX_AUTHORITY_CREDENTIAL") or os.environ.get("LOOPX_REMOTE_AUTHORITY_CREDENTIAL") - if not cred: - return { - "ok": False, - "should_run": False, - "status": "unavailable", - "reason_code": "remote_authority_credential_required", - "reason": ( - f"Goal {goal_id!r} is configured for remote authority but " - "LOOPX_AUTHORITY_CREDENTIAL is not set in the environment" - ), - "remote_gateway": projection, - } - - return { - "ok": True, - "status": "connected", - "remote_gateway": projection, - } + agent_id: str | None, + cfg: Mapping[str, Any], +) -> dict[str, Any]: + """Read one known remote Todo through the existing TLS client; return no remote content.""" + projection = get_remote_authority_projection(goal_id, cfg) + if projection["status"] == "unconfigured": + return {**projection, "reason_code": "remote_authority_binding_incomplete"} + if agent_id != cfg["actor_id"]: + return {**projection, "status": "unavailable", "reason_code": "remote_authority_actor_mismatch"} + credential = os.environ.get("LOOPX_AUTHORITY_CREDENTIAL") or os.environ.get("LOOPX_REMOTE_AUTHORITY_CREDENTIAL") + if not credential: + return {**projection, "status": "unavailable", "reason_code": "remote_authority_credential_required"} + from ..effect_runtime import EffectRuntimeStartupError, _node_executable + + try: + # The user-owned registry supplies the origin and optional private CA. + # Pass the credential and binding through the private pipe, not argv. + bridge = Path(__file__).with_name("remote_authority_probe.ts") + completed = subprocess.run( + [_node_executable(), "--no-warnings", "--experimental-strip-types", str(bridge)], + input=json.dumps({ + **{key: cfg[key] for key in _REQUIRED_BINDING}, + "goal_id": goal_id, "credential": credential, "ca_file": cfg.get("ca_file"), + }), + env={key: os.environ[key] for key in ("PATH", "SystemRoot", "WINDIR", "TMPDIR", "TEMP") + if key in os.environ}, + text=True, + capture_output=True, + check=True, + timeout=5, + ) + if completed.stdout == "verified\n": + return {**projection, "status": "verified"} + except (EffectRuntimeStartupError, OSError, subprocess.SubprocessError): + pass + return {**projection, "status": "unavailable", "reason_code": "remote_authority_read_unverified"} diff --git a/loopx/control_plane/coordination/remote_authority_http.ts b/loopx/control_plane/coordination/remote_authority_http.ts index 2f79d3b44c..55aca59b90 100644 --- a/loopx/control_plane/coordination/remote_authority_http.ts +++ b/loopx/control_plane/coordination/remote_authority_http.ts @@ -23,7 +23,7 @@ export function createRemoteAuthorityServer(service: RemoteAuthorityService, response.writeHead(status, {Connection: "close"}); response.end(JSON.stringify({status: "rejected", reason_code: code})); }; - const mode = request.url === "/v0/tasks/execute" || request.url === "/v0/tasks/snapshot" ? "execute" : request.url === "/v0/tasks/recover" ? "recover" : null; + const mode = request.url === "/v0/tasks/execute" ? "execute" : request.url === "/v0/tasks/recover" ? "recover" : null; if (request.method !== "POST" || !mode) return reject(404, "route_unavailable"); if (draining || active >= maxInFlight) return reject(503, "remote_authority_busy"); const authorization = request.headers.authorization; diff --git a/loopx/control_plane/coordination/remote_authority_probe.ts b/loopx/control_plane/coordination/remote_authority_probe.ts new file mode 100644 index 0000000000..a21846231e --- /dev/null +++ b/loopx/control_plane/coordination/remote_authority_probe.ts @@ -0,0 +1,40 @@ +/** Private stdin bridge to the same scoped TLS client used by task commands. */ +import {open} from "node:fs/promises"; +import {isAbsolute} from "node:path"; +import {canonicalAuthorityObject} from "./authority_store_codec.ts"; +import {decodeRemoteAuthorityRequest, REMOTE_AUTHORITY_REQUEST} from "./remote_authority_contract.ts"; +import {callRemoteAuthority} from "./remote_authority_http.ts"; + +async function readCa(path: unknown): Promise { + if (path === null || path === undefined) return undefined; + if (typeof path !== "string" || !isAbsolute(path)) throw new TypeError("Invalid CA path"); + const file = await open(path, "r"); + try { + const info = await file.stat(); + if (!info.isFile() || info.size === 0 || info.size > 64 * 1024) throw new TypeError("Invalid CA file"); + const bytes = Buffer.alloc(info.size); + if ((await file.read(bytes, 0, bytes.length, 0)).bytesRead !== bytes.length) throw new TypeError("Incomplete CA file"); + return bytes; + } finally {await file.close();} +} + +try { + let input = ""; + process.stdin.setEncoding("utf8"); + for await (const chunk of process.stdin) { + input += chunk; + if (Buffer.byteLength(input) > 16 * 1024) throw new TypeError("Invalid probe input"); + } + const cfg = canonicalAuthorityObject(JSON.parse(input), "remote authority probe"); + if (typeof cfg.endpoint !== "string" || typeof cfg.credential !== "string") throw new TypeError("Invalid probe binding"); + const request = decodeRemoteAuthorityRequest({ + schema_version: REMOTE_AUTHORITY_REQUEST, operation: "read", + tenant_id: cfg.tenant_id, goal_id: cfg.goal_id, actor_id: cfg.actor_id, + store_identity: cfg.store_identity, todo_id: cfg.probe_todo_id, + }); + const response = await callRemoteAuthority(cfg.endpoint, cfg.credential, request, {ca: await readCa(cfg.ca_file), timeoutMs: 3000}); + // A missing Todo is not proof of usable access to the expected authority. + process.stdout.write(response.status === "found" ? "verified\n" : "unavailable\n"); +} catch { + process.stdout.write("unavailable\n"); +} diff --git a/loopx/control_plane/coordination/remote_authority_service.ts b/loopx/control_plane/coordination/remote_authority_service.ts index 65748f6b0e..6adce348e1 100644 --- a/loopx/control_plane/coordination/remote_authority_service.ts +++ b/loopx/control_plane/coordination/remote_authority_service.ts @@ -12,7 +12,7 @@ import {executeCanonicalTaskLeaseLifecycle} from "./task_lease_lifecycle.ts"; import {leaseVersion, leaseEpoch} from "../work_items/task_lease_acquire.ts"; import {TODO_DOMAIN_ITEM_SCHEMA} from "./coordination_state_contract.ts"; import {indexCoordinationProjection, validateCoordinationTodoReadModel} from "./coordination_projection.ts"; -import {decodeRemoteAuthorityRequest, remoteAuthorityOperationId, remoteAuthorityResponse, +import {decodeRemoteAuthorityRequest, remoteAuthorityOperationId, remoteAuthorityResponse, remoteAuthoritySnapshotDigest, type RemoteAuthorityMode, type RemoteAuthorityRequest} from "./remote_authority_contract.ts"; export type RemoteAuthorityAdmission = @@ -71,6 +71,10 @@ function pinnedStore(store: AuthorityStore, identity: string, admission: JsonObj if (commit.operation_id !== operationId || commit.expected_provider_revision === null || !matches(commit.expected_provider_revision)) return {status: "failed", reason_code: "remote_authority_commit_binding_invalid", reason: "Command does not match admitted authority"}; + if (commit.next_projection.handoff_mode !== "hard_lease") return { + status: "failed", reason_code: "remote_authority_requires_hard_lease", + reason: "The committing authority must use hard leases", + }; // Metadata shares the existing business receipt transaction; no second ledger. return store.commitAuthority({...commit, receipts: commit.receipts.map(receipt => ({...receipt, remote_admission: admission}))}); @@ -78,13 +82,23 @@ function pinnedStore(store: AuthorityStore, identity: string, admission: JsonObj }; } +function projectReadTodo(todo: JsonObject, includePlanning: boolean): JsonObject { + const fields = ["todo_id", "status", "claimed_by", "note", ...(includePlanning ? ["role", "priority"] : [])]; + const output = Object.fromEntries(fields.filter(field => Object.hasOwn(todo, field)).map(field => [field, todo[field]])); + if (Object.hasOwn(output, "note") && Buffer.byteLength(JSON.stringify(output.note)) > MAX_READ_NOTE_BYTES) { + delete output.note; + output.note_omitted = true; + } + return output; +} + /** Deliberately bounded response projection: no SQL errors, paths, arbitrary * canonical fields, workspace observations or provider credentials cross here. */ function commandResult(result: JsonObject): JsonObject { const output: JsonObject = {}; for (const key of ["status", "reason_code", "changed", "todo_id", "claimed_by", "operation_id", "claim_operation_id", "provider_revision", "cursor", "handoff_mode", "released", "renewed", "missing", - "completed", "completion_receipt_id", "evidence", "receipt_sha256"]) { + "completed", "completed_at", "completion_receipt_id"]) { if (Object.hasOwn(result, key)) output[key] = result[key]; } if (result.lease && typeof result.lease === "object" && !Array.isArray(result.lease)) { @@ -97,6 +111,14 @@ function commandResult(result: JsonObject): JsonObject { const todo = result.todo as JsonObject; output.todo = Object.fromEntries(["todo_id", "status", "role", "priority", "claimed_by", "note"].filter(k => k in todo).map(k => [k, todo[k]])); } + for (const key of ["blocked_lifecycle_transition", "deferred_transition"]) { + const raw = result[key]; + if (raw && typeof raw === "object" && !Array.isArray(raw)) { + const transition = raw as JsonObject; + output[key] = Object.fromEntries(["kind", "lease_retirement", "execution_authority_granted"] + .filter(field => Object.hasOwn(transition, field)).map(field => [field, transition[field]])); + } + } return output; } @@ -158,54 +180,31 @@ export class RemoteAuthorityService { if (request.operation === "read") { const todo = index.todos.get(request.todo_id), lease = index.leases.get(request.todo_id); if (!todo) return reply({status: "missing"}); - const projectedTodo = Object.fromEntries(["todo_id", "status", "claimed_by", "note"].filter(k => k in todo).map(k => [k, todo[k]])); - // Notes authored through other canonical routes can exceed the remote - // write limit. Preserve the small lease facts needed for renew/release. - if (Object.hasOwn(projectedTodo, "note") && Buffer.byteLength(JSON.stringify(projectedTodo.note)) > MAX_READ_NOTE_BYTES) { - delete projectedTodo.note; projectedTodo.note_omitted = true; - } return reply({status: "found", provider_revision: head.provider_revision, - todo: projectedTodo, + todo: projectReadTodo(todo, false), lease: lease ? {...Object.fromEntries(["owner", "status", "expires_at"].filter(k => k in lease).map(k => [k, lease[k]])), version: leaseVersion(lease), lease_epoch: leaseEpoch(lease)} : null}); } if (request.operation === "snapshot") { - const todos = [...index.todos.values()] - .filter(t => !request.todo_id || t.todo_id === request.todo_id) - .map(t => { - const projected = Object.fromEntries( - ["todo_id", "status", "role", "priority", "claimed_by", "note"] - .filter(k => k in t) - .map(k => [k, t[k]]), - ); - if (Object.hasOwn(projected, "note") && Buffer.byteLength(JSON.stringify(projected.note)) > MAX_READ_NOTE_BYTES) { - delete projected.note; - projected.note_omitted = true; - } - return projected; + const todos = index.todo_ids + .filter(id => !request.todo_id || id === request.todo_id) + .map(id => projectReadTodo(index.todos.get(id)!, true)); + const leases = index.lease_todo_ids + .filter(id => !request.todo_id || id === request.todo_id) + .map(id => { + const l = index.leases.get(id)!; + return { + ...Object.fromEntries(["goal_id", "todo_id", "owner", "status", "expires_at"].filter(k => k in l).map(k => [k, l[k]])), + version: leaseVersion(l), + lease_epoch: leaseEpoch(l), + }; }); - const leases = [...index.leases.values()] - .filter(l => !request.todo_id || l.todo_id === request.todo_id) - .map(l => ({ - ...Object.fromEntries(["goal_id", "todo_id", "owner", "status", "expires_at"].filter(k => k in l).map(k => [k, l[k]])), - version: leaseVersion(l), - lease_epoch: leaseEpoch(l), - })); - const receiptSha = canonicalAuthoritySha256({ - schema_version: "loopx_goal_snapshot_receipt_v0", - tenant_id: request.tenant_id, - goal_id: request.goal_id, - store_identity: request.store_identity, - provider_revision: head.provider_revision, - todos, - leases, - }); return reply({ status: "snapshot", provider_revision: head.provider_revision, store_identity: request.store_identity, goal_id: request.goal_id, - receipt_sha256: receiptSha, + receipt_sha256: remoteAuthoritySnapshotDigest(request, head.provider_revision, todos, leases), todos, leases, }); @@ -274,6 +273,11 @@ export class RemoteAuthorityService { goal_id: request.goal_id, todo: todoRecord, actor_agent_id: request.actor_id, registered_agents: admission.registered_agents, operation_id: operationId, dry_run: false, now, }); + // The create owner can return a different, semantically duplicate + // Todo. That is a valid local no-op but not proof of this requested id. + if (result.status === "no_change" && result.todo_id !== request.todo_id) { + result = {status: "failed", reason_code: "remote_authority_todo_id_not_created"}; + } } else result = await executeCanonicalTaskLeaseLifecycle(store, {...common, operation: request.operation, owner: request.actor_id, idempotency_key: request.idempotency_key, required_handoff_mode: "hard_lease", expected_version: request.expected_version, ttl_seconds: request.operation === "renew" ? request.ttl_seconds : null}); diff --git a/loopx/control_plane/heartbeat/builder.py b/loopx/control_plane/heartbeat/builder.py index 2736cbe16c..2f1e587d57 100644 --- a/loopx/control_plane/heartbeat/builder.py +++ b/loopx/control_plane/heartbeat/builder.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from ...agent_registry import normalize_registered_agents +from ...agent_registry import load_goal_from_registry, normalize_registered_agents from ...ark_managed_agent_host import build_ark_managed_agent_host_contract from ...execution_profile import ( TURN_GRANULARITY_FINE, @@ -660,25 +660,14 @@ def build_heartbeat_prompt( if fine_grained: payload["turn_granularity"] = TURN_GRANULARITY_FINE payload["turn_mode"] = "fine_grained" - from ..coordination.remote_authority_gateway import ( - is_remote_authority_goal, - get_remote_authority_projection, - ) - registry_goals = None if registry_path: - try: - from ...history import load_registry - registry_goals = { - str(g.get("id") or ""): g - for g in (load_registry(Path(registry_path)).get("goals") or []) - if isinstance(g, dict) and g.get("id") - } - except Exception: - registry_goals = None - if is_remote_authority_goal(goal_id, registry_goals): - remote_proj = get_remote_authority_projection(goal_id, registry_goals) - if remote_proj: - payload["remote_authority"] = remote_proj + from ..coordination.remote_authority_gateway import ( + get_remote_authority_projection, + remote_authority_config, + ) + + if cfg := remote_authority_config(load_goal_from_registry(Path(registry_path), goal_id)): + payload["remote_authority"] = get_remote_authority_projection(goal_id, cfg) payload["agent_model"] = AgentRuntimeModel.PEER_V1.value return _prune_prompt_metadata(payload, thin=thin, brief=brief) diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index 19ac663ebf..c7f652833b 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -7,7 +7,7 @@ from typing import Any from ...quota import build_quota_should_run -from ...agent_registry import load_goal_from_registry +from ...agent_registry import load_goal_from_registry, registered_agent_ids_for_goal from ..agent_context import project_agent_context, project_goal_agent_context from ..runtime.time import now_utc_iso from ..goals.goal_frontier.fallback_source import live_fallback_authority_items @@ -688,6 +688,9 @@ def build_live_quota_should_run_decision( ) -> dict[str, Any]: """Build one live CLI decision while keeping host observation injectable.""" resolved_context = resolve_scheduler_execution_context(scheduler_execution_context) + from ..coordination.remote_authority_gateway import remote_authority_config + + registered_goal = load_goal_from_registry(registry_path, goal_id) app_automation_applicable = ( resolved_context.ok and resolved_context.context is not None @@ -725,6 +728,20 @@ def build_live_quota_should_run_decision( **decision_status_payload, "bounded_research_frontier": dict(frontier), } + remote_config = remote_authority_config(registered_goal) + if remote_config is not None: + # Use the same hard-pause precedence and keep fresh local selection + # outside the remote route, even when a TLS read succeeds. + return _build_remote_authority_wait_decision( + decision_status_payload, configured_goal=registered_goal or {}, + goal_id=goal_id, agent_id=agent_id, remote_config=remote_config, + registry_path=registry_path, runtime_root=runtime_root, goal_ref=goal_ref, + scheduler_context=resolved_context, available_capabilities=available_capabilities, + include_scheduler_detail=include_scheduler_detail, + codex_app_rrule=observed_rrule, codex_app_automation_id=observed_automation_id or None, + turn_instance_id=turn_instance_id, receipt_bound_todo_id=receipt_bound_todo_id, + receipt_bound_replan_obligation_id=receipt_bound_replan_obligation_id, + ) settlement_readback = read_heartbeat_settlement( runtime_root, goal_id=goal_id, @@ -939,7 +956,7 @@ def build_live_quota_should_run_decision( "agent_id": agent_id, "todo_id": (payload.get("selected_todo") or {}).get("todo_id"), } - goal = load_goal_from_registry(registry_path, goal_id) + goal = registered_goal if goal is not None: context = project_goal_agent_context( phase="before_plan", @@ -971,27 +988,97 @@ def build_live_quota_should_run_decision( registry_path=registry_path, runtime_root=runtime_root, ) + from ..work_items.context_readback import attach_work_context + + attach_work_context(payload, registry_path=registry_path, runtime_root=runtime_root, + hook_dispatch=turn_start_hook_dispatch, context_navigation=context_navigation, + inline_goal_source=inline_goal_source) + return payload + + +def _build_remote_authority_wait_decision( + status_payload: dict[str, Any], *, configured_goal: dict[str, Any], + goal_id: str, agent_id: str | None, remote_config: Mapping[str, Any], + registry_path: Path, runtime_root: Path, goal_ref: Mapping[str, Any] | None, + scheduler_context: SchedulerExecutionContextResolution, + available_capabilities: list[str] | None, turn_instance_id: str | None, + include_scheduler_detail: bool, codex_app_rrule: str, codex_app_automation_id: str | None, + receipt_bound_todo_id: str | None, receipt_bound_replan_obligation_id: str | None, +) -> dict[str, Any]: + from ...quota import _build_quota_plan_for_goal from ..coordination.remote_authority_gateway import ( - is_remote_authority_goal, check_remote_authority_gateway_status, get_remote_authority_projection, ) - goals = registry_goal_by_id(decision_status_payload) - if is_remote_authority_goal(goal_id, goals): - gateway_status = check_remote_authority_gateway_status(goal_id, goals) - if gateway_status and gateway_status.get("ok") is False: - payload["ok"] = False - payload["should_run"] = False - payload["status"] = gateway_status.get("status", "unavailable") - payload["reason_code"] = gateway_status.get("reason_code", "remote_authority_unreachable") - payload["reason"] = gateway_status.get("reason", "Remote authority unavailable") - payload["remote_gateway"] = gateway_status.get("remote_gateway") - else: - payload["remote_gateway"] = get_remote_authority_projection(goal_id, goals) + from .decision_summary import quota_plan_items + from .settlement_precedence import settled_replay_fields + from .should_run import build_quota_paused_should_run_payload + from .states import quota_item_is_paused - from ..work_items.context_readback import attach_work_context + plan, health_ok = _build_quota_plan_for_goal(status_payload, goal_id=goal_id) + item = next((row for row in quota_plan_items(plan) if row.get("goal_id") == goal_id), None) + if item is not None and quota_item_is_paused(item): + return build_quota_paused_should_run_payload( + status_payload, safe_goal_id=goal_id, requested_agent_id=agent_id, item=item, plan=plan, + goal_health_ok=health_ok, include_scheduler_detail=include_scheduler_detail, + codex_app_current_rrule=codex_app_rrule, codex_app_automation_id=codex_app_automation_id, + resolved_scheduler_context=scheduler_context, runtime_root=runtime_root, goal_ref=goal_ref, + ) + scope_matches = ( + (goal_ref is None or goal_ref.get("goal_instance_id") == configured_goal.get("goal_instance_id")) + and agent_id in registered_agent_ids_for_goal(configured_goal) + ) + readback = read_heartbeat_settlement( + runtime_root, goal_id=goal_id, agent_id=agent_id, todo_id=receipt_bound_todo_id, + turn_instance_id=turn_instance_id, registry_path=registry_path, goal_ref=goal_ref, + replan_obligation_id=None if receipt_bound_todo_id else receipt_bound_replan_obligation_id, + ) if scope_matches and turn_instance_id else None + settled = readback is not None and readback.replay_phase is ReceiptBoundReplayPhase.SETTLED + payload: dict[str, Any] = { + "ok": True, "mode": "should-run", "goal_id": goal_id, "agent_id": agent_id, + "should_run": False, "normal_delivery_allowed": False, "requires_user_action": False, + } + if goal_ref is not None: + payload["goal_ref"] = dict(goal_ref) + if scope_matches and not settled: + if (receipt_bound_todo_id and agent_id and turn_instance_id and readback is not None + and readback.replay_phase is ReceiptBoundReplayPhase.SETTLEMENT_PENDING): + attach_settlement_progress(payload, readback, registry_path=registry_path, + runtime_root=runtime_root, goal_ref=goal_ref) + if apply_receipt_bound_wait_recovery( + payload, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, + agent_id=agent_id, todo_id=receipt_bound_todo_id, turn_instance_id=turn_instance_id, + available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, + monitor_phase=readback.monitor_phase.value if readback.monitor_phase else None, + settlement_progress=readback.progress, replay_phase=readback.replay_phase, + heartbeat_receipt=readback.heartbeat_receipt, + ): + return payload + if apply_unsettled_host_turn_recovery_if_required( + payload, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id, + agent_id=agent_id, current_turn_instance_id=turn_instance_id, + available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, + goal_ref=goal_ref, + ): + return payload - attach_work_context(payload, registry_path=registry_path, runtime_root=runtime_root, - hook_dispatch=turn_start_hook_dispatch, context_navigation=context_navigation, - inline_goal_source=inline_goal_source) + if settled: + payload.update(settled_replay_fields()) + else: + gateway = (check_remote_authority_gateway_status(goal_id, agent_id, remote_config) if scope_matches else { + **get_remote_authority_projection(goal_id, remote_config), + "status": "unavailable", "reason_code": "remote_authority_local_scope_mismatch", + }) + reason_code = gateway.get("reason_code", "remote_authority_execution_not_admitted") + reason = ("Remote authority read verified; shared Turn admission is unavailable" + if gateway["status"] == "verified" else "Waiting for a verified read from the configured remote authority") + action = EffectiveAction.BLOCKED_WAIT.value + payload.update(decision="wait", state="waiting", effective_action=action, + remote_gateway=gateway, reason_code=reason_code, reason=reason, recommended_action=reason, + execution_obligation={"kind": action, "must_attempt_work": False, "delivery_allowed": False, + "reason": reason, "spend_policy": "no quota spend for remote authority observation"}) + payload["interaction_contract"] = build_interaction_contract( + payload, available_capabilities=available_capabilities, scheduler_execution_context=scheduler_context, + turn_instance_id=turn_instance_id, registry_path=str(registry_path), runtime_root=str(runtime_root), + ) return payload diff --git a/tests/control_plane/test_remote_authority_gateway.py b/tests/control_plane/test_remote_authority_gateway.py index 864a2b29c8..591aeaf34b 100644 --- a/tests/control_plane/test_remote_authority_gateway.py +++ b/tests/control_plane/test_remote_authority_gateway.py @@ -1,101 +1,51 @@ -"""Unit tests for remote authority gateway and opt-in canary access.""" +"""The real local CLI does not admit remote work or alter default-off decisions.""" from __future__ import annotations -import os -from pathlib import Path -from unittest.mock import patch - -from loopx.control_plane.coordination.remote_authority_gateway import ( - REMOTE_AUTHORITY_CANARY_GOAL_ID, - check_remote_authority_gateway_status, - get_remote_authority_projection, - is_remote_authority_goal, -) -from loopx.control_plane.heartbeat.builder import build_heartbeat_prompt -from loopx.control_plane.quota.live_decision import build_live_quota_should_run_decision - - -def test_private_goals_default_disabled(): - # Private goals must never be treated as remote authority goals - assert not is_remote_authority_goal("personal-loopx-maintenance") - assert not is_remote_authority_goal("my-private-goal") - assert not is_remote_authority_goal("loopx-quality-review") # Disabled by default without explicit opt-in - - assert get_remote_authority_projection("personal-loopx-maintenance") is None - assert check_remote_authority_gateway_status("personal-loopx-maintenance") is None - - -def test_canary_goal_opt_in(): - with patch.dict(os.environ, {"LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED": "1", "LOOPX_AUTHORITY_CREDENTIAL": "test-credential-token"}): - assert is_remote_authority_goal(REMOTE_AUTHORITY_CANARY_GOAL_ID) - projection = get_remote_authority_projection(REMOTE_AUTHORITY_CANARY_GOAL_ID) - assert projection is not None - assert projection["canary"] is True - assert projection["goal_id"] == REMOTE_AUTHORITY_CANARY_GOAL_ID - assert projection["tenant_id"] == "tenant-canary" - assert projection["handoff_mode"] == "hard_lease" - # Secret credential must NEVER be present in projection - assert "credential" not in projection - assert "token" not in projection - - status = check_remote_authority_gateway_status(REMOTE_AUTHORITY_CANARY_GOAL_ID) - assert status is not None - assert status["ok"] is True - assert status["status"] == "connected" - - -def test_canary_goal_fails_closed_without_credential(): - with patch.dict(os.environ, {"LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED": "1"}, clear=True): - assert is_remote_authority_goal(REMOTE_AUTHORITY_CANARY_GOAL_ID) - status = check_remote_authority_gateway_status(REMOTE_AUTHORITY_CANARY_GOAL_ID) - assert status is not None - assert status["ok"] is False - assert status["should_run"] is False - assert status["status"] == "unavailable" - assert status["reason_code"] == "remote_authority_credential_required" - - -def test_explicit_registry_remote_authority_goal(): - registry_goals = { - "shared-team-goal": { - "id": "shared-team-goal", - "remote_authority": { - "enabled": True, - "endpoint": "https://coordination.internal.net:8443", - "tenant_id": "team-infra", - "store_identity": "postgresql:1234567890abcdef1234567890abcdef", - }, - }, - "private-local-goal": { - "id": "private-local-goal", - }, - } - - assert is_remote_authority_goal("shared-team-goal", registry_goals) - assert not is_remote_authority_goal("private-local-goal", registry_goals) - - proj = get_remote_authority_projection("shared-team-goal", registry_goals) - assert proj is not None - assert proj["endpoint"] == "https://coordination.internal.net:8443" - assert proj["tenant_id"] == "team-infra" - assert proj["canary"] is False - - -def test_heartbeat_prompt_incorporates_remote_gateway_for_canary(tmp_path: Path): - # For private goals, remote_authority is absent from payload - private_payload = build_heartbeat_prompt( - goal_id="personal-loopx-maintenance", - thin=True, - ) - assert "remote_authority" not in private_payload - - # When canary is enabled - with patch.dict(os.environ, {"LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED": "1"}): - canary_payload = build_heartbeat_prompt( - goal_id=REMOTE_AUTHORITY_CANARY_GOAL_ID, - thin=True, - ) - assert "remote_authority" in canary_payload - assert canary_payload["remote_authority"]["canary"] is True - assert canary_payload["remote_authority"]["goal_id"] == REMOTE_AUTHORITY_CANARY_GOAL_ID +import json + +import pytest + +from examples.control_plane.quota_plan_fixtures import SCOPED_AGENT_ID, write_cli_fixture +from loopx.control_plane.testing.canary_harness import run_json_cli + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_local_cli_ignores_disabled_and_other_goal_remote_binding(tmp_path, monkeypatch, provider): + registry, runtime, _ = write_cli_fixture(tmp_path, scoped_agents=True, provider=provider) + # Old environment-only canary inputs cannot activate an unconfigured target. + monkeypatch.setenv("LOOPX_REMOTE_AUTHORITY_CANARY_ENABLED", "1") + monkeypatch.setenv("LOOPX_AUTHORITY_CREDENTIAL", "unused-synthetic-credential-" + "a" * 32) + goal_id = "throttled-half" + scope = ("--goal-id", goal_id, "--agent-id", SCOPED_AGENT_ID) + + def cli(*args): + return run_json_cli(*args, registry_path=registry, runtime_root=runtime) + + def read(): + prompt = cli("heartbeat-prompt", "--thin", "--codex-app", *scope) + assert "remote_authority" not in prompt + decision = cli("quota", "should-run", "--runtime-profile", "outer_controller", *scope) + assert "remote_gateway" not in decision + assert decision["ok"] is True + return tuple(decision.get(key) for key in ("decision", "effective_action", "should_run", "reason")) + + local = read() + assert local[:3] == ("skip", "throttled_skip", False), local + document = json.loads(registry.read_text()) + cfg = {"enabled": True, "endpoint": "https://127.0.0.1:1", "tenant_id": "other-synthetic-tenant", + "actor_id": SCOPED_AGENT_ID, "store_identity": "postgresql:" + "a" * 32, "probe_todo_id": "todo_probe"} + for goal in document["goals"]: + if goal["id"] == goal_id: + goal["remote_authority"] = {**cfg, "enabled": False} + if goal["id"] == "full-speed": + goal["remote_authority"] = cfg + registry.write_text(json.dumps(document), encoding="utf-8") + assert read() == local + + other = cli("heartbeat-prompt", "--full", "--codex-app", + "--goal-id", "full-speed", "--agent-id", SCOPED_AGENT_ID) + assert other["remote_authority"]["status"] == "unverified" + for key in ("endpoint", "tenant_id", "actor_id", "store_identity", "probe_todo_id"): + assert key not in other["remote_authority"] + assert cfg["endpoint"] not in other["task_body"] and cfg["tenant_id"] not in other["task_body"] diff --git a/tests/control_plane_ts/remote_authority.integration.test.ts b/tests/control_plane_ts/remote_authority.integration.test.ts index 7d864c37b0..23c1cf5205 100644 --- a/tests/control_plane_ts/remote_authority.integration.test.ts +++ b/tests/control_plane_ts/remote_authority.integration.test.ts @@ -2,7 +2,10 @@ import assert from "node:assert/strict"; import test from "node:test"; import {createServer, request as httpsRequest} from "node:https"; import type {AddressInfo} from "node:net"; -import {writeFile} from "node:fs/promises"; +import {mkdir, writeFile} from "node:fs/promises"; +import {join} from "node:path"; +import {execFile} from "node:child_process"; +import {promisify} from "node:util"; import {Client} from "pg"; import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; import {RemoteAuthorityService} from "../../loopx/control_plane/coordination/remote_authority_service.ts"; @@ -12,12 +15,86 @@ import {PostgreSqlAuthorityStore, rotatePostgreSqlAuthorityStoreIdentity, type P import {remoteAuthorityOperationId, type RemoteAuthorityRequest} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; import {canonicalAuthoritySha256} from "../../loopx/control_plane/coordination/authority_store_codec.ts"; import {postgresRemoteFixture, claimRequest, credentials, serviceIdentity, servicePolicy} from "./remote_authority_fixture.ts"; +import {openLocalAuthorityStore} from "../../loopx/control_plane/coordination/local_authority_provider.ts"; +import {engageLegacyCoordinationWriterFence} from "../../loopx/control_plane/coordination/legacy_writer_fence.ts"; +import {authorityProjectionFixture} from "./authority_projection_fixture.ts"; +import {resolveTestPython} from "../../scripts/test-python.mjs"; import {registerRemoteAuthorityModeConformance} from "./remote_authority_mode_conformance.ts"; import {registerRemoteAuthorityLifecycleConformance} from "./remote_authority_lifecycle_conformance.ts"; const connectionString = process.env.LOOPX_TEST_POSTGRES_URL; registerRemoteAuthorityModeConformance(connectionString); registerRemoteAuthorityLifecycleConformance(connectionString); +test("quota CLI verifies the exact remote binding and never grants local execution", {skip: !connectionString}, async t => { + const fixture = await postgresRemoteFixture(t, connectionString!); + await fixture.seed(["todo_gateway_probe"], {todo_gateway_probe: "private-synthetic-note"}); + const initialHead = await fixture.store.loadAuthority(); + const running = await fixture.start(); + const python = resolveTestPython(); + const invoke = promisify(execFile); + const workspace = join(fixture.root, "workspace"), runtime = join(fixture.root, "runtime"); + const registry = join(workspace, "registry.json"), state = join(workspace, "STATE.md"); + await mkdir(workspace); + await writeFile(state, "---\nstatus: active\nupdated_at: 2026-01-01T00:00:00Z\n---\n# Synthetic Goal\n"); + // The candidate observes a remote authority alongside its established + // private host state. It does not auto-initialize or migrate a Goal. + const local = await openLocalAuthorityStore(runtime, "goal-one"); + const localHead = authorityProjectionFixture("goal-one", [], [], "native", {handoff_mode: "soft_claim"}); + const localSeed = await local.commitAuthority({operation_id: "gateway-local-fixture", + expected_provider_revision: null, next_projection: localHead, events: [], receipts: []}); + assert.equal(localSeed.status, "applied"); + if (localSeed.status !== "applied") throw new Error("local fixture did not initialize"); + const fence = await engageLegacyCoordinationWriterFence({schema_version: "loopx_legacy_coordination_writer_fence_engage_request_v0", + runtime_root: runtime, goal_id: "goal-one", state_path: state, fence: { + schema_version: "loopx_legacy_coordination_writer_fence_v0", state: "engaged", goal_id: "goal-one", + fence_id: "gateway-local-fixture", source_version: "gateway-local-fixture", + source_projection_sha256: canonicalAuthoritySha256(localHead), expected_shadow_provider_revision: localSeed.provider_revision, + }}); + assert.equal(fence.status, "applied"); + const cfg = {enabled: true, endpoint: running.endpoint, actor_id: "actor-a", + tenant_id: "tenant-one", store_identity: serviceIdentity, probe_todo_id: "todo_gateway_probe", ca_file: fixture.certPath}; + const env: NodeJS.ProcessEnv = {...process.env, LOOPX_CONTEXT_DELIVERY: "off", LOOPX_AUTHORITY_CREDENTIAL: credentials.a}; + delete env.LOOPX_TEST_POSTGRES_URL; + const writeBinding = async (overrides: JsonObject = {}) => writeFile(registry, JSON.stringify({ + schema_version: "0.1", common_runtime_root: runtime, + goals: [{id: "goal-one", status: "active", repo: workspace, state_file: "STATE.md", + coordination: {agent_model: "peer_v1", registered_agents: ["actor-a", "actor-b"]}, + quota: {compute: 1}, remote_authority: {...cfg, ...overrides}}], + })); + const args = ["-m", "loopx.cli", "--registry", registry, "--runtime-root", runtime, "--format", "json"]; + const scope = ["--goal-id", "goal-one", "--agent-id", "actor-a"]; + async function check(expected: string, credential = credentials.a) { + const {stdout} = await invoke(python, [...args, "quota", "should-run", "--runtime-profile", "outer_controller", ...scope], + {env: {...env, LOOPX_AUTHORITY_CREDENTIAL: credential}, timeout: 25_000, maxBuffer: 1024 * 1024}); + const decision = JSON.parse(stdout); + assert.equal(decision.remote_gateway?.status, expected, JSON.stringify({ + reason: decision.reason, code: decision.reason_code, fields: Object.keys(decision), + })); + assert.equal(decision.effective_action, "blocked_wait"); + assert.equal(decision.should_run, false); + assert.equal(decision.normal_delivery_allowed, false); + assert.equal(decision.interaction_contract.agent_channel.must_attempt, false); + assert.equal(decision.interaction_contract.agent_channel.delivery_allowed, false); + assert.equal(decision.interaction_contract.cli_channel.spend_allowed_now, false); + assert(!decision.selected_todo); + for (const value of [cfg.endpoint, cfg.tenant_id, cfg.store_identity, cfg.ca_file, credential, "private-synthetic-note"]) { + assert(!stdout.includes(value)); + } + } + await writeBinding(); await check("verified"); + await check("unavailable", credentials.b); // Valid credential, wrong actor authorization. + for (const binding of [{tenant_id: "wrong-tenant"}, {store_identity: `postgresql:${"f".repeat(32)}`}, + {ca_file: null}, {probe_todo_id: "todo_missing"}, {actor_id: "actor-b"}]) { + await writeBinding(binding); await check("unavailable"); + } + await writeBinding(); + await fixture.writePolicy({...servicePolicy(), principals: []}); await check("unavailable"); + await fixture.writePolicy(); await check("verified"); + await running.stop(); await check("unavailable"); + await fixture.start(running.port); await check("verified"); + assert.deepEqual(await fixture.store.loadAuthority(), initialHead, "gateway observations never change the remote head"); +}); + test("remote authority real TLS/PostgreSQL task journey and recovery", {skip: !connectionString}, async t => { const fixture = await postgresRemoteFixture(t, connectionString!); let running = await fixture.start(); diff --git a/tests/control_plane_ts/remote_authority.test.ts b/tests/control_plane_ts/remote_authority.test.ts index 04c1fa2614..e65dae3eb3 100644 --- a/tests/control_plane_ts/remote_authority.test.ts +++ b/tests/control_plane_ts/remote_authority.test.ts @@ -10,6 +10,7 @@ import {join} from "node:path"; import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; import {decodeRemoteAuthorityRequest, decodeRemoteAuthorityResponse, remoteAuthorityOperationId, remoteAuthorityResponse} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; +import {canonicalAuthoritySha256} from "../../loopx/control_plane/coordination/authority_store_codec.ts"; import {decodeRemoteAuthorityPolicy, remoteAuthorityPolicyAdmission, readRemoteAuthorityPolicy} from "../../loopx/control_plane/coordination/remote_authority_policy.ts"; import {RemoteAuthorityService} from "../../loopx/control_plane/coordination/remote_authority_service.ts"; import {callRemoteAuthority, createRemoteAuthorityServer} from "../../loopx/control_plane/coordination/remote_authority_http.ts"; @@ -209,7 +210,7 @@ test("complete, block, defer, todo_add and snapshot request decoding validates t goal_id: "goal-one", actor_id: "actor-a", store_identity: `postgresql:${"c".repeat(32)}`, - todo_id: "todo_0aaf8c3edda601ecc3c5b8c0", + todo_id: "todo_lifecycle_fixture", }; // Complete @@ -224,6 +225,8 @@ test("complete, block, defer, todo_add and snapshot request decoding validates t assert.deepEqual(decodeRemoteAuthorityRequest(completeReq), completeReq); assert.throws(() => decodeRemoteAuthorityRequest({...completeReq, evidence: ""})); assert.throws(() => decodeRemoteAuthorityRequest({...completeReq, claim_request_id: ""})); + assert.throws(() => decodeRemoteAuthorityRequest({...completeReq, claim_request_id: 12})); + assert.throws(() => decodeRemoteAuthorityRequest({...completeReq, request_id: null})); // Block const blockReq = { @@ -231,11 +234,12 @@ test("complete, block, defer, todo_add and snapshot request decoding validates t operation: "block" as const, request_id: "req-block-1", reason: "waiting on upstream schema", - resume_when: "schema_applied", + resume_when: "todo_done:todo_schema_fixture", note: "paused temporarily", }; assert.deepEqual(decodeRemoteAuthorityRequest(blockReq), blockReq); assert.throws(() => decodeRemoteAuthorityRequest({...blockReq, reason: ""})); + assert.throws(() => decodeRemoteAuthorityRequest({...blockReq, resume_when: "schema_applied"})); // Defer const deferReq = { @@ -261,7 +265,7 @@ test("complete, block, defer, todo_add and snapshot request decoding validates t }; assert.deepEqual(decodeRemoteAuthorityRequest(addReq), addReq); assert.throws(() => decodeRemoteAuthorityRequest({...addReq, priority: "P9"})); - assert.throws(() => decodeRemoteAuthorityRequest({...addReq, role: "invalid" as any})); + assert.throws(() => decodeRemoteAuthorityRequest({...addReq, role: "invalid"})); // Snapshot (todo_id is optional) const snapshotGoal = { @@ -273,8 +277,9 @@ test("complete, block, defer, todo_add and snapshot request decoding validates t operation: "snapshot" as const, }; assert.deepEqual(decodeRemoteAuthorityRequest(snapshotGoal), snapshotGoal); - const snapshotTodo = {...snapshotGoal, todo_id: "todo_0aaf8c3edda601ecc3c5b8c0"}; + const snapshotTodo = {...snapshotGoal, todo_id: "todo_lifecycle_fixture"}; assert.deepEqual(decodeRemoteAuthorityRequest(snapshotTodo), snapshotTodo); + assert.throws(() => decodeRemoteAuthorityRequest({...snapshotTodo, todo_id: false})); }); test("policy admits complete, block, defer, todo_add and snapshot grants and rejects ungranted operations", () => { @@ -326,16 +331,25 @@ test("decodeRemoteAuthorityResponse validates snapshot and complete response pay todo_id: "todo_one", }; const snapshotReq = decodeRemoteAuthorityRequest({...base, operation: "snapshot"}); - const snapResponse = remoteAuthorityResponse(snapshotReq, "execute", { - status: "snapshot", - provider_revision: `${base.store_identity}:10`, - store_identity: base.store_identity, - goal_id: base.goal_id, - receipt_sha256: "a".repeat(64), - todos: [{todo_id: "todo_one", status: "open", role: "agent"}], - leases: [], - }); - assert.deepEqual(decodeRemoteAuthorityResponse(snapResponse, snapshotReq, "execute"), snapResponse); + const snapResponse = (todos: JsonObject[], leases: JsonObject[] = []) => { + const facts = {provider_revision: `${base.store_identity}:10`, store_identity: base.store_identity, + goal_id: base.goal_id, todos, leases}; + return remoteAuthorityResponse(snapshotReq, "execute", {status: "snapshot", ...facts, + receipt_sha256: canonicalAuthoritySha256({schema_version: "loopx_goal_snapshot_receipt_v0", + tenant_id: base.tenant_id, ...facts})}); + }; + const todo = {todo_id: "todo_one", status: "open", role: "agent"}; + const valid = snapResponse([todo]); + assert.deepEqual(decodeRemoteAuthorityResponse(valid, snapshotReq, "execute"), valid); + assert.throws(() => decodeRemoteAuthorityResponse({...valid, receipt_sha256: "a".repeat(64)}, snapshotReq, "execute")); + assert.throws(() => decodeRemoteAuthorityResponse(snapResponse([{...todo, todo_id: "todo_wrong"}]), snapshotReq, "execute")); + assert.throws(() => decodeRemoteAuthorityResponse(snapResponse([todo, todo]), snapshotReq, "execute")); + const lease = {todo_id: "todo_one", goal_id: base.goal_id, owner: base.actor_id, status: "released", + version: 1, lease_epoch: 1}; + assert.throws(() => decodeRemoteAuthorityResponse(snapResponse([todo], [{...lease, goal_id: "wrong-goal"}]), snapshotReq, "execute")); + assert.throws(() => decodeRemoteAuthorityResponse(snapResponse([], [lease]), snapshotReq, "execute")); + assert.throws(() => decodeRemoteAuthorityResponse(snapResponse([todo], [{...lease, idempotency_key: "must-not-leak"}]), snapshotReq, "execute")); + assert.throws(() => decodeRemoteAuthorityResponse(valid, snapshotReq, "recover")); const completeReq = decodeRemoteAuthorityRequest({ ...base, operation: "complete", request_id: "req-c", claim_request_id: "req-cl", evidence: "proof", @@ -347,11 +361,15 @@ test("decodeRemoteAuthorityResponse validates snapshot and complete response pay result: { status: "applied", changed: true, - completed: true, + completed_at: "2026-12-01T12:00:00Z", + completion_receipt_id: "receipt-1", todo_id: "todo_one", provider_revision: `${base.store_identity}:11`, cursor: "11", }, }); assert.deepEqual(decodeRemoteAuthorityResponse(compResponse, completeReq, "execute"), compResponse); + const incomplete = structuredClone(compResponse); + delete (incomplete.result as JsonObject).completion_receipt_id; + assert.throws(() => decodeRemoteAuthorityResponse(incomplete, completeReq, "execute")); }); diff --git a/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts b/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts index 5ff52c1f97..ef1c16b9ea 100644 --- a/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts +++ b/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts @@ -191,4 +191,31 @@ export function registerRemoteAuthorityLifecycleConformance(connectionString: st assert.equal(defer.code, 0); assert.equal((defer.value.result as JsonObject).status, "applied"); }); + + test("snapshot is bounded, targeted, and read only across independent clients", {skip: !connectionString}, async t => { + const fixture = await postgresRemoteFixture(t, connectionString!); + const running = await fixture.start(); + const ids = Array.from({length: 24}, (_, i) => `todo_snapshot_${String(i).padStart(2, "0")}`); + await fixture.seed(ids, Object.fromEntries(ids.map(id => [id, "x".repeat(3100)]))); + const initialHead = await fixture.store.loadAuthority(); + const {request_id: _, todo_id, ...base} = claimRequest("a", ids[0]); + const wide: RemoteAuthorityRequest = {...base, operation: "snapshot"}; + const tooLarge = await fixture.client(running.endpoint, "a", wide); + assert.equal(tooLarge.code, 1); + assert.equal(tooLarge.value.status, "unavailable"); + assert.equal(tooLarge.value.reason_code, "remote_authority_response_too_large"); + + const targeted: RemoteAuthorityRequest = {...wide, todo_id}; + const first = await fixture.client(running.endpoint, "a", targeted); + const second = await fixture.client(running.endpoint, "b", {...targeted, actor_id: "actor-b"}); + assert.equal(first.code, 0); assert.equal(second.code, 0); + assert.deepEqual((first.value.todos as JsonObject[]).map(todo => todo.todo_id), [todo_id]); + assert.equal(first.value.receipt_sha256, second.value.receipt_sha256); + const wrongGoal = await fixture.client(running.endpoint, "a", {...targeted, goal_id: "another-goal"}); + assert.equal(wrongGoal.code, 1); assert.equal(wrongGoal.value.status, "rejected"); + const query = await fixture.client(running.endpoint, "a", targeted, true); + assert.equal(query.code, 1); assert.equal(query.value.status, "rejected"); + assert.deepEqual(await fixture.store.loadAuthority(), initialHead); + }); + } From c52aef1561fe4125f26d8be85ca03bfe4daa49cf Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:41:54 +0800 Subject: [PATCH 4/5] fix(coordination): preserve existing successors on remote completion Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../remote-authority-task-service-v0.md | 10 +- .../coordination/remote_authority_contract.ts | 7 +- .../coordination/remote_authority_service.ts | 15 ++- .../remote_authority_lifecycle_conformance.ts | 96 ++++++++++++++++++- 4 files changed, 120 insertions(+), 8 deletions(-) diff --git a/docs/reference/remote-authority-task-service-v0.md b/docs/reference/remote-authority-task-service-v0.md index 3d5241da40..3e9ec583c2 100644 --- a/docs/reference/remote-authority-task-service-v0.md +++ b/docs/reference/remote-authority-task-service-v0.md @@ -71,7 +71,7 @@ canonical `todo_` syntax. Unknown fields and operations are rejected. | `progress` | `request_id`, `claim_request_id`, `note` | Update a note using the original claim and current execution fence. | | `renew` | `idempotency_key`, `expected_version`, `ttl_seconds` | Renew the current lease through the existing lifecycle owner. | | `release` | `idempotency_key`, `expected_version` | Release execution; this does not clear the task's assignment or complete it. | -| `complete` | `request_id`, `claim_request_id`, `evidence`, `note?` | Record an evidence reference and complete the task through the existing completion and lease owners. | +| `complete` | `request_id`, `claim_request_id`, `evidence`, `note?`, `no_followup?` | Complete the task with evidence, preserve a recorded successor and release the execution lease. | | `block` | `request_id`, `reason`, `claim_request_id?`, `resume_when?`, `note?` | Pause task as blocked with explicit reason, releasing execution lease. | | `defer` | `request_id`, `reason`, `resume_when`, `claim_request_id?`, `note?` | Defer task with resumption condition and rationale, releasing execution lease. | | `todo_add` | `request_id`, `text`, `priority?`, `role?`, ... | Create a canonical task under policy grant and tenant fence. | @@ -101,6 +101,14 @@ the principal/tenant/Goal/actor scope. Reusing one with different intent fails. Renew/release retain the canonical lease operation identity. Their receipts also verify the admitted principal, actor and request digest before reuse. +Ordinary `complete` follows the Todo's continuation policy. A recorded successor +stays linked. With neither a successor nor a terminal declaration, completion +records `active_goal`. Supply `no_followup: true` only +for an explicit terminal completion. A recorded successor conflicts with that +intent, so the operation fails before writing a completion receipt or releasing +the lease. Existing successful completions replay their original continuation +when clients supply the same request. + For `block` and `defer`, use the existing Todo resume condition syntax, such as `todo_done:todo_dependency` or `resume_at:2026-12-01T00:00:00Z`. A reason alone can block a task; deferral requires a valid condition. diff --git a/loopx/control_plane/coordination/remote_authority_contract.ts b/loopx/control_plane/coordination/remote_authority_contract.ts index 89d15ad74c..d37ef5f7bb 100644 --- a/loopx/control_plane/coordination/remote_authority_contract.ts +++ b/loopx/control_plane/coordination/remote_authority_contract.ts @@ -33,7 +33,7 @@ export type RemoteAuthorityRequest = Base & ( | {operation: "progress"; todo_id: string; request_id: string; claim_request_id: string; note: string} | {operation: "renew"; todo_id: string; idempotency_key: string; expected_version: number; ttl_seconds: number} | {operation: "release"; todo_id: string; idempotency_key: string; expected_version: number} - | {operation: "complete"; todo_id: string; request_id: string; claim_request_id: string; evidence: string; note?: string} + | {operation: "complete"; todo_id: string; request_id: string; claim_request_id: string; evidence: string; note?: string; no_followup?: boolean} | {operation: "block"; todo_id: string; request_id: string; claim_request_id?: string; reason: string; resume_when?: string; note?: string} | {operation: "defer"; todo_id: string; request_id: string; claim_request_id?: string; reason: string; resume_when: string; note?: string} | { @@ -77,7 +77,7 @@ const OPTIONAL_KEYS: Record = { progress: [], renew: [], release: [], - complete: ["note"], + complete: ["note", "no_followup"], block: ["claim_request_id", "resume_when", "note"], defer: ["claim_request_id", "note"], todo_add: ["role", "priority", "task_class", "action_kind", "task_repository", "required_write_scopes", "claimed_by", "note"], @@ -121,6 +121,9 @@ export function decodeRemoteAuthorityRequest(raw: unknown): RemoteAuthorityReque if ("note" in value && (typeof value.note !== "string" || !value.note.trim() || Buffer.byteLength(value.note) > 4096)) { throw new TypeError("Note requires a bounded non-empty string"); } + if ("no_followup" in value && typeof value.no_followup !== "boolean") { + throw new TypeError("no_followup must be a boolean"); + } if ("evidence" in value && (typeof value.evidence !== "string" || !value.evidence.trim() || Buffer.byteLength(value.evidence) > 16 * 1024)) { throw new TypeError("Evidence requires a bounded non-empty string"); } diff --git a/loopx/control_plane/coordination/remote_authority_service.ts b/loopx/control_plane/coordination/remote_authority_service.ts index 6adce348e1..75f045e7e0 100644 --- a/loopx/control_plane/coordination/remote_authority_service.ts +++ b/loopx/control_plane/coordination/remote_authority_service.ts @@ -7,7 +7,7 @@ import type {PostgreSqlAuthorityDatabase} from "./postgresql_authority_store.ts" import {executeCoordinationTodoClaim} from "./todo_claim.ts"; import {executeCoordinationTodoUpdate} from "./todo_update.ts"; import {executeCoordinationTodoCreate} from "./todo_create.ts"; -import {executeCoordinationTodoTerminalLifecycle} from "./todo_terminal_lifecycle.ts"; +import {executeCoordinationTodoTerminalLifecycle, type CoordinationTodoTerminalLifecycleInput} from "./todo_terminal_lifecycle.ts"; import {executeCanonicalTaskLeaseLifecycle} from "./task_lease_lifecycle.ts"; import {leaseVersion, leaseEpoch} from "../work_items/task_lease_acquire.ts"; import {TODO_DOMAIN_ITEM_SCHEMA} from "./coordination_state_contract.ts"; @@ -225,18 +225,25 @@ export class RemoteAuthorityService { patch: {note: request.note}, clear_fields: []}); else if (request.operation === "complete") { const claimOpId = remoteAuthorityOperationId(request, admission.principal_id, request.claim_request_id); - result = await executeCoordinationTodoTerminalLifecycle(store, { + const input: CoordinationTodoTerminalLifecycleInput = { goal_id: request.goal_id, todo_id: request.todo_id, command: "complete", expected_role: "agent", actor_agent_id: request.actor_id, registered_agents: admission.registered_agents, lifecycle_grants: [], authority_reason: null, decision_outcome: null, operation_identity: {kind: "explicit", operation_id: operationId}, claim_operation_id: claimOpId, lease_idempotency_key: null, lease_expected_version: null, - allow_user_gate_auto_acquire: false, requested_no_followup: true, + allow_user_gate_auto_acquire: false, requested_no_followup: request.no_followup === true, requested_completion_turn_key: null, requested_completion_identity_source: null, linked_successor_todo_ids: [], successor_intents: [], note: request.note ?? null, evidence: request.evidence, reason: null, clear_claim: true, validation_declaration: null, validation_receipt: null, completion_policy_request: null, dry_run: false, now, - }); + }; + result = await executeCoordinationTodoTerminalLifecycle(store, input); + // Older complete requests implied no_followup. Use that intent only to + // replay an already committed receipt for the exact authenticated request. + if (previous?.status === "found" && request.no_followup === undefined && + result.reason_code === "coordination_operation_identity_mismatch") { + result = await executeCoordinationTodoTerminalLifecycle(store, {...input, requested_no_followup: true}); + } if (result.status === "applied") result.completed = true; } else if (request.operation === "block" || request.operation === "defer") { const claimOpId = request.claim_request_id diff --git a/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts b/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts index ef1c16b9ea..37f8271eac 100644 --- a/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts +++ b/tests/control_plane_ts/remote_authority_lifecycle_conformance.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; import {prepareCoordinationProjectionCommit, type CoordinationProjectionMutation} from "../../loopx/control_plane/coordination/coordination_projection.ts"; +import {executeCoordinationTodoTerminalLifecycle} from "../../loopx/control_plane/coordination/todo_terminal_lifecycle.ts"; import {executeCanonicalTaskLeaseLifecycle} from "../../loopx/control_plane/coordination/task_lease_lifecycle.ts"; import type {RemoteAuthorityRequest} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; import {postgresRemoteFixture, claimRequest, serviceIdentity} from "./remote_authority_fixture.ts"; @@ -90,10 +91,89 @@ export function registerRemoteAuthorityLifecycleConformance(connectionString: st assert.deepEqual(await fixture.store.loadAuthority(), committed, "replay and recovery retain the original no-op receipt"); }); + test("remote completion keeps a recorded successor available to the next actor", {skip: !connectionString}, async t => { + const fixture = await postgresRemoteFixture(t, connectionString!); + const running = await fixture.start(); + await fixture.seed(["todo_remote", "todo_local", "todo_next_remote", "todo_next_local"]); + const seed = await fixture.store.loadAuthority(); + assert.equal(seed.status, "loaded"); + if (seed.status !== "loaded") return; + const links: Record = { + todo_remote: {successor_todo_ids: ["todo_next_remote"]}, + todo_local: {successor_todo_ids: ["todo_next_local"]}, + todo_next_remote: {resume_when: "todo_done:todo_remote"}, + todo_next_local: {resume_when: "todo_done:todo_local"}, + }; + const mutations: CoordinationProjectionMutation[] = (seed.head.todos as JsonObject[]).map(todo => ({ + kind: "todo_upsert", todo: {...todo, ...links[String(todo.todo_id)]}, + })); + const linked = await fixture.store.commitAuthority(prepareCoordinationProjectionCommit({goal_id: "goal-one", + operation_id: "existing-successors", expected_provider_revision: seed.provider_revision, projection: seed.head, mutations})); + assert.equal(linked.status, "applied"); + + const client = (actor: "a" | "b", request: RemoteAuthorityRequest, recover = false) => + fixture.client(running.endpoint, actor, request, recover); + const localClaim = await client("a", claimRequest("a", "todo_local")); + assert.equal(localClaim.code, 0); + const control = await executeCoordinationTodoTerminalLifecycle(fixture.store, { + goal_id: "goal-one", todo_id: "todo_local", command: "complete", + expected_role: "agent", actor_agent_id: "actor-a", registered_agents: ["actor-a", "actor-b"], + lifecycle_grants: [], authority_reason: null, decision_outcome: null, + operation_identity: {kind: "explicit", operation_id: "finish-local"}, + claim_operation_id: String((localClaim.value.result as JsonObject).claim_operation_id), + lease_idempotency_key: null, lease_expected_version: null, allow_user_gate_auto_acquire: false, + requested_no_followup: false, requested_completion_turn_key: null, requested_completion_identity_source: null, + linked_successor_todo_ids: [], successor_intents: [], note: null, evidence: "verified fixture result", + reason: null, clear_claim: true, validation_declaration: null, validation_receipt: null, + completion_policy_request: null, dry_run: false, now: new Date(), + }); + assert.equal(control.status, "applied"); + assert.equal(control.completion_continuation, "successor"); + + const next = claimRequest("b", "todo_next_remote"); + const pending = await client("b", next); + assert.equal(pending.code, 1); + assert.equal((pending.value.result as JsonObject).reason_code, "todo_dependency_pending"); + const {request_id: claim_request_id, ...scope} = claimRequest("a", "todo_remote"); + assert.equal((await client("a", {...scope, request_id: claim_request_id})).code, 0); + const command = {...scope, operation: "complete" as const, request_id: "finish-remote", + claim_request_id, evidence: "verified fixture result"}; + const finished = await client("a", command); + assert.equal(finished.code, 0, JSON.stringify(finished.value)); + assert.equal((finished.value.result as JsonObject).status, "applied"); + const receipt = (finished.value.result as JsonObject).completion_receipt_id; + + const committed = await fixture.store.loadAuthority(); + assert.equal(committed.status, "loaded"); + if (committed.status !== "loaded") return; + const todo = (committed.head.todos as JsonObject[]).find(row => row.todo_id === command.todo_id); + assert.equal(todo?.completion_continuation, control.completion_continuation); + assert.deepEqual(todo?.successor_todo_ids, [next.todo_id]); + assert.equal(todo?.completion_receipt_id, receipt); + const replay = await client("a", command); + assert.equal(replay.code, 0); + assert.equal((replay.value.result as JsonObject).status, "replayed"); + const recovered = await client("a", command, true); + assert.equal(recovered.code, 0); + assert.equal(recovered.value.historical_only, true); + assert.equal((recovered.value.result as JsonObject).completion_receipt_id, receipt); + const changedIntent = await client("a", {...command, no_followup: true}); + assert.equal(changedIntent.code, 1); + assert.equal(changedIntent.value.reason_code, "remote_authority_operation_mismatch"); + assert.deepEqual(await fixture.store.loadAuthority(), committed); + + const {request_id: _, ...reader} = claimRequest("b", command.todo_id); + const observed = await client("b", {...reader, operation: "read"}); + assert.equal(observed.code, 0); + assert.equal((observed.value.todo as JsonObject).status, "done"); + assert.equal((observed.value.lease as JsonObject).status, "released"); + assert.equal((await client("b", next)).code, 0); + }); + test("remote authority full lifecycle: snapshot, todo_add, complete, block, and defer", {skip: !connectionString}, async t => { const fixture = await postgresRemoteFixture(t, connectionString!); const running = await fixture.start(); - await fixture.seed(["todo_for_complete", "todo_for_block", "todo_for_defer"]); + await fixture.seed(["todo_for_complete", "todo_for_terminal", "todo_for_block", "todo_for_defer"]); const baseScope = { schema_version: "loopx_remote_authority_request_v0" as const, @@ -150,6 +230,20 @@ export function registerRemoteAuthorityLifecycleConformance(connectionString: st assert.equal((comp.value.result as JsonObject).status, "applied"); assert.equal((comp.value.result as JsonObject).completed, true); + const {request_id: terminalClaimId, ...terminalScope} = claimRequest("a", "todo_for_terminal"); + assert.equal((await fixture.client(running.endpoint, "a", {...terminalScope, request_id: terminalClaimId})).code, 0); + const terminal = await fixture.client(running.endpoint, "a", { + ...terminalScope, operation: "complete", request_id: "finish-terminal", claim_request_id: terminalClaimId, + evidence: "verified fixture result", no_followup: true, + }); + assert.equal(terminal.code, 0); + const completedHead = await fixture.store.loadAuthority(); + assert.equal(completedHead.status, "loaded"); + if (completedHead.status !== "loaded") return; + const completedTodos = completedHead.head.todos as JsonObject[]; + assert.equal(completedTodos.find(row => row.todo_id === "todo_for_complete")?.completion_continuation, "active_goal"); + assert.equal(completedTodos.find(row => row.todo_id === "todo_for_terminal")?.completion_continuation, "no_followup"); + // 4. Claim -> Block const claimBlock = await fixture.client(running.endpoint, "a", { ...baseScope, From 6e38c725a90fcd628f9445a5db6807bcfec02dcd Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:21:11 +0800 Subject: [PATCH 5/5] test(remote-authority): prove ordinary admission stays local Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- tests/control_plane/test_remote_authority_gateway.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/control_plane/test_remote_authority_gateway.py b/tests/control_plane/test_remote_authority_gateway.py index 591aeaf34b..a597f39431 100644 --- a/tests/control_plane/test_remote_authority_gateway.py +++ b/tests/control_plane/test_remote_authority_gateway.py @@ -30,8 +30,17 @@ def read(): assert decision["ok"] is True return tuple(decision.get(key) for key in ("decision", "effective_action", "should_run", "reason")) + def read_local(goal): + decision = cli( + "quota", "should-run", "--runtime-profile", "outer_controller", + "--goal-id", goal, "--agent-id", SCOPED_AGENT_ID, + ) + assert "remote_gateway" not in decision + return tuple(decision.get(key) for key in ("decision", "effective_action", "should_run")) + local = read() assert local[:3] == ("skip", "throttled_skip", False), local + assert read_local("half-speed") == ("run", "normal_run", True) document = json.loads(registry.read_text()) cfg = {"enabled": True, "endpoint": "https://127.0.0.1:1", "tenant_id": "other-synthetic-tenant", "actor_id": SCOPED_AGENT_ID, "store_identity": "postgresql:" + "a" * 32, "probe_todo_id": "todo_probe"} @@ -42,6 +51,7 @@ def read(): goal["remote_authority"] = cfg registry.write_text(json.dumps(document), encoding="utf-8") assert read() == local + assert read_local("half-speed") == ("run", "normal_run", True) other = cli("heartbeat-prompt", "--full", "--codex-app", "--goal-id", "full-speed", "--agent-id", SCOPED_AGENT_ID)