From 601ebcd6133bc8003b81f9691fe694bbaa468b79 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:01:18 +0800 Subject: [PATCH 1/5] fix(codex relay): fence incomplete model requests before drain Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../loopx-codex-provider-routing/NATIVE.md | 16 ++- .../openai_relay.py | 65 +++++++-- .../tests/test_provider_lifecycle.py | 125 +++++++++++++++++- 3 files changed, 187 insertions(+), 19 deletions(-) diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index 10fc6aba04..da8c8073f4 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -91,8 +91,9 @@ configured steward route as well as the worker, without creating credentials. The model transport's ordinary `ensure` also performs reconciliation for its own route. An authenticated new-protocol daemon with a different loaded build stops new admissions, drains -already accepted requests and releases its existing owner lock before one new -daemon starts. The model, route credentials, key policy and message queue stay +already admitted model requests, and releases its existing owner lock before one +new daemon starts. It closes connections still sending headers or request bodies. +The model, route credentials, key policy and message queue stay with their existing owners. A bounded timeout returns `api_relay_drain_pending`; read status later instead of forcing exit or replaying a Turn. Old `/health` response bytes remain compatible, but do not qualify version adoption. @@ -165,15 +166,18 @@ is presented to OpenAI and rejected with `401 invalid_api_key`. Installers that add chat transports should write the relay endpoint into each provider definition and verify it on readback. -A local authenticated transport sends the original selected model and Responses -body to OpenAI. An explicit upstream quota rejection, 401, 403 or temporary 429 can +A local authenticated transport sends the original selected model and complete +Responses body to OpenAI. Model requests require one valid Content-Length and +are eligible for dispatch after every declared byte has arrived. A connection +that stalls while sending the request closes after ten seconds of inactivity. +An explicit upstream quota rejection, 401, 403 or temporary 429 can try the next configured key before output. Each key is attempted at most once per request, with typed cooldowns; model permission failures are scoped to that model. Unknown network effects, ordinary request/server failures and any started stream are not retried on another key. No job replay, model substitution or synthetic completion is added. There are no background paid probes. The transport survives -the Chat/App restart used for native handoff; disabling stops new admissions and -drains already accepted requests before its process exits. With the policy off, +the Chat/App restart used for native handoff. Disabling stops new admissions and +drains admitted model requests before its process exits. With the policy off, an existing subscription is never routed through this transport. Version 0.15.2 recognizes the official `credit_balance_exhausted` error in both diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py index 0ff0c41e98..5252717b75 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py @@ -28,6 +28,7 @@ SCHEMA = "codex_openai_api_relay_v1" API_SLOT = "openai-api" RUNTIME_SCHEMA = "codex_openai_api_relay_runtime_v1" +REQUEST_IDLE_TIMEOUT = 10 # Loaded once. Replacing an installed wheel cannot make its old daemon claim # the newly installed implementation. No credential or request data is hashed. BUILD = hashlib.sha256(b"".join(Path(__file__).with_name(name).read_bytes() for name in ( @@ -133,10 +134,33 @@ def __init__(self, owner: NativeAccounts, value: dict, *, opener=None): self.opener = opener or urllib.request.build_opener(NoRedirect()) self.instance_id = secrets.token_hex(16) self._admission_lock = threading.Lock() + self._receiving: set[socket.socket] = set() self.draining, self.active_requests = False, 0 self.started = time.monotonic() super().__init__(("127.0.0.1", value["port"]), RelayHandler) + def process_request(self, request, client_address): + with self._admission_lock: + self._receiving.add(request) + super().process_request(request, client_address) + + def shutdown_request(self, request): + with self._admission_lock: + self._receiving.discard(request) + super().shutdown_request(request) + + def server_close(self): + with self._admission_lock: + self.draining = True + # Stop reads on connections still outside model admission. Keep their + # write half open for the control acknowledgement and error responses. + for request in self._receiving: + try: + request.shutdown(socket.SHUT_RD) + except OSError: + pass + super().server_close() + def runtime(self): with self._admission_lock: return {"schema_version": RUNTIME_SCHEMA, "build": BUILD, @@ -163,6 +187,7 @@ def service_actions(self): class RelayHandler(BaseHTTPRequestHandler): server: RelayServer + timeout = REQUEST_IDLE_TIMEOUT def log_message(self, *_args): pass @@ -237,19 +262,10 @@ def do_POST(self): self._json(401, "relay_auth_required") return with self.server._admission_lock: - admitted = not self.server.draining - if admitted: - self.server.active_requests += 1 - if not admitted: + draining = self.server.draining + if draining: self._json(503, "api_relay_draining") return - try: - self._model_request() - finally: - with self.server._admission_lock: - self.server.active_requests -= 1 - - def _model_request(self): upstream_path = self.path.removeprefix("/steward") if self.consumer == "steward" else self.path if upstream_path not in ("/v1/responses", "/v1/responses/compact"): self._json(404, "unsupported_route") @@ -258,10 +274,18 @@ def _model_request(self): self._json(503, "api_fallback_disabled") return try: - length = int(self.headers.get("Content-Length", "0")) + lengths = self.headers.get_all("Content-Length", []) + if len(lengths) != 1 or "Transfer-Encoding" in self.headers: + raise ValueError() + declared = lengths[0].strip(" \t") + if not re.fullmatch(r"[0-9]+", declared): + raise ValueError() + length = int(declared) if not 1 <= length <= 32 * 1024 * 1024: raise ValueError() body = self.rfile.read(length) + if len(body) != length: + raise ValueError() value = json.loads(body) model = value["model"] if not isinstance(model, str) or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,160}", model) or model.startswith("sk-"): @@ -269,6 +293,23 @@ def _model_request(self): except (TypeError, ValueError, KeyError): self._json(400, "invalid_request") return + with self.server._admission_lock: + admitted = not self.server.draining + if admitted: + self.server._receiving.discard(self.connection) + self.server.active_requests += 1 + if not admitted: + self._json(503, "api_relay_draining") + return + try: + # The request idle timeout applies only before model admission. + self.connection.settimeout(None) + self._forward(upstream_path, body, model) + finally: + with self.server._admission_lock: + self.server.active_requests -= 1 + + def _forward(self, upstream_path, body, model): keys = self.server.owner.api_keys() candidates = keys.candidates(model) last_status, last_code = 429, "api_keys_unavailable" diff --git a/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py b/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py index 57994001f7..6a2bb89d26 100644 --- a/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py +++ b/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py @@ -1,9 +1,11 @@ """Real loopback admission/drain and real detached provider replacement.""" import importlib +from http.client import HTTPResponse import io import json import os from pathlib import Path +import socket import subprocess import sys import threading @@ -91,11 +93,132 @@ def open(self, call, timeout): client.join(5) +def _send_partial_request(server, request): + connection = socket.create_connection(("127.0.0.1", server.server_port), timeout=3) + connection.sendall(request) + return connection + + +@pytest.mark.parametrize("framing", ["short", "ambiguous", "transfer_encoded"]) +def test_invalid_model_request_framing_never_reaches_upstream(owner, framing): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + body = b'{"model":"selected-model","input":"fixture"}' + calls = [] + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + response = io.BytesIO(b'{"ok":true}') + response.status, response.headers = 200, {"Content-Type": "application/json"} + return response + + length = len(body) + 20 if framing == "short" else len(body) + extra_header = { + "short": b"", + "ambiguous": b"Content-Length: 1\r\n", + "transfer_encoded": b"Transfer-Encoding: chunked\r\n", + }[framing] + headers = (b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer worker\r\n" + + f"Content-Length: {length}\r\n".encode() + extra_header + b"\r\n") + server = relay.RelayServer(owner, {"port": 0, "token": "worker"}, opener=Upstream()) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + with _send_partial_request(server, headers + body) as client, HTTPResponse(client) as result: + client.shutdown(socket.SHUT_WR) + result.begin() + assert result.status == 400 + assert json.loads(result.read())["error"]["code"] == "invalid_request" + assert calls == [] + assert request(server, "/v1/responses", "worker", body=body) == (200, b'{"ok":true}') + assert calls == [body] + finally: + server.shutdown() + server.server_close() + thread.join(5) + + +def test_drain_releases_stalled_request_headers_and_body_without_waiting_for_clients(owner, monkeypatch): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + accepted_headers, accepted_body = threading.Event(), threading.Event() + original_parse, original_admitted = relay.RelayHandler.parse_request, relay.RelayHandler._admitted + + def parse_request(handler): + if handler.raw_requestline.startswith(b"GET /half"): + accepted_headers.set() + return original_parse(handler) + + def admitted(handler): + if handler.path == "/v1/responses": + accepted_body.set() + return original_admitted(handler) + + monkeypatch.setattr(relay.RelayHandler, "parse_request", parse_request) + monkeypatch.setattr(relay.RelayHandler, "_admitted", admitted) + calls = [] + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + raise AssertionError("Draining incomplete request reached the upstream") + + server = relay.RelayServer(owner, {"port": 0, "token": "worker", "control_token": "control"}, opener=Upstream()) + thread = threading.Thread(target=server.serve_forever) + closed = threading.Event() + def close_after_drain(): + thread.join() + server.server_close() + closed.set() + + closer = threading.Thread(target=close_after_drain) + thread.start() + closer.start() + try: + with ( + _send_partial_request(server, b"GET /half HTTP/1.1\r\nHost: lo"), + _send_partial_request(server, b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\n" + b"Authorization: Bearer worker\r\nContent-Length: 40\r\n\r\n{"), + ): + assert accepted_headers.wait(3) and accepted_body.wait(3) + live = json.loads(request(server, "/runtime", "worker")[1]) + assert request(server, "/control/drain", "control", body=b"", instance=live["instance_id"])[0] == 202 + assert closed.wait(3) + assert server.runtime()["active_requests"] == 0 + assert live["active_requests"] == 0 and calls == [] + finally: + server.shutdown() + closer.join(5) + + +def test_idle_body_read_closes_before_model_admission(owner, monkeypatch): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + monkeypatch.setattr(relay.RelayHandler, "timeout", 0.2) + calls = [] + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + raise AssertionError("Incomplete request reached the upstream") + + server = relay.RelayServer(owner, {"port": 0, "token": "worker"}, opener=Upstream()) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + incomplete = (b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer worker\r\n" + b"Content-Length: 40\r\n\r\n{") + with _send_partial_request(server, incomplete) as client: + assert client.recv(1) == b"" + assert calls == [] and server.runtime()["active_requests"] == 0 + finally: + server.shutdown() + server.server_close() + thread.join(5) + + def test_real_detached_old_build_replaced_once_with_same_routes_and_private_state(owner, monkeypatch): relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") life = importlib.import_module("loopx_codex_provider_routing.relay_lifecycle") native = importlib.import_module("loopx_codex_provider_routing.native") - import socket with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] From e50c0d8c1256881aeea36f3204a5dd27c52e0ec8 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:57:19 +0800 Subject: [PATCH 2/5] fix(codex relay): surface incomplete upstream responses Port the verified response framing fix to the request-admission branch. Original candidate 27bc3aee311cdfc6a00466ff9d3b58c7db36e01b. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../loopx-codex-provider-routing/NATIVE.md | 6 +- .../openai_relay.py | 64 +++++-- .../references/repair-patterns.md | 1 + tests/presentation/test_openai_relay.py | 159 +++++++++++++++++- 4 files changed, 211 insertions(+), 19 deletions(-) diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index da8c8073f4..f99ad6cedb 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -175,7 +175,11 @@ try the next configured key before output. Each key is attempted at most once pe request, with typed cooldowns; model permission failures are scoped to that model. Unknown network effects, ordinary request/server failures and any started stream are not retried on another key. No job replay, model substitution or synthetic -completion is added. There are no background paid probes. The transport survives +completion is added. For HTTP/1.1 clients, the relay streams bytes as they arrive +and leaves the response incomplete when an upstream transfer is truncated. A +complete HTTP transfer alone does not prove a completed model response. A +truncated rejection body or one larger than the 64 KiB inspection limit cannot +trigger another key. There are no background paid probes. The transport survives the Chat/App restart used for native handoff. Disabling stops new admissions and drains admitted model requests before its process exits. With the policy off, an existing subscription is never routed through this transport. diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py index 5252717b75..dc8da20248 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py @@ -8,6 +8,7 @@ import argparse import hashlib import hmac +from http.client import HTTPException from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os @@ -29,6 +30,7 @@ API_SLOT = "openai-api" RUNTIME_SCHEMA = "codex_openai_api_relay_runtime_v1" REQUEST_IDLE_TIMEOUT = 10 +ERROR_BODY_LIMIT = 65536 # Loaded once. Replacing an installed wheel cannot make its old daemon claim # the newly installed implementation. No credential or request data is hashed. BUILD = hashlib.sha256(b"".join(Path(__file__).with_name(name).read_bytes() for name in ( @@ -329,48 +331,84 @@ def _forward(self, upstream_path, body, model): try: response = self.server.opener.open(request, timeout=300) except urllib.error.HTTPError as error: - encoded = error.read(65537) + with error: + try: + encoded = error.read(ERROR_BODY_LIMIT + 1) + except (OSError, HTTPException): + encoded = None + remaining = getattr(error.fp, "length", None) + # A bounded read can silently reach early EOF, and a + # body larger than our limit cannot be verified in full. + if encoded is None or len(encoded) > ERROR_BODY_LIMIT or (remaining or 0) > 0: + self._json(502, "upstream_unavailable") + return try: - code = json.loads(encoded).get("error", {}).get("code") if len(encoded) <= 65536 else None + code = json.loads(encoded).get("error", {}).get("code") code = code if isinstance(code, str) and code.isascii() and code.replace("_", "").isalnum() else None except (TypeError, ValueError, AttributeError): code = None headers = {k.lower(): v for k, v in error.headers.items()} status = error.code - error.close() outcome = keys.record(row["id"], status=status, code=code, model=model, headers=headers) last_status, last_code = status, code or "upstream_rejected" if outcome["retry_next_key"]: continue self._json(status, last_code) return - except (OSError, HostError): + except (OSError, HTTPException, HostError): # Unknown upstream effects never authorize trying another key. self._json(502, "upstream_unavailable") return with response: headers = {k.lower(): v for k, v in response.headers.items()} - keys.record(row["id"], status=response.status, code=None, model=model, headers=headers) + + def observe(status: int, code: str | None = None): + try: + keys.record(row["id"], status=status, code=code, model=model, headers=headers) + except (OSError, ValueError, HostError, subprocess.SubprocessError): + # Once admitted, an observation failure must not + # replace or discard the upstream's response. + pass + + observe(response.status) + # urllib removes the upstream chunk framing. Use the decoded + # length when known; otherwise frame the HTTP/1.1 response + # so a read failure cannot look like clean EOF. + can_have_body = response.status not in (204, 304) + length = getattr(response, "length", None) + chunked = can_have_body and length is None and self.request_version == "HTTP/1.1" + if chunked: + self.protocol_version = "HTTP/1.1" self.send_response(response.status) self.send_header("Content-Type", headers.get("content-type", "text/event-stream")) self.send_header("Cache-Control", "no-store") + if chunked: + self.send_header("Transfer-Encoding", "chunked") + self.send_header("Connection", "close") + elif can_have_body and length is not None: + self.send_header("Content-Length", str(length)) self.end_headers() + if not can_have_body: + return # Once headers/output are admitted, disconnects and SSE errors # end this request. No key rotation, synthetic completion or job replay. - observer = _StreamOutcome() if headers.get("content-type", "").split(";", 1)[0].strip().lower() == "text/event-stream" else None + content_type = headers.get("content-type", "").split(";", 1)[0].strip().lower() + observer = _StreamOutcome() if content_type == "text/event-stream" else None try: read = getattr(response, "read1", response.read) while chunk := read(8192): outcome = observer.feed(chunk) if observer is not None else None if outcome is not None: - try: - keys.record(row["id"], status=outcome["status"], code=outcome["code"], - model=model, headers=headers) - except (OSError, ValueError, HostError, subprocess.SubprocessError): - pass - self.wfile.write(chunk) + observe(outcome["status"], outcome["code"]) + if chunked: + self.wfile.write(f"{len(chunk):x}\r\n".encode() + chunk + b"\r\n") + else: + self.wfile.write(chunk) + self.wfile.flush() + if chunked: + self.wfile.write(b"0\r\n\r\n") self.wfile.flush() - except (OSError, TimeoutError): + except (OSError, HTTPException): self.close_connection = True return self._json(last_status, last_code) diff --git a/skills/loopx-self-repair/references/repair-patterns.md b/skills/loopx-self-repair/references/repair-patterns.md index 5753a1733b..cae07c9a27 100644 --- a/skills/loopx-self-repair/references/repair-patterns.md +++ b/skills/loopx-self-repair/references/repair-patterns.md @@ -159,6 +159,7 @@ teaches a reusable control-plane lesson. | `future_monitor_heartbeat_liveness_overreach` | A valid future-scheduled monitor repeatedly enters `frontier_exhausted_monitor_lane` or `dead_monitor_repeat` even though it is not due and no real monitor execution occurred. | generated heartbeat guard command, trigger `current_time_iso`, quota `heartbeat_receipt`, monitor Todo `next_due_at`, run-history `quota_monitor_poll` rows, concrete Todo/target identity, and monitor mode. | Future wait state was collapsed into empty-frontier exhaustion, while distinct heartbeat turn ids were treated as monitor executions and lowered the dead-monitor threshold. | Add typed `future_monitor_wait` and `due_monitor_execution` frontier rules. Keep future monitors quiet/no-spend until due; count `dead_monitor_repeat` only from unchanged due/external executions carrying a concrete Todo or target identity. Heartbeat turn ids remain idempotency/liveness evidence and never reduce the real execution threshold. | | `replan_context_delivery_gap` | A replan obligation is visible, but a weak protocol-following model ignores or mis-executes a manual evidence-log read ritual, then spends turns debugging ACK/read order instead of choosing a new direction. | current obligation id, host-projected `replan_context_v0`, compact coverage ledger, delivery receipt, typed action packet, selected model tool action, semantic writeback receipt, and successor turn boundary. | Evidence access was modeled as an agent CLI obligation even though the control plane already owned the ledger; read execution and semantic use were conflated. A separate Todo-add then refresh ACK also let the model execute the successor in the replan turn. | Have the host prefetch and project bounded evidence coverage into quota. Treat the delivery receipt as context delivery only; require the model's real next action to produce an accepted typed semantic delta bound to the current obligation. When the delta is a runnable successor, bind the exact obligation on the Todo write itself and return `end_current_heartbeat`; do not require a second ACK or run the successor in that turn. Remove the standalone evidence command and generated read rituals. Reuse the typed replan context for the core Goal, dense scoped decision evidence, repetition reduction and exact history-reference reads. Keep replan selection independent of the routine display window, and validate novelty against complete available history even when the readable context is truncated; reject malformed schemas and unavailable references instead of treating them as empty evidence. Preserve the durable history and historical receipt decoder, not a compatibility command. Cover real quota→model action→Todo/refresh CLI behavior plus missing/wrong obligation identity. | | `machine_cli_capture_projection_gap` | A native hook or non-model JSON consumer treats context navigation as the effective decision and denies ready work because a top-level admission field is absent. | Original child argv/runtime profile; outer schema versus the same immutable captured Core decision; parent/child context settings; real Core ready, user-gate and zero-quota cases. | Model-facing context transport wrapped the formerly inline machine payload. A locator is not a policy decision; a locale/codec repair alone cannot restore this path. | Use the existing machine-output contract only for the non-model child process, retaining Core gates, identity, UTF-8 and environment isolation. Reuse the established per-child `LOOPX_CONTEXT_DELIVERY=off`; do not disable model context globally or interpret navigation `ok` as admission. Actual model consumers must follow the original readers and retain current bytes/digests. Validate ready, user-gate and zero-quota behavior with parent context on/off unchanged; distinguish source, installed-host and live-model qualification. | +| `relay_truncation_looks_like_clean_eof` | A Responses client receives an output or tool-item prefix followed by apparently clean HTTP EOF after the upstream body was truncated. | Observe upstream and downstream framing, exact prefix bytes, client status, and actual request count; compare a short declared length, missing final HTTP chunk, and a healthy completion. | The relay replaced a framed upstream response with close-delimited output, which erases the transport's incomplete-body signal. A syntactically complete error JSON can also be carried in an incomplete HTTP body. | Preserve a verified response length or terminate downstream chunked framing only after clean transfer. Do not manufacture a model terminal event or retry a request with unknown effects. Verify full and truncated paths over real loopback HTTP, keep already received bytes, and separately qualify explicit same-session recovery. | | `native_policy_test_consumer_navigation` | A source-owned policy smoke intentionally exercises native output but raises `KeyError` on an admission field, or mistakes navigation `ok` for decision readiness. | Original CLI result and the same current immutable recovery, unchanged semantic assertions, and full connection/own-Todo readback. | The test helper decoded only outer JSON and treated the model-facing navigation envelope as the policy. This differs from the machine-output host adapter in `machine_cli_capture_projection_gap`. | Reuse the existing test-only `read_cli_policy_payload` oracle to verify the current complete recovery; preserve inline results, nonzero exits, physical-file/digest checks and all original semantic assertions. Keep native output enabled for this qualification. Test generic/domain connection and the caller's own first Todo through the real CLI, plus failed/inline/native and damaged-reference cases. Isolate temporary runtimes and telemetry before child invocation and retire only authenticated test-owned runtimes before removing their directory. The oracle proves test observations, not model retention, execution authority or whole-Goal benefit. | | `replan_settlement_binding_projection_gap` | A replan packet and selected Todo are both visible, so an agent combines `--todo-id` with `--replan-obligation-id`, alternates between them after rejection, or performs the Todo-bound refresh but omits the matching spend. | Original quota receipt identity, selected Todo, semantic obligation id, interaction settlement plan, projected refresh/spend commands, and accepted semantic ACK. | The semantic obligation target and causal settlement identity were adjacent but unlabeled, while one boolean conflated direct obligation binding with whether a full settlement chain was required. | Keep exactly one causal settlement binding. For a turn-scoped executable plan, project a domain-local typed replan settlement contract that labels the selected Todo or Todo-less obligation as the binding and keeps the replan id as a separate semantic target when necessary. Todo-bound and directly bound replan writes both project material refresh followed by spend; never combine their CLI flags. Keep unscoped diagnostic reads compact and non-executable. Cover TS projection, hot-path envelope, real Todo-bound/Todo-less CLI chains, output budget, and in-flight delivery continuity. | | `native_goal_todoless_replan_reentry_gap` | A native visible Goal completes all advancement Todos, semantic closeout derives a Todo-less autonomous replan obligation, and the next unbound quota read projects a bare refresh with no Turn identity or spend chain. | Native Goal runtime profile, selected Todo absence, replan obligation id, interaction CLI actions before and after Turn creation, and settlement identity. | Native-Goal Turn re-entry recognized only selected Todos as actionable settlement bindings and ignored a valid Todo-less replan obligation. | Treat either a selected Todo or a valid replan obligation as a native-Goal settlement binding. When no Turn or settlement plan exists, project only the exact `quota should-run ... --begin-turn` re-entry. After execution, require the obligation-bound refresh and spend chain; keep generic unscoped diagnostic behavior unchanged. | diff --git a/tests/presentation/test_openai_relay.py b/tests/presentation/test_openai_relay.py index 2b993d31f9..aa85444b50 100644 --- a/tests/presentation/test_openai_relay.py +++ b/tests/presentation/test_openai_relay.py @@ -1,9 +1,11 @@ """Real loopback HTTP and private store, with synthetic upstream transport.""" import importlib +import http.client import io import json import os from pathlib import Path +import socketserver import threading import urllib.error import urllib.request @@ -45,13 +47,16 @@ def open(self, request, timeout): thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() body = {"model": "unchanged-selected-model", "input": "synthetic prompt", "stream": True, "tools": []} - def post(token="example-loopback-token", origin=None): + def post(token="example-loopback-token", origin=None, *, return_response=False): headers = {"Authorization": "Bearer " + token, "Content-Type": "application/json"} if origin: headers["Origin"] = origin request = urllib.request.Request(f"http://127.0.0.1:{server.server_port}/v1/responses", data=json.dumps(body).encode(), headers=headers) try: - with urllib.request.urlopen(request, timeout=10) as response: + response = urllib.request.urlopen(request, timeout=10) + if return_response: + return response + with response: return response.status, response.read() except urllib.error.HTTPError as error: return error.code, error.read() @@ -98,9 +103,153 @@ def read1(self, _size): return self.read() partial = b'data: {"type":"response.created"}\n\n' outcomes.append(Interrupted(partial)) - status, result = post() - assert status == 200 and result == partial and len(calls) == 1 - assert b"response.completed" not in result + with pytest.raises(http.client.IncompleteRead) as caught: + post() + assert caught.value.partial == partial and len(calls) == 1 + + +def test_bad_upstream_headers_return_one_unknown_failure(relay): + _, keys, calls, outcomes, post, _ = relay + outcomes.append(http.client.LineTooLong("upstream header")) + status, response = post() + assert status == 502 and json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + + +@pytest.mark.parametrize("oversized", [False, True]) +def test_unverified_rejection_body_cannot_trigger_another_key(relay, oversized): + _, keys, calls, outcomes, post, _ = relay + + class TruncatedRejection(io.BytesIO): + def read(self, *_args): + raise http.client.IncompleteRead(b'{"error":') + + outcomes.extend([ + urllib.error.HTTPError( + "https://api.openai.com/v1/responses", 429, "fixture", {}, + io.BytesIO(b"x" * 65537) if oversized else TruncatedRejection(), + ), + Response(b"must-not-be-used"), + ]) + status, response = post() + assert status == 502 and json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + + +def test_unavailable_key_observation_does_not_discard_admitted_response(relay, monkeypatch): + _, keys, calls, outcomes, post, _ = relay + + def unavailable(*_args, **_kwargs): + raise BlockingIOError("fixture observation lock") + + monkeypatch.setattr(type(keys), "record", unavailable) + stream = b'data: {"type":"response.completed","response":{"status":"completed"}}\n\n' + outcomes.append(Response(stream)) + assert post() == (200, stream) and len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + + +@pytest.mark.parametrize( + ("framing", "complete"), [ + ("length", False), ("chunked", False), ("length", True), ("chunked", True), + ], +) +def test_loopback_truncation_framing_and_exact_tool_prefix(relay, framing, complete): + _, _, calls, outcomes, post, body = relay + prefix = b'data: {"type":"response.output_item.added","item":{"type":"function_call","call_id":"fixture"}}\n\n' + terminal = b'data: {"type":"response.completed","response":{"status":"completed"}}\n\n' + upstream_calls = [] + upstream_finished = threading.Event() + downstream_saw_prefix = threading.Event() + + class Upstream(socketserver.StreamRequestHandler): + def handle(self): + self.rfile.readline(4097) + headers = http.client.parse_headers(self.rfile) + upstream_calls.append(self.rfile.read(int(headers["Content-Length"]))) + self.wfile.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n") + if framing == "length": + self.wfile.write(f"Content-Length: {len(prefix + terminal)}\r\n\r\n".encode()) + self.wfile.write(prefix) + else: + self.wfile.write(b"Transfer-Encoding: chunked\r\n\r\n") + self.wfile.write(f"{len(prefix):x}\r\n".encode() + prefix + b"\r\n") + self.wfile.flush() + try: + if complete: + # The relay must send the first tool item before the + # upstream sends a terminal item or closes the stream. + if not downstream_saw_prefix.wait(timeout=5): + return + if framing == "length": + self.wfile.write(terminal) + else: + self.wfile.write(f"{len(terminal):x}\r\n".encode() + terminal + b"\r\n0\r\n\r\n") + finally: + upstream_finished.set() + + with socketserver.TCPServer(("127.0.0.1", 0), Upstream) as server: + serve = threading.Thread(target=server.serve_forever, daemon=True) + serve.start() + try: + connection = http.client.HTTPConnection(*server.server_address, timeout=5) + try: + connection.request("POST", "/v1/responses", json.dumps(body).encode()) + outcomes.append(connection.getresponse()) + if complete: + with post(return_response=True) as response: + assert response.status == 200 + assert response.read(len(prefix)) == prefix + assert not upstream_finished.is_set() + downstream_saw_prefix.set() + assert response.read() == terminal + else: + with pytest.raises(http.client.IncompleteRead) as caught: + post() + assert caught.value.partial == prefix + assert len(calls) == len(upstream_calls) == 1 + assert calls[0].data == upstream_calls[0] == json.dumps(body).encode() + finally: + downstream_saw_prefix.set() + connection.close() + finally: + server.shutdown() + serve.join(timeout=5) + + +def test_valid_quota_json_in_short_content_length_cannot_rotate(relay): + _, keys, calls, outcomes, post, _ = relay + payload = json.dumps({"error": {"code": "insufficient_quota"}}).encode() + with socketserver.TCPServer(("127.0.0.1", 0), RejectedQuota) as server: + server.wire_response = ( + f"HTTP/1.1 429 Too Many Requests\r\nContent-Length: {len(payload) + 1}\r\n" + "Content-Type: application/json\r\nConnection: close\r\n\r\n" + ).encode() + payload + serve = threading.Thread(target=server.serve_forever, daemon=True) + serve.start() + try: + try: + urllib.request.urlopen(f"http://127.0.0.1:{server.server_address[1]}/", timeout=5) + except urllib.error.HTTPError as rejection: + outcomes.extend([rejection, Response(b"must-not-be-used")]) + else: + pytest.fail("fixture did not return HTTP 429") + status, response = post() + assert status == 502 and json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + finally: + server.shutdown() + serve.join(timeout=5) + + +class RejectedQuota(socketserver.StreamRequestHandler): + def handle(self): + self.rfile.readline(4097) + http.client.parse_headers(self.rfile) + self.wfile.write(self.server.wire_response) def test_pool_exhaustion_is_bounded_and_does_not_probe_in_background(relay): From e5e4f80c5eb64058c0cc2a2bd43429daaf50b76b Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:01:10 +0800 Subject: [PATCH 3/5] Bound Codex relay writes to clients that stop reading Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../loopx-codex-provider-routing/NATIVE.md | 10 ++- .../openai_relay.py | 8 +-- .../tests/test_provider_lifecycle.py | 70 ++++++++++++++++++- .../references/repair-patterns.md | 2 +- 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index f99ad6cedb..108a9b3734 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -93,6 +93,8 @@ own route. An authenticated new-protocol daemon with a different loaded build stops new admissions, drains already admitted model requests, and releases its existing owner lock before one new daemon starts. It closes connections still sending headers or request bodies. +An admitted response can also finish draining when its client stops reading and +the local socket write times out. The model, route credentials, key policy and message queue stay with their existing owners. A bounded timeout returns `api_relay_drain_pending`; read status later instead of forcing exit or replaying a Turn. Old `/health` @@ -168,8 +170,12 @@ definition and verify it on readback. A local authenticated transport sends the original selected model and complete Responses body to OpenAI. Model requests require one valid Content-Length and -are eligible for dispatch after every declared byte has arrived. A connection -that stalls while sending the request closes after ten seconds of inactivity. +are eligible for dispatch after every declared byte has arrived. Loopback +socket reads and writes have a ten-second timeout per operation. A timed-out +response write closes the relay's upstream connection and leaves the HTTP +transfer incomplete. Upstream effects remain unknown. Use the existing Turn +recovery path to determine any further action. Waiting for the upstream response +uses the upstream timeout. An explicit upstream quota rejection, 401, 403 or temporary 429 can try the next configured key before output. Each key is attempted at most once per request, with typed cooldowns; model permission failures are scoped to that model. diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py index dc8da20248..c018c3fded 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py @@ -29,7 +29,7 @@ SCHEMA = "codex_openai_api_relay_v1" API_SLOT = "openai-api" RUNTIME_SCHEMA = "codex_openai_api_relay_runtime_v1" -REQUEST_IDLE_TIMEOUT = 10 +CLIENT_IO_TIMEOUT = 10 ERROR_BODY_LIMIT = 65536 # Loaded once. Replacing an installed wheel cannot make its old daemon claim # the newly installed implementation. No credential or request data is hashed. @@ -189,7 +189,9 @@ def service_actions(self): class RelayHandler(BaseHTTPRequestHandler): server: RelayServer - timeout = REQUEST_IDLE_TIMEOUT + # This bounds I/O on the loopback client socket. The upstream uses its + # own timeout and may produce a response long after the request arrives. + timeout = CLIENT_IO_TIMEOUT def log_message(self, *_args): pass @@ -304,8 +306,6 @@ def do_POST(self): self._json(503, "api_relay_draining") return try: - # The request idle timeout applies only before model admission. - self.connection.settimeout(None) self._forward(upstream_path, body, model) finally: with self.server._admission_lock: diff --git a/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py b/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py index 6a2bb89d26..4f56d1e37a 100644 --- a/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py +++ b/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py @@ -1,9 +1,8 @@ """Real loopback admission/drain and real detached provider replacement.""" import importlib -from http.client import HTTPResponse +from http.client import HTTPResponse, IncompleteRead import io import json -import os from pathlib import Path import socket import subprocess @@ -215,6 +214,73 @@ def open(self, call, timeout): thread.join(5) +def test_drain_releases_a_response_with_a_stalled_client(owner, monkeypatch): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + monkeypatch.setattr(relay.RelayHandler, "timeout", 0.25) + owner.api_keys().add(expected_revision=1, label="Spare", api_key="sk-" + "synthetic-spare-" * 4) + release, stopped = threading.Event(), threading.Event() + calls = [] + + class StreamingResponse(io.BytesIO): + status, headers = 200, {"Content-Type": "application/octet-stream"} + + def read1(self, size): + assert release.wait(5) + return b"x" * size + + stream = StreamingResponse() + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + return stream + + class SmallSendBufferRelay(relay.RelayServer): + def get_request(self): + connection, address = super().get_request() + connection.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 4096) + return connection, address + + server = SmallSendBufferRelay(owner, {"port": 0, "token": "worker", "control_token": "control"}, + opener=Upstream()) + + def serve_and_close(): + try: + server.serve_forever() + finally: + server.server_close() + stopped.set() + + thread = threading.Thread(target=serve_and_close) + thread.start() + body = b'{"model":"selected-model","stream":true}' + headers = (b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer worker\r\n" + + f"Content-Length: {len(body)}\r\n\r\n".encode()) + try: + with socket.socket() as client, HTTPResponse(client) as response: + client.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096) + client.settimeout(5) + client.connect(("127.0.0.1", server.server_port)) + client.sendall(headers + body) + response.begin() + assert response.status == 200 and response.chunked + live = json.loads(request(server, "/runtime", "worker")[1]) + assert live["active_requests"] == 1 + assert request(server, "/control/drain", "control", body=b"", instance=live["instance_id"])[0] == 202 + release.set() + # Keep the socket open without reading any body bytes until the + # listener and all its admitted request threads have drained. + assert stopped.wait(3), "A stalled client kept the relay from closing" + assert stream.closed and server.runtime()["active_requests"] == 0 + with pytest.raises(IncompleteRead): + response.read() + assert calls == [body] + finally: + release.set() + server.shutdown() + thread.join(5) + + def test_real_detached_old_build_replaced_once_with_same_routes_and_private_state(owner, monkeypatch): relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") life = importlib.import_module("loopx_codex_provider_routing.relay_lifecycle") diff --git a/skills/loopx-self-repair/references/repair-patterns.md b/skills/loopx-self-repair/references/repair-patterns.md index eb3a735edf..e8ecb988e9 100644 --- a/skills/loopx-self-repair/references/repair-patterns.md +++ b/skills/loopx-self-repair/references/repair-patterns.md @@ -161,7 +161,7 @@ teaches a reusable control-plane lesson. | `future_monitor_heartbeat_liveness_overreach` | A valid future-scheduled monitor repeatedly enters `frontier_exhausted_monitor_lane` or `dead_monitor_repeat` even though it is not due and no real monitor execution occurred. | generated heartbeat guard command, trigger `current_time_iso`, quota `heartbeat_receipt`, monitor Todo `next_due_at`, run-history `quota_monitor_poll` rows, concrete Todo/target identity, and monitor mode. | Future wait state was collapsed into empty-frontier exhaustion, while distinct heartbeat turn ids were treated as monitor executions and lowered the dead-monitor threshold. | Add typed `future_monitor_wait` and `due_monitor_execution` frontier rules. Keep future monitors quiet/no-spend until due; count `dead_monitor_repeat` only from unchanged due/external executions carrying a concrete Todo or target identity. Heartbeat turn ids remain idempotency/liveness evidence and never reduce the real execution threshold. | | `replan_context_delivery_gap` | A replan obligation is visible, but a weak protocol-following model ignores or mis-executes a manual evidence-log read ritual, then spends turns debugging ACK/read order instead of choosing a new direction. | current obligation id, host-projected `replan_context_v0`, compact coverage ledger, delivery receipt, typed action packet, selected model tool action, semantic writeback receipt, and successor turn boundary. | Evidence access was modeled as an agent CLI obligation even though the control plane already owned the ledger; read execution and semantic use were conflated. A separate Todo-add then refresh ACK also let the model execute the successor in the replan turn. | Have the host prefetch and project bounded evidence coverage into quota. Treat the delivery receipt as context delivery only; require the model's real next action to produce an accepted typed semantic delta bound to the current obligation. When the delta is a runnable successor, bind the exact obligation on the Todo write itself and return `end_current_heartbeat`; do not require a second ACK or run the successor in that turn. Remove the standalone evidence command and generated read rituals. Reuse the typed replan context for the core Goal, dense scoped decision evidence, repetition reduction and exact history-reference reads. Keep replan selection independent of the routine display window, and validate novelty against complete available history even when the readable context is truncated; reject malformed schemas and unavailable references instead of treating them as empty evidence. Preserve the durable history and historical receipt decoder, not a compatibility command. Cover real quota→model action→Todo/refresh CLI behavior plus missing/wrong obligation identity. | | `machine_cli_capture_projection_gap` | A native hook or non-model JSON consumer treats context navigation as the effective decision and denies ready work because a top-level admission field is absent. | Original child argv/runtime profile; outer schema versus the same immutable captured Core decision; parent/child context settings; real Core ready, user-gate and zero-quota cases. | Model-facing context transport wrapped the formerly inline machine payload. A locator is not a policy decision; a locale/codec repair alone cannot restore this path. | Use the existing machine-output contract only for the non-model child process, retaining Core gates, identity, UTF-8 and environment isolation. Reuse the established per-child `LOOPX_CONTEXT_DELIVERY=off`; do not disable model context globally or interpret navigation `ok` as admission. Actual model consumers must follow the original readers and retain current bytes/digests. Validate ready, user-gate and zero-quota behavior with parent context on/off unchanged; distinguish source, installed-host and live-model qualification. | -| `relay_truncation_looks_like_clean_eof` | A Responses client receives an output or tool-item prefix followed by apparently clean HTTP EOF after the upstream body was truncated. | Observe upstream and downstream framing, exact prefix bytes, client status, and actual request count; compare a short declared length, missing final HTTP chunk, and a healthy completion. | The relay replaced a framed upstream response with close-delimited output, which erases the transport's incomplete-body signal. A syntactically complete error JSON can also be carried in an incomplete HTTP body. | Preserve a verified response length or terminate downstream chunked framing only after clean transfer. Do not manufacture a model terminal event or retry a request with unknown effects. Verify full and truncated paths over real loopback HTTP, keep already received bytes, and separately qualify explicit same-session recovery. | +| `relay_truncation_looks_like_clean_eof` | A Responses client receives an output or tool-item prefix followed by apparently clean HTTP EOF after the upstream body was truncated. A client that stops reading can also keep the relay from draining. | Observe upstream and downstream framing, exact prefix bytes, client status, and actual request count; compare a short declared length, missing final HTTP chunk, and a healthy completion. For drain, check lock release with an open stalled client and an independent healthy reader. | The relay replaced a framed upstream response with close-delimited output, which erases the transport's incomplete-body signal. A syntactically complete error JSON can also be carried in an incomplete HTTP body. Clearing the local socket timeout at model admission leaves a response write waiting indefinitely. | Preserve a verified response length or terminate downstream chunked framing only after clean transfer. Do not manufacture a model terminal event or retry a request with unknown effects. Verify full and truncated paths over real loopback HTTP, keep already received bytes, and separately qualify explicit same-session recovery. Bound loopback writes independently of the upstream timeout. Verify that a healthy reader receives its original response before the instance releases its lock. | | `native_policy_test_consumer_navigation` | A source-owned policy smoke intentionally exercises native output but raises `KeyError` on an admission field, or mistakes navigation `ok` for decision readiness. | Original CLI result and the same current immutable recovery, unchanged semantic assertions, and full connection/own-Todo readback. | The test helper decoded only outer JSON and treated the model-facing navigation envelope as the policy. This differs from the machine-output host adapter in `machine_cli_capture_projection_gap`. | Reuse the existing test-only `read_cli_policy_payload` oracle to verify the current complete recovery; preserve inline results, nonzero exits, physical-file/digest checks and all original semantic assertions. Keep native output enabled for this qualification. Test generic/domain connection and the caller's own first Todo through the real CLI, plus failed/inline/native and damaged-reference cases. Isolate temporary runtimes and telemetry before child invocation and retire only authenticated test-owned runtimes before removing their directory. The oracle proves test observations, not model retention, execution authority or whole-Goal benefit. | | `replan_settlement_binding_projection_gap` | A replan packet and selected Todo are both visible, so an agent combines `--todo-id` with `--replan-obligation-id`, alternates between them after rejection, or performs the Todo-bound refresh but omits the matching spend. | Original quota receipt identity, selected Todo, semantic obligation id, interaction settlement plan, projected refresh/spend commands, and accepted semantic ACK. | The semantic obligation target and causal settlement identity were adjacent but unlabeled, while one boolean conflated direct obligation binding with whether a full settlement chain was required. | Keep exactly one causal settlement binding. For a turn-scoped executable plan, project a domain-local typed replan settlement contract that labels the selected Todo or Todo-less obligation as the binding and keeps the replan id as a separate semantic target when necessary. Todo-bound and directly bound replan writes both project material refresh followed by spend; never combine their CLI flags. Keep unscoped diagnostic reads compact and non-executable. Cover TS projection, hot-path envelope, real Todo-bound/Todo-less CLI chains, output budget, and in-flight delivery continuity. | | `native_goal_todoless_replan_reentry_gap` | A native visible Goal completes all advancement Todos, semantic closeout derives a Todo-less autonomous replan obligation, and the next unbound quota read projects a bare refresh with no Turn identity or spend chain. | Native Goal runtime profile, selected Todo absence, replan obligation id, interaction CLI actions before and after Turn creation, and settlement identity. | Native-Goal Turn re-entry recognized only selected Todos as actionable settlement bindings and ignored a valid Todo-less replan obligation. | Treat either a selected Todo or a valid replan obligation as a native-Goal settlement binding. When no Turn or settlement plan exists, project only the exact `quota should-run ... --begin-turn` re-entry. After execution, require the obligation-bound refresh and spend chain; keep generic unscoped diagnostic behavior unchanged. | From 700bb92f3b8d63064bb03b27677f1697e74a5441 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:14:56 +0800 Subject: [PATCH 4/5] Reject upstream chunked responses missing the trailer terminator Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../loopx-codex-provider-routing/NATIVE.md | 3 +- .../openai_relay.py | 27 +++++++- tests/presentation/test_openai_relay.py | 65 ++++++++++++++++++- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index 108a9b3734..42281da57a 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -183,7 +183,8 @@ Unknown network effects, ordinary request/server failures and any started stream are not retried on another key. No job replay, model substitution or synthetic completion is added. For HTTP/1.1 clients, the relay streams bytes as they arrive and leaves the response incomplete when an upstream transfer is truncated. A -complete HTTP transfer alone does not prove a completed model response. A +chunked upstream transfer is complete after its trailer section's final blank +line. A complete HTTP transfer alone does not prove a completed model response. A truncated rejection body or one larger than the 64 KiB inspection limit cannot trigger another key. There are no background paid probes. The transport survives the Chat/App restart used for native handoff. Disabling stops new admissions and diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py index c018c3fded..ca6f8666a5 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py @@ -8,7 +8,7 @@ import argparse import hashlib import hmac -from http.client import HTTPException +from http.client import HTTPException, HTTPResponse, HTTPSConnection, IncompleteRead, LineTooLong, _MAXHEADERS, _MAXLINE from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os @@ -39,6 +39,29 @@ "openai_policy.ts", "openai_policy_bridge.ts", "native-extension.toml"))).hexdigest() +class _RelayHTTPResponse(HTTPResponse): + def _read_and_discard_trailer(self): + # Require the final blank line while retaining the standard trailer bounds. + for _ in range(_MAXHEADERS + 1): + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("trailer line") + if not line: + raise IncompleteRead(b"") + if line in (b"\r\n", b"\n"): + return + raise HTTPException(f"got more than {_MAXHEADERS} trailers") + + +class _RelayHTTPSConnection(HTTPSConnection): + response_class = _RelayHTTPResponse + + +class _RelayHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, request): + return self.do_open(_RelayHTTPSConnection, request, context=self._context) + + class _StreamOutcome: """Bounded, transient SSE framing; the existing TS owner classifies errors.""" def __init__(self): @@ -133,7 +156,7 @@ class RelayServer(ThreadingHTTPServer): def __init__(self, owner: NativeAccounts, value: dict, *, opener=None): self.owner, self.value = owner, value - self.opener = opener or urllib.request.build_opener(NoRedirect()) + self.opener = opener or urllib.request.build_opener(NoRedirect(), _RelayHTTPSHandler()) self.instance_id = secrets.token_hex(16) self._admission_lock = threading.Lock() self._receiving: set[socket.socket] = set() diff --git a/tests/presentation/test_openai_relay.py b/tests/presentation/test_openai_relay.py index aa85444b50..f4807429a5 100644 --- a/tests/presentation/test_openai_relay.py +++ b/tests/presentation/test_openai_relay.py @@ -222,7 +222,7 @@ def handle(self): def test_valid_quota_json_in_short_content_length_cannot_rotate(relay): _, keys, calls, outcomes, post, _ = relay payload = json.dumps({"error": {"code": "insufficient_quota"}}).encode() - with socketserver.TCPServer(("127.0.0.1", 0), RejectedQuota) as server: + with socketserver.TCPServer(("127.0.0.1", 0), WireResponse) as server: server.wire_response = ( f"HTTP/1.1 429 Too Many Requests\r\nContent-Length: {len(payload) + 1}\r\n" "Content-Type: application/json\r\nConnection: close\r\n\r\n" @@ -245,13 +245,74 @@ def test_valid_quota_json_in_short_content_length_cannot_rotate(relay): serve.join(timeout=5) -class RejectedQuota(socketserver.StreamRequestHandler): +class WireResponse(socketserver.StreamRequestHandler): def handle(self): self.rfile.readline(4097) http.client.parse_headers(self.rfile) self.wfile.write(self.server.wire_response) +@pytest.mark.parametrize("status", [200, 429]) +@pytest.mark.parametrize(("trailer", "complete"), [ + (b"0\r\n", False), + (b"0\r\nX-Fixture: retained\r\n", False), + (b"0\r\n\r\n", True), + (b"0\r\nX-Fixture: retained\r\n\r\n", True), +]) +def test_chunked_upstream_requires_the_end_of_trailers(relay, monkeypatch, status, trailer, complete): + owner, keys, calls, outcomes, post, _ = relay + module = importlib.import_module("loopx_codex_provider_routing.openai_relay") + # Keep the production HTTPS response parser while serving a raw local + # socket fixture. These cases exercise framing above the TLS transport. + monkeypatch.setattr(http.client.HTTPSConnection, "connect", http.client.HTTPConnection.connect) + monkeypatch.setenv("no_proxy", "127.0.0.1") + prefix = b'data: {"type":"response.created"}\n\n' + error = b'{"error":{"code":"insufficient_quota"}}' + payload = prefix if status == 200 else error + status_line = "200 OK" if status == 200 else "429 Too Many Requests" + content_type = "text/event-stream" if status == 200 else "application/json" + with module.RelayServer(owner, {"port": 0}) as relay_server: + origin_opener = relay_server.opener + with socketserver.TCPServer(("127.0.0.1", 0), WireResponse) as server: + server.wire_response = ( + f"HTTP/1.1 {status_line}\r\nTransfer-Encoding: chunked\r\n" + f"Content-Type: {content_type}\r\nConnection: close\r\n\r\n" + f"{len(payload):x}\r\n" + ).encode() + payload + b"\r\n" + trailer + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"https://127.0.0.1:{server.server_address[1]}/" + if status == 200: + outcomes.append(origin_opener.open(url, timeout=5)) + if complete: + assert post() == (200, prefix) + else: + with pytest.raises(http.client.IncompleteRead) as caught: + post() + assert caught.value.partial == prefix + assert len(calls) == 1 + else: + with pytest.raises(urllib.error.HTTPError) as rejected: + origin_opener.open(url, timeout=5) + fallback = b'{"accepted":true}' + outcomes.extend([rejected.value, Response(fallback)]) + if complete: + assert post() == (200, fallback) + assert len(calls) == 2 + assert calls[0].data == calls[1].data + assert calls[0].get_header("Authorization") != calls[1].get_header("Authorization") + else: + response_status, response = post() + assert response_status == 502 + assert json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + finally: + server.shutdown() + thread.join(timeout=5) + + def test_pool_exhaustion_is_bounded_and_does_not_probe_in_background(relay): _, _, calls, outcomes, post, _ = relay outcomes.extend([rejected(), rejected()]) From 19036f034d866c64de34ba7703faf61a1f77e162 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:40:48 +0800 Subject: [PATCH 5/5] Validate upstream chunk data delimiters before relay completion Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../loopx-codex-provider-routing/NATIVE.md | 11 +++++----- .../openai_relay.py | 9 +++++++++ tests/presentation/test_openai_relay.py | 20 +++++++++++-------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index 42281da57a..e370acd0fe 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -183,11 +183,12 @@ Unknown network effects, ordinary request/server failures and any started stream are not retried on another key. No job replay, model substitution or synthetic completion is added. For HTTP/1.1 clients, the relay streams bytes as they arrive and leaves the response incomplete when an upstream transfer is truncated. A -chunked upstream transfer is complete after its trailer section's final blank -line. A complete HTTP transfer alone does not prove a completed model response. A -truncated rejection body or one larger than the 64 KiB inspection limit cannot -trigger another key. There are no background paid probes. The transport survives -the Chat/App restart used for native handoff. Disabling stops new admissions and +chunked upstream transfer requires CRLF after each data chunk and a final blank +line after the trailer fields. A complete HTTP transfer alone does not prove a +completed model response. A truncated rejection body or one larger than the +64 KiB inspection limit cannot trigger another key. There are no background paid +probes. The transport survives the Chat/App restart used for native handoff. +Disabling stops new admissions and drains admitted model requests before its process exits. With the policy off, an existing subscription is never routed through this transport. diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py index ca6f8666a5..dbab25786f 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py @@ -40,6 +40,15 @@ class _RelayHTTPResponse(HTTPResponse): + def _get_chunk_left(self): + if self.chunk_left == 0: + # The standard parser discards the delimiter without validating it. + if self._safe_read(2) != b"\r\n": + raise IncompleteRead(b"") + # The delimiter is consumed. Let the standard parser advance. + self.chunk_left = None + return super()._get_chunk_left() + def _read_and_discard_trailer(self): # Require the final blank line while retaining the standard trailer bounds. for _ in range(_MAXHEADERS + 1): diff --git a/tests/presentation/test_openai_relay.py b/tests/presentation/test_openai_relay.py index f4807429a5..f3f9acc02c 100644 --- a/tests/presentation/test_openai_relay.py +++ b/tests/presentation/test_openai_relay.py @@ -253,13 +253,14 @@ def handle(self): @pytest.mark.parametrize("status", [200, 429]) -@pytest.mark.parametrize(("trailer", "complete"), [ - (b"0\r\n", False), - (b"0\r\nX-Fixture: retained\r\n", False), - (b"0\r\n\r\n", True), - (b"0\r\nX-Fixture: retained\r\n\r\n", True), +@pytest.mark.parametrize(("delimiter", "trailer", "complete"), [ + pytest.param(b"\r\n", b"0\r\n", False, id="missing-final-line"), + pytest.param(b"\r\n", b"0\r\nX-Fixture: retained\r\n", False, id="truncated-trailer"), + pytest.param(b"\r\n", b"0\r\n\r\n", True, id="no-trailer-fields"), + pytest.param(b"\r\n", b"0\r\nX-Fixture: retained\r\n\r\n", True, id="complete-trailer"), + pytest.param(b" ", b"0\r\n\r\n", False, id="malformed-data-delimiter"), ]) -def test_chunked_upstream_requires_the_end_of_trailers(relay, monkeypatch, status, trailer, complete): +def test_chunked_upstream_requires_complete_framing(relay, monkeypatch, status, delimiter, trailer, complete): owner, keys, calls, outcomes, post, _ = relay module = importlib.import_module("loopx_codex_provider_routing.openai_relay") # Keep the production HTTPS response parser while serving a raw local @@ -271,14 +272,17 @@ def test_chunked_upstream_requires_the_end_of_trailers(relay, monkeypatch, statu payload = prefix if status == 200 else error status_line = "200 OK" if status == 200 else "429 Too Many Requests" content_type = "text/event-stream" if status == 200 else "application/json" + midpoint = len(payload) // 2 + first, second = payload[:midpoint], payload[midpoint:] + chunks = f"{len(first):x}\r\n".encode() + first + b"\r\n" + chunks += f"{len(second):x}\r\n".encode() + second + delimiter with module.RelayServer(owner, {"port": 0}) as relay_server: origin_opener = relay_server.opener with socketserver.TCPServer(("127.0.0.1", 0), WireResponse) as server: server.wire_response = ( f"HTTP/1.1 {status_line}\r\nTransfer-Encoding: chunked\r\n" f"Content-Type: {content_type}\r\nConnection: close\r\n\r\n" - f"{len(payload):x}\r\n" - ).encode() + payload + b"\r\n" + trailer + ).encode() + chunks + trailer thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: