diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index 10fc6aba04..e370acd0fe 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -91,8 +91,11 @@ configured steward route as well as the worker, without creating credentials. The model transport's ordinary `ensure` also performs reconciliation for its own route. An authenticated new-protocol daemon with a different loaded build stops new admissions, drains -already accepted requests and releases its existing owner lock before one new -daemon starts. The model, route credentials, key policy and message queue stay +already admitted model requests, and releases its existing owner lock before one +new daemon starts. It closes connections still sending headers or request bodies. +An admitted response can also finish draining when its client stops reading and +the local socket write times out. +The model, route credentials, key policy and message queue stay with their existing owners. A bounded timeout returns `api_relay_drain_pending`; read status later instead of forcing exit or replaying a Turn. Old `/health` response bytes remain compatible, but do not qualify version adoption. @@ -165,15 +168,28 @@ is presented to OpenAI and rejected with `401 invalid_api_key`. Installers that add chat transports should write the relay endpoint into each provider definition and verify it on readback. -A local authenticated transport sends the original selected model and Responses -body to OpenAI. An explicit upstream quota rejection, 401, 403 or temporary 429 can +A local authenticated transport sends the original selected model and complete +Responses body to OpenAI. Model requests require one valid Content-Length and +are eligible for dispatch after every declared byte has arrived. Loopback +socket reads and writes have a ten-second timeout per operation. A timed-out +response write closes the relay's upstream connection and leaves the HTTP +transfer incomplete. Upstream effects remain unknown. Use the existing Turn +recovery path to determine any further action. Waiting for the upstream response +uses the upstream timeout. +An explicit upstream quota rejection, 401, 403 or temporary 429 can try the next configured key before output. Each key is attempted at most once per request, with typed cooldowns; model permission failures are scoped to that model. Unknown network effects, ordinary request/server failures and any started stream are not retried on another key. No job replay, model substitution or synthetic -completion is added. There are no background paid probes. The transport survives -the Chat/App restart used for native handoff; disabling stops new admissions and -drains already accepted requests before its process exits. With the policy off, +completion is added. For HTTP/1.1 clients, the relay streams bytes as they arrive +and leaves the response incomplete when an upstream transfer is truncated. A +chunked upstream transfer requires CRLF after each data chunk and a final blank +line after the trailer fields. A complete HTTP transfer alone does not prove a +completed model response. A truncated rejection body or one larger than the +64 KiB inspection limit cannot trigger another key. There are no background paid +probes. The transport survives the Chat/App restart used for native handoff. +Disabling stops new admissions and +drains admitted model requests before its process exits. With the policy off, an existing subscription is never routed through this transport. Version 0.15.2 recognizes the official `credit_balance_exhausted` error in both diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py index 0ff0c41e98..dbab25786f 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/openai_relay.py @@ -8,6 +8,7 @@ import argparse import hashlib import hmac +from http.client import HTTPException, HTTPResponse, HTTPSConnection, IncompleteRead, LineTooLong, _MAXHEADERS, _MAXLINE from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os @@ -28,6 +29,8 @@ SCHEMA = "codex_openai_api_relay_v1" API_SLOT = "openai-api" RUNTIME_SCHEMA = "codex_openai_api_relay_runtime_v1" +CLIENT_IO_TIMEOUT = 10 +ERROR_BODY_LIMIT = 65536 # Loaded once. Replacing an installed wheel cannot make its old daemon claim # the newly installed implementation. No credential or request data is hashed. BUILD = hashlib.sha256(b"".join(Path(__file__).with_name(name).read_bytes() for name in ( @@ -36,6 +39,38 @@ "openai_policy.ts", "openai_policy_bridge.ts", "native-extension.toml"))).hexdigest() +class _RelayHTTPResponse(HTTPResponse): + def _get_chunk_left(self): + if self.chunk_left == 0: + # The standard parser discards the delimiter without validating it. + if self._safe_read(2) != b"\r\n": + raise IncompleteRead(b"") + # The delimiter is consumed. Let the standard parser advance. + self.chunk_left = None + return super()._get_chunk_left() + + def _read_and_discard_trailer(self): + # Require the final blank line while retaining the standard trailer bounds. + for _ in range(_MAXHEADERS + 1): + line = self.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("trailer line") + if not line: + raise IncompleteRead(b"") + if line in (b"\r\n", b"\n"): + return + raise HTTPException(f"got more than {_MAXHEADERS} trailers") + + +class _RelayHTTPSConnection(HTTPSConnection): + response_class = _RelayHTTPResponse + + +class _RelayHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, request): + return self.do_open(_RelayHTTPSConnection, request, context=self._context) + + class _StreamOutcome: """Bounded, transient SSE framing; the existing TS owner classifies errors.""" def __init__(self): @@ -130,13 +165,36 @@ class RelayServer(ThreadingHTTPServer): def __init__(self, owner: NativeAccounts, value: dict, *, opener=None): self.owner, self.value = owner, value - self.opener = opener or urllib.request.build_opener(NoRedirect()) + self.opener = opener or urllib.request.build_opener(NoRedirect(), _RelayHTTPSHandler()) self.instance_id = secrets.token_hex(16) self._admission_lock = threading.Lock() + self._receiving: set[socket.socket] = set() self.draining, self.active_requests = False, 0 self.started = time.monotonic() super().__init__(("127.0.0.1", value["port"]), RelayHandler) + def process_request(self, request, client_address): + with self._admission_lock: + self._receiving.add(request) + super().process_request(request, client_address) + + def shutdown_request(self, request): + with self._admission_lock: + self._receiving.discard(request) + super().shutdown_request(request) + + def server_close(self): + with self._admission_lock: + self.draining = True + # Stop reads on connections still outside model admission. Keep their + # write half open for the control acknowledgement and error responses. + for request in self._receiving: + try: + request.shutdown(socket.SHUT_RD) + except OSError: + pass + super().server_close() + def runtime(self): with self._admission_lock: return {"schema_version": RUNTIME_SCHEMA, "build": BUILD, @@ -163,6 +221,9 @@ def service_actions(self): class RelayHandler(BaseHTTPRequestHandler): server: RelayServer + # This bounds I/O on the loopback client socket. The upstream uses its + # own timeout and may produce a response long after the request arrives. + timeout = CLIENT_IO_TIMEOUT def log_message(self, *_args): pass @@ -237,19 +298,10 @@ def do_POST(self): self._json(401, "relay_auth_required") return with self.server._admission_lock: - admitted = not self.server.draining - if admitted: - self.server.active_requests += 1 - if not admitted: + draining = self.server.draining + if draining: self._json(503, "api_relay_draining") return - try: - self._model_request() - finally: - with self.server._admission_lock: - self.server.active_requests -= 1 - - def _model_request(self): upstream_path = self.path.removeprefix("/steward") if self.consumer == "steward" else self.path if upstream_path not in ("/v1/responses", "/v1/responses/compact"): self._json(404, "unsupported_route") @@ -258,10 +310,18 @@ def _model_request(self): self._json(503, "api_fallback_disabled") return try: - length = int(self.headers.get("Content-Length", "0")) + lengths = self.headers.get_all("Content-Length", []) + if len(lengths) != 1 or "Transfer-Encoding" in self.headers: + raise ValueError() + declared = lengths[0].strip(" \t") + if not re.fullmatch(r"[0-9]+", declared): + raise ValueError() + length = int(declared) if not 1 <= length <= 32 * 1024 * 1024: raise ValueError() body = self.rfile.read(length) + if len(body) != length: + raise ValueError() value = json.loads(body) model = value["model"] if not isinstance(model, str) or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,160}", model) or model.startswith("sk-"): @@ -269,6 +329,21 @@ def _model_request(self): except (TypeError, ValueError, KeyError): self._json(400, "invalid_request") return + with self.server._admission_lock: + admitted = not self.server.draining + if admitted: + self.server._receiving.discard(self.connection) + self.server.active_requests += 1 + if not admitted: + self._json(503, "api_relay_draining") + return + try: + self._forward(upstream_path, body, model) + finally: + with self.server._admission_lock: + self.server.active_requests -= 1 + + def _forward(self, upstream_path, body, model): keys = self.server.owner.api_keys() candidates = keys.candidates(model) last_status, last_code = 429, "api_keys_unavailable" @@ -288,48 +363,84 @@ def _model_request(self): try: response = self.server.opener.open(request, timeout=300) except urllib.error.HTTPError as error: - encoded = error.read(65537) + with error: + try: + encoded = error.read(ERROR_BODY_LIMIT + 1) + except (OSError, HTTPException): + encoded = None + remaining = getattr(error.fp, "length", None) + # A bounded read can silently reach early EOF, and a + # body larger than our limit cannot be verified in full. + if encoded is None or len(encoded) > ERROR_BODY_LIMIT or (remaining or 0) > 0: + self._json(502, "upstream_unavailable") + return try: - code = json.loads(encoded).get("error", {}).get("code") if len(encoded) <= 65536 else None + code = json.loads(encoded).get("error", {}).get("code") code = code if isinstance(code, str) and code.isascii() and code.replace("_", "").isalnum() else None except (TypeError, ValueError, AttributeError): code = None headers = {k.lower(): v for k, v in error.headers.items()} status = error.code - error.close() outcome = keys.record(row["id"], status=status, code=code, model=model, headers=headers) last_status, last_code = status, code or "upstream_rejected" if outcome["retry_next_key"]: continue self._json(status, last_code) return - except (OSError, HostError): + except (OSError, HTTPException, HostError): # Unknown upstream effects never authorize trying another key. self._json(502, "upstream_unavailable") return with response: headers = {k.lower(): v for k, v in response.headers.items()} - keys.record(row["id"], status=response.status, code=None, model=model, headers=headers) + + def observe(status: int, code: str | None = None): + try: + keys.record(row["id"], status=status, code=code, model=model, headers=headers) + except (OSError, ValueError, HostError, subprocess.SubprocessError): + # Once admitted, an observation failure must not + # replace or discard the upstream's response. + pass + + observe(response.status) + # urllib removes the upstream chunk framing. Use the decoded + # length when known; otherwise frame the HTTP/1.1 response + # so a read failure cannot look like clean EOF. + can_have_body = response.status not in (204, 304) + length = getattr(response, "length", None) + chunked = can_have_body and length is None and self.request_version == "HTTP/1.1" + if chunked: + self.protocol_version = "HTTP/1.1" self.send_response(response.status) self.send_header("Content-Type", headers.get("content-type", "text/event-stream")) self.send_header("Cache-Control", "no-store") + if chunked: + self.send_header("Transfer-Encoding", "chunked") + self.send_header("Connection", "close") + elif can_have_body and length is not None: + self.send_header("Content-Length", str(length)) self.end_headers() + if not can_have_body: + return # Once headers/output are admitted, disconnects and SSE errors # end this request. No key rotation, synthetic completion or job replay. - observer = _StreamOutcome() if headers.get("content-type", "").split(";", 1)[0].strip().lower() == "text/event-stream" else None + content_type = headers.get("content-type", "").split(";", 1)[0].strip().lower() + observer = _StreamOutcome() if content_type == "text/event-stream" else None try: read = getattr(response, "read1", response.read) while chunk := read(8192): outcome = observer.feed(chunk) if observer is not None else None if outcome is not None: - try: - keys.record(row["id"], status=outcome["status"], code=outcome["code"], - model=model, headers=headers) - except (OSError, ValueError, HostError, subprocess.SubprocessError): - pass - self.wfile.write(chunk) + observe(outcome["status"], outcome["code"]) + if chunked: + self.wfile.write(f"{len(chunk):x}\r\n".encode() + chunk + b"\r\n") + else: + self.wfile.write(chunk) + self.wfile.flush() + if chunked: + self.wfile.write(b"0\r\n\r\n") self.wfile.flush() - except (OSError, TimeoutError): + except (OSError, HTTPException): self.close_connection = True return self._json(last_status, last_code) diff --git a/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py b/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py index 406b402db0..4f56d1e37a 100644 --- a/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py +++ b/packages/loopx-codex-provider-routing/tests/test_provider_lifecycle.py @@ -1,8 +1,10 @@ """Real loopback admission/drain and real detached provider replacement.""" import importlib +from http.client import HTTPResponse, IncompleteRead import io import json from pathlib import Path +import socket import subprocess import sys import threading @@ -90,11 +92,199 @@ def open(self, call, timeout): client.join(5) +def _send_partial_request(server, request): + connection = socket.create_connection(("127.0.0.1", server.server_port), timeout=3) + connection.sendall(request) + return connection + + +@pytest.mark.parametrize("framing", ["short", "ambiguous", "transfer_encoded"]) +def test_invalid_model_request_framing_never_reaches_upstream(owner, framing): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + body = b'{"model":"selected-model","input":"fixture"}' + calls = [] + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + response = io.BytesIO(b'{"ok":true}') + response.status, response.headers = 200, {"Content-Type": "application/json"} + return response + + length = len(body) + 20 if framing == "short" else len(body) + extra_header = { + "short": b"", + "ambiguous": b"Content-Length: 1\r\n", + "transfer_encoded": b"Transfer-Encoding: chunked\r\n", + }[framing] + headers = (b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer worker\r\n" + + f"Content-Length: {length}\r\n".encode() + extra_header + b"\r\n") + server = relay.RelayServer(owner, {"port": 0, "token": "worker"}, opener=Upstream()) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + with _send_partial_request(server, headers + body) as client, HTTPResponse(client) as result: + client.shutdown(socket.SHUT_WR) + result.begin() + assert result.status == 400 + assert json.loads(result.read())["error"]["code"] == "invalid_request" + assert calls == [] + assert request(server, "/v1/responses", "worker", body=body) == (200, b'{"ok":true}') + assert calls == [body] + finally: + server.shutdown() + server.server_close() + thread.join(5) + + +def test_drain_releases_stalled_request_headers_and_body_without_waiting_for_clients(owner, monkeypatch): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + accepted_headers, accepted_body = threading.Event(), threading.Event() + original_parse, original_admitted = relay.RelayHandler.parse_request, relay.RelayHandler._admitted + + def parse_request(handler): + if handler.raw_requestline.startswith(b"GET /half"): + accepted_headers.set() + return original_parse(handler) + + def admitted(handler): + if handler.path == "/v1/responses": + accepted_body.set() + return original_admitted(handler) + + monkeypatch.setattr(relay.RelayHandler, "parse_request", parse_request) + monkeypatch.setattr(relay.RelayHandler, "_admitted", admitted) + calls = [] + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + raise AssertionError("Draining incomplete request reached the upstream") + + server = relay.RelayServer(owner, {"port": 0, "token": "worker", "control_token": "control"}, opener=Upstream()) + thread = threading.Thread(target=server.serve_forever) + closed = threading.Event() + def close_after_drain(): + thread.join() + server.server_close() + closed.set() + + closer = threading.Thread(target=close_after_drain) + thread.start() + closer.start() + try: + with ( + _send_partial_request(server, b"GET /half HTTP/1.1\r\nHost: lo"), + _send_partial_request(server, b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\n" + b"Authorization: Bearer worker\r\nContent-Length: 40\r\n\r\n{"), + ): + assert accepted_headers.wait(3) and accepted_body.wait(3) + live = json.loads(request(server, "/runtime", "worker")[1]) + assert request(server, "/control/drain", "control", body=b"", instance=live["instance_id"])[0] == 202 + assert closed.wait(3) + assert server.runtime()["active_requests"] == 0 + assert live["active_requests"] == 0 and calls == [] + finally: + server.shutdown() + closer.join(5) + + +def test_idle_body_read_closes_before_model_admission(owner, monkeypatch): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + monkeypatch.setattr(relay.RelayHandler, "timeout", 0.2) + calls = [] + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + raise AssertionError("Incomplete request reached the upstream") + + server = relay.RelayServer(owner, {"port": 0, "token": "worker"}, opener=Upstream()) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + incomplete = (b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer worker\r\n" + b"Content-Length: 40\r\n\r\n{") + with _send_partial_request(server, incomplete) as client: + assert client.recv(1) == b"" + assert calls == [] and server.runtime()["active_requests"] == 0 + finally: + server.shutdown() + server.server_close() + thread.join(5) + + +def test_drain_releases_a_response_with_a_stalled_client(owner, monkeypatch): + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + monkeypatch.setattr(relay.RelayHandler, "timeout", 0.25) + owner.api_keys().add(expected_revision=1, label="Spare", api_key="sk-" + "synthetic-spare-" * 4) + release, stopped = threading.Event(), threading.Event() + calls = [] + + class StreamingResponse(io.BytesIO): + status, headers = 200, {"Content-Type": "application/octet-stream"} + + def read1(self, size): + assert release.wait(5) + return b"x" * size + + stream = StreamingResponse() + + class Upstream: + def open(self, call, timeout): + calls.append(call.data) + return stream + + class SmallSendBufferRelay(relay.RelayServer): + def get_request(self): + connection, address = super().get_request() + connection.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 4096) + return connection, address + + server = SmallSendBufferRelay(owner, {"port": 0, "token": "worker", "control_token": "control"}, + opener=Upstream()) + + def serve_and_close(): + try: + server.serve_forever() + finally: + server.server_close() + stopped.set() + + thread = threading.Thread(target=serve_and_close) + thread.start() + body = b'{"model":"selected-model","stream":true}' + headers = (b"POST /v1/responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer worker\r\n" + + f"Content-Length: {len(body)}\r\n\r\n".encode()) + try: + with socket.socket() as client, HTTPResponse(client) as response: + client.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096) + client.settimeout(5) + client.connect(("127.0.0.1", server.server_port)) + client.sendall(headers + body) + response.begin() + assert response.status == 200 and response.chunked + live = json.loads(request(server, "/runtime", "worker")[1]) + assert live["active_requests"] == 1 + assert request(server, "/control/drain", "control", body=b"", instance=live["instance_id"])[0] == 202 + release.set() + # Keep the socket open without reading any body bytes until the + # listener and all its admitted request threads have drained. + assert stopped.wait(3), "A stalled client kept the relay from closing" + assert stream.closed and server.runtime()["active_requests"] == 0 + with pytest.raises(IncompleteRead): + response.read() + assert calls == [body] + finally: + release.set() + server.shutdown() + thread.join(5) + + def test_real_detached_old_build_replaced_once_with_same_routes_and_private_state(owner, monkeypatch): relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") life = importlib.import_module("loopx_codex_provider_routing.relay_lifecycle") native = importlib.import_module("loopx_codex_provider_routing.native") - import socket with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] diff --git a/skills/loopx-self-repair/references/repair-patterns.md b/skills/loopx-self-repair/references/repair-patterns.md index 8c05e75f6d..0356dc38ec 100644 --- a/skills/loopx-self-repair/references/repair-patterns.md +++ b/skills/loopx-self-repair/references/repair-patterns.md @@ -167,6 +167,7 @@ teaches a reusable control-plane lesson. | `machine_cli_capture_projection_gap` | A native hook or non-model JSON consumer treats context navigation as the effective decision and denies ready work because a top-level admission field is absent. | Original child argv/runtime profile; outer schema versus the same immutable captured Core decision; parent/child context settings; real Core ready, user-gate and zero-quota cases. | Model-facing context transport wrapped the formerly inline machine payload. A locator is not a policy decision; a locale/codec repair alone cannot restore this path. | Use the existing machine-output contract only for the non-model child process, retaining Core gates, identity, UTF-8 and environment isolation. Reuse the established per-child `LOOPX_CONTEXT_DELIVERY=off`; do not disable model context globally or interpret navigation `ok` as admission. Actual model consumers must follow the original readers and retain current bytes/digests. Validate ready, user-gate and zero-quota behavior with parent context on/off unchanged; distinguish source, installed-host and live-model qualification. | | `dsh_quota_capture_read_transport_loss` | DSH abandons a continuation after LoopX has saved its admission and a quota reader subprocess fails to return a page. | Compare the original Turn receipt, recorded reader arguments and page cursor with the subsequent reservation and settlement records. Separate lost transport from a typed denial or changed digest. | Guard transport retry did not apply to the separate read of its saved capture. | Retry the same verified page with the existing bounded CLI retry. Preserve the reference, offset, selection digest, cancellation, identity revalidation and queue ownership. Verify with a lost response on a later page, an unrecoverable read and the complete original-Turn settlement. | | `cli_descendant_holds_timeout_pipe` | A CLI deadline has elapsed and the leader exited, but the host still waits for output held open by a descendant. | Compare the leader and owned group state, stdout/stderr closure, and the retry or mutation readback. Reproduce in an isolated process group. | Killing the direct child does not retire its inherited pipe writers. A closed pipe is not proof that a writer stopped. | Confirm cleanup of the command's owned group and drain its pipes before retrying a safe read. An escaped writer or unverifiable cleanup must not authorize replay or a fallback install. A mutation keeps its unknown-result and authoritative readback path; do not kill unrelated process trees. Keep POSIX group, detached-session and Windows evidence separate. | +| `relay_truncation_looks_like_clean_eof` | A Responses client receives an output or tool-item prefix followed by apparently clean HTTP EOF after the upstream body was truncated. A client that stops reading can also keep the relay from draining. | Observe upstream and downstream framing, exact prefix bytes, client status, and actual request count; compare a short declared length, missing final HTTP chunk, and a healthy completion. For drain, check lock release with an open stalled client and an independent healthy reader. | The relay replaced a framed upstream response with close-delimited output, which erases the transport's incomplete-body signal. A syntactically complete error JSON can also be carried in an incomplete HTTP body. Clearing the local socket timeout at model admission leaves a response write waiting indefinitely. | Preserve a verified response length or terminate downstream chunked framing only after clean transfer. Do not manufacture a model terminal event or retry a request with unknown effects. Verify full and truncated paths over real loopback HTTP, keep already received bytes, and separately qualify explicit same-session recovery. Bound loopback writes independently of the upstream timeout. Verify that a healthy reader receives its original response before the instance releases its lock. | | `native_policy_test_consumer_navigation` | A source-owned policy smoke intentionally exercises native output but raises `KeyError` on an admission field, or mistakes navigation `ok` for decision readiness. | Original CLI result and the same current immutable recovery, unchanged semantic assertions, and full connection/own-Todo readback. | The test helper decoded only outer JSON and treated the model-facing navigation envelope as the policy. This differs from the machine-output host adapter in `machine_cli_capture_projection_gap`. | Reuse the existing test-only `read_cli_policy_payload` oracle to verify the current complete recovery; preserve inline results, nonzero exits, physical-file/digest checks and all original semantic assertions. Keep native output enabled for this qualification. Test generic/domain connection and the caller's own first Todo through the real CLI, plus failed/inline/native and damaged-reference cases. Isolate temporary runtimes and telemetry before child invocation and retire only authenticated test-owned runtimes before removing their directory. The oracle proves test observations, not model retention, execution authority or whole-Goal benefit. | | `replan_settlement_binding_projection_gap` | A replan packet and selected Todo are both visible, so an agent combines `--todo-id` with `--replan-obligation-id`, alternates between them after rejection, or performs the Todo-bound refresh but omits the matching spend. | Original quota receipt identity, selected Todo, semantic obligation id, interaction settlement plan, projected refresh/spend commands, and accepted semantic ACK. | The semantic obligation target and causal settlement identity were adjacent but unlabeled, while one boolean conflated direct obligation binding with whether a full settlement chain was required. | Keep exactly one causal settlement binding. For a turn-scoped executable plan, project a domain-local typed replan settlement contract that labels the selected Todo or Todo-less obligation as the binding and keeps the replan id as a separate semantic target when necessary. Todo-bound and directly bound replan writes both project material refresh followed by spend; never combine their CLI flags. Keep unscoped diagnostic reads compact and non-executable. Cover TS projection, hot-path envelope, real Todo-bound/Todo-less CLI chains, output budget, and in-flight delivery continuity. | | `native_goal_todoless_replan_reentry_gap` | A native visible Goal completes all advancement Todos, semantic closeout derives a Todo-less autonomous replan obligation, and the next unbound quota read projects a bare refresh with no Turn identity or spend chain. | Native Goal runtime profile, selected Todo absence, replan obligation id, interaction CLI actions before and after Turn creation, and settlement identity. | Native-Goal Turn re-entry recognized only selected Todos as actionable settlement bindings and ignored a valid Todo-less replan obligation. | Treat either a selected Todo or a valid replan obligation as a native-Goal settlement binding. When no Turn or settlement plan exists, project only the exact `quota should-run ... --begin-turn` re-entry. After execution, require the obligation-bound refresh and spend chain; keep generic unscoped diagnostic behavior unchanged. | diff --git a/tests/presentation/test_openai_relay.py b/tests/presentation/test_openai_relay.py index 2b993d31f9..f3f9acc02c 100644 --- a/tests/presentation/test_openai_relay.py +++ b/tests/presentation/test_openai_relay.py @@ -1,9 +1,11 @@ """Real loopback HTTP and private store, with synthetic upstream transport.""" import importlib +import http.client import io import json import os from pathlib import Path +import socketserver import threading import urllib.error import urllib.request @@ -45,13 +47,16 @@ def open(self, request, timeout): thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() body = {"model": "unchanged-selected-model", "input": "synthetic prompt", "stream": True, "tools": []} - def post(token="example-loopback-token", origin=None): + def post(token="example-loopback-token", origin=None, *, return_response=False): headers = {"Authorization": "Bearer " + token, "Content-Type": "application/json"} if origin: headers["Origin"] = origin request = urllib.request.Request(f"http://127.0.0.1:{server.server_port}/v1/responses", data=json.dumps(body).encode(), headers=headers) try: - with urllib.request.urlopen(request, timeout=10) as response: + response = urllib.request.urlopen(request, timeout=10) + if return_response: + return response + with response: return response.status, response.read() except urllib.error.HTTPError as error: return error.code, error.read() @@ -98,9 +103,218 @@ def read1(self, _size): return self.read() partial = b'data: {"type":"response.created"}\n\n' outcomes.append(Interrupted(partial)) - status, result = post() - assert status == 200 and result == partial and len(calls) == 1 - assert b"response.completed" not in result + with pytest.raises(http.client.IncompleteRead) as caught: + post() + assert caught.value.partial == partial and len(calls) == 1 + + +def test_bad_upstream_headers_return_one_unknown_failure(relay): + _, keys, calls, outcomes, post, _ = relay + outcomes.append(http.client.LineTooLong("upstream header")) + status, response = post() + assert status == 502 and json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + + +@pytest.mark.parametrize("oversized", [False, True]) +def test_unverified_rejection_body_cannot_trigger_another_key(relay, oversized): + _, keys, calls, outcomes, post, _ = relay + + class TruncatedRejection(io.BytesIO): + def read(self, *_args): + raise http.client.IncompleteRead(b'{"error":') + + outcomes.extend([ + urllib.error.HTTPError( + "https://api.openai.com/v1/responses", 429, "fixture", {}, + io.BytesIO(b"x" * 65537) if oversized else TruncatedRejection(), + ), + Response(b"must-not-be-used"), + ]) + status, response = post() + assert status == 502 and json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + + +def test_unavailable_key_observation_does_not_discard_admitted_response(relay, monkeypatch): + _, keys, calls, outcomes, post, _ = relay + + def unavailable(*_args, **_kwargs): + raise BlockingIOError("fixture observation lock") + + monkeypatch.setattr(type(keys), "record", unavailable) + stream = b'data: {"type":"response.completed","response":{"status":"completed"}}\n\n' + outcomes.append(Response(stream)) + assert post() == (200, stream) and len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + + +@pytest.mark.parametrize( + ("framing", "complete"), [ + ("length", False), ("chunked", False), ("length", True), ("chunked", True), + ], +) +def test_loopback_truncation_framing_and_exact_tool_prefix(relay, framing, complete): + _, _, calls, outcomes, post, body = relay + prefix = b'data: {"type":"response.output_item.added","item":{"type":"function_call","call_id":"fixture"}}\n\n' + terminal = b'data: {"type":"response.completed","response":{"status":"completed"}}\n\n' + upstream_calls = [] + upstream_finished = threading.Event() + downstream_saw_prefix = threading.Event() + + class Upstream(socketserver.StreamRequestHandler): + def handle(self): + self.rfile.readline(4097) + headers = http.client.parse_headers(self.rfile) + upstream_calls.append(self.rfile.read(int(headers["Content-Length"]))) + self.wfile.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n") + if framing == "length": + self.wfile.write(f"Content-Length: {len(prefix + terminal)}\r\n\r\n".encode()) + self.wfile.write(prefix) + else: + self.wfile.write(b"Transfer-Encoding: chunked\r\n\r\n") + self.wfile.write(f"{len(prefix):x}\r\n".encode() + prefix + b"\r\n") + self.wfile.flush() + try: + if complete: + # The relay must send the first tool item before the + # upstream sends a terminal item or closes the stream. + if not downstream_saw_prefix.wait(timeout=5): + return + if framing == "length": + self.wfile.write(terminal) + else: + self.wfile.write(f"{len(terminal):x}\r\n".encode() + terminal + b"\r\n0\r\n\r\n") + finally: + upstream_finished.set() + + with socketserver.TCPServer(("127.0.0.1", 0), Upstream) as server: + serve = threading.Thread(target=server.serve_forever, daemon=True) + serve.start() + try: + connection = http.client.HTTPConnection(*server.server_address, timeout=5) + try: + connection.request("POST", "/v1/responses", json.dumps(body).encode()) + outcomes.append(connection.getresponse()) + if complete: + with post(return_response=True) as response: + assert response.status == 200 + assert response.read(len(prefix)) == prefix + assert not upstream_finished.is_set() + downstream_saw_prefix.set() + assert response.read() == terminal + else: + with pytest.raises(http.client.IncompleteRead) as caught: + post() + assert caught.value.partial == prefix + assert len(calls) == len(upstream_calls) == 1 + assert calls[0].data == upstream_calls[0] == json.dumps(body).encode() + finally: + downstream_saw_prefix.set() + connection.close() + finally: + server.shutdown() + serve.join(timeout=5) + + +def test_valid_quota_json_in_short_content_length_cannot_rotate(relay): + _, keys, calls, outcomes, post, _ = relay + payload = json.dumps({"error": {"code": "insufficient_quota"}}).encode() + with socketserver.TCPServer(("127.0.0.1", 0), WireResponse) as server: + server.wire_response = ( + f"HTTP/1.1 429 Too Many Requests\r\nContent-Length: {len(payload) + 1}\r\n" + "Content-Type: application/json\r\nConnection: close\r\n\r\n" + ).encode() + payload + serve = threading.Thread(target=server.serve_forever, daemon=True) + serve.start() + try: + try: + urllib.request.urlopen(f"http://127.0.0.1:{server.server_address[1]}/", timeout=5) + except urllib.error.HTTPError as rejection: + outcomes.extend([rejection, Response(b"must-not-be-used")]) + else: + pytest.fail("fixture did not return HTTP 429") + status, response = post() + assert status == 502 and json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + finally: + server.shutdown() + serve.join(timeout=5) + + +class WireResponse(socketserver.StreamRequestHandler): + def handle(self): + self.rfile.readline(4097) + http.client.parse_headers(self.rfile) + self.wfile.write(self.server.wire_response) + + +@pytest.mark.parametrize("status", [200, 429]) +@pytest.mark.parametrize(("delimiter", "trailer", "complete"), [ + pytest.param(b"\r\n", b"0\r\n", False, id="missing-final-line"), + pytest.param(b"\r\n", b"0\r\nX-Fixture: retained\r\n", False, id="truncated-trailer"), + pytest.param(b"\r\n", b"0\r\n\r\n", True, id="no-trailer-fields"), + pytest.param(b"\r\n", b"0\r\nX-Fixture: retained\r\n\r\n", True, id="complete-trailer"), + pytest.param(b" ", b"0\r\n\r\n", False, id="malformed-data-delimiter"), +]) +def test_chunked_upstream_requires_complete_framing(relay, monkeypatch, status, delimiter, trailer, complete): + owner, keys, calls, outcomes, post, _ = relay + module = importlib.import_module("loopx_codex_provider_routing.openai_relay") + # Keep the production HTTPS response parser while serving a raw local + # socket fixture. These cases exercise framing above the TLS transport. + monkeypatch.setattr(http.client.HTTPSConnection, "connect", http.client.HTTPConnection.connect) + monkeypatch.setenv("no_proxy", "127.0.0.1") + prefix = b'data: {"type":"response.created"}\n\n' + error = b'{"error":{"code":"insufficient_quota"}}' + payload = prefix if status == 200 else error + status_line = "200 OK" if status == 200 else "429 Too Many Requests" + content_type = "text/event-stream" if status == 200 else "application/json" + midpoint = len(payload) // 2 + first, second = payload[:midpoint], payload[midpoint:] + chunks = f"{len(first):x}\r\n".encode() + first + b"\r\n" + chunks += f"{len(second):x}\r\n".encode() + second + delimiter + with module.RelayServer(owner, {"port": 0}) as relay_server: + origin_opener = relay_server.opener + with socketserver.TCPServer(("127.0.0.1", 0), WireResponse) as server: + server.wire_response = ( + f"HTTP/1.1 {status_line}\r\nTransfer-Encoding: chunked\r\n" + f"Content-Type: {content_type}\r\nConnection: close\r\n\r\n" + ).encode() + chunks + trailer + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"https://127.0.0.1:{server.server_address[1]}/" + if status == 200: + outcomes.append(origin_opener.open(url, timeout=5)) + if complete: + assert post() == (200, prefix) + else: + with pytest.raises(http.client.IncompleteRead) as caught: + post() + assert caught.value.partial == prefix + assert len(calls) == 1 + else: + with pytest.raises(urllib.error.HTTPError) as rejected: + origin_opener.open(url, timeout=5) + fallback = b'{"accepted":true}' + outcomes.extend([rejected.value, Response(fallback)]) + if complete: + assert post() == (200, fallback) + assert len(calls) == 2 + assert calls[0].data == calls[1].data + assert calls[0].get_header("Authorization") != calls[1].get_header("Authorization") + else: + response_status, response = post() + assert response_status == 502 + assert json.loads(response)["error"]["code"] == "upstream_unavailable" + assert len(calls) == 1 + assert all(row["runtime"] is None for row in keys.status()["keys"]) + finally: + server.shutdown() + thread.join(timeout=5) def test_pool_exhaustion_is_bounded_and_does_not_probe_in_background(relay):