From 5831cb867dd307e2aea3e813a3a4ccfd9ea61ca6 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:56:14 +0800 Subject: [PATCH 1/2] Verify the historical Short benchmark carrier before launch Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/edgebench/README.md | 46 ++++++------- benchmark/edgebench/run.py | 2 +- benchmark/runtime/RUNTIME.md | 27 +++++--- benchmark/runtime/SETTINGS.md | 16 +++-- benchmark/runtime/worker.py | 64 +++++++++++++++-- benchmark/tests/test_shared_codex_runtime.py | 68 +++++++++++++++++++ benchmark/tests/test_task_entry.py | 3 +- .../contract-and-context.md | 9 +++ .../contract-and-context.zh-CN.md | 6 ++ docs/reference/protocols/turn-envelope-v0.md | 23 +++---- .../test_codex_context_capture.py | 41 +++++++---- 11 files changed, 226 insertions(+), 79 deletions(-) diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index da5253e145..7a790c98e5 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -1003,30 +1003,28 @@ budget or submissions. Freeze it with exact sources under a new run ID; do not change live workers. Receipts qualify requested configuration, while actual review, proposal adoption and task gains require trajectory evidence. -## Optional short-context treatment - -Add `--turn-envelope` to an otherwise identical `heartbeat-resume` or -`heartbeat-explore` command to use the product's short normal packet and local -same-invocation full detail. Other workers reject the option before creating a -trial. The default remains off; the runtime and worker-profile receipts record -it explicitly. Use a new run ID and freeze this as a separate context treatment, -keeping objective guidance, task/scorer version, model and budgets matched. -Omit the flag on the next run to disable it. No active experiment is changed by -installing this code. Inspect wake receipts and the saved decisions before -claiming adoption; score countability still requires the usual integrity review. - -For an explicitly complete v1 Host contract, choose `--complete-turn` instead -on either heartbeat profile. The shared worker forwards that choice through -the real heartbeat renderer and guard; it retains the original ready claim, -receipt and single-command settlement. It cannot be combined with the legacy -short-envelope option. Context delivery remains independent, and unselected -runs keep their existing behavior. See the [Host protocol](../../docs/reference/protocols/turn-envelope-v0.md#explicit-complete-v1-host-entry--显式完整-v1-host-入口) -for source readback, failure recovery and rollback. User task time requirements -remain readable; private Host deadlines stay outside model context. - -中文:heartbeat profile 可显式选择 `--complete-turn` 以消费完整 v1,不能与旧短包 -选项同时启用;共用原 ready claim、回执和单条结算 owner,不改变未选择的运行。 -按原 source reader 核实采用与恢复;该开关不新增模型、子 worker、时钟或权限。 +## Quota input and historical Short + +Current C1 sources give both heartbeat profiles the complete v1 input by +default. `--complete-turn` explicitly requests the complete contract, including +on supported historical product revisions. Context delivery remains a separate +setting. The [Host protocol](../../docs/reference/protocols/turn-envelope-v0.md#explicit-complete-v1-host-entry--显式完整-v1-host-入口) +documents original captures, readback and settlement. + +For a historical Short benchmark arm, use its recorded product and runner +revisions. With `--turn-envelope`, the current worker verifies the selected CLI's +actual v0 response using a read-only preview before starting a model. C1 or an +unavailable preview rejects the request. A `fresh_replan` checkpoint preloads a +different view and is incompatible with Short. Inspect the wake's +`quota_input_contract` eligibility and the actual saved decision before using +a result in a comparison. Include startup and wake costs under the same study +protocol. + +中文:当前 C1 的两种 heartbeat profile 默认交付完整 v1,`--complete-turn` +也可在支持的历史产品版本中显式选择完整输入。旧 Short 组应按记录的产品和 runner +版本复现。当前 worker 在启动模型前用只读预演核实 v0,回执 +`quota_input_contract` 记录预期、实测版本与资格。C1 或无法核实的来源会拒绝请求。 +`fresh_replan` 会预载另一种输入,因此不能与 Short 合用。 ## Close a managed native attempt diff --git a/benchmark/edgebench/run.py b/benchmark/edgebench/run.py index 480ba091b2..e679d04b05 100644 --- a/benchmark/edgebench/run.py +++ b/benchmark/edgebench/run.py @@ -126,7 +126,7 @@ def main(argv=None): help="Opt-in private binding for native terminal proof, consumed by the controller") turn_view = parser.add_mutually_exclusive_group() turn_view.add_argument("--turn-envelope", action="store_true", - help="Opt-in short heartbeat context with same-invocation full captures") + help="Historical Short envelope treatment; the worker requires a verified v0 quota response") turn_view.add_argument("--complete-turn", action="store_true", help="Explicit complete v1 Turn contract with shared exact-Turn settlement; heartbeat workers only") parser.add_argument("--context-delivery", choices=("off", "on"), diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md index 60b7c40607..bc44dd1f19 100644 --- a/benchmark/runtime/RUNTIME.md +++ b/benchmark/runtime/RUNTIME.md @@ -81,17 +81,22 @@ agents: replan_after_turns: 6 ``` -For an explicitly selected **heartbeat-only** context experiment, add -`turn_envelope: true` to `kwargs` (default `false`). Each wake creates a private -capture root and asks the product renderer for a short TurnEnvelope dispatcher. -Full decisions remain available through `detail_ref.full_decision` from the same -guard invocation. Selection/reentry still evaluate current authority; generated -selection commands retain the capture root. When enabled, the setting appears in install, -execution and wake receipts as `turn_envelope: true`. When disabled, the field -and worker environment override are absent, preserving the pre-opt-in shape. Invalid native-Goal/plain/Turn-driver combinations -fail before execution. Set it back to `false` for the next run to roll back; -do not change a running trial's treatment. Captures contain private Goal context. -Transport tests do not establish lower token cost or better model decisions. +Historical Short trials request `turn_envelope: true` in heartbeat `kwargs`. +Before planning or execution starts a model, the worker previews the selected +product's quota response through the same CLI. Its wake receipt records the +expected and observed versions and eligibility in `quota_input_contract`. C1 +sources return v1 and the Short request fails before Host preparation. A +`fresh_replan` checkpoint preloads a distinct view and also prevents a Short +launch. The preview has no Turn identity and creates no admission or capture. +Ordinary and explicit Complete wakes proceed without this Short preview. + +Use the recorded product and runner revisions to reproduce a historical +treatment. On a qualified v0 source, an execution wake records +`turn_envelope_source=explicit` once it renders the Short bootstrap. Its +same-invocation full decision remains available through `detail_ref.full_decision`. +The planning prompt has its own input. Include the preview in startup +and wake cost when measuring a new attempt with this runner. Keep captures +private and preserve the settings and receipts of existing attempts. New native heartbeat trials default to context delivery `on`. Add `context_delivery: "off"` to the same heartbeat agent `kwargs` to disable it. diff --git a/benchmark/runtime/SETTINGS.md b/benchmark/runtime/SETTINGS.md index ae11ff5544..c6004e9b2f 100644 --- a/benchmark/runtime/SETTINGS.md +++ b/benchmark/runtime/SETTINGS.md @@ -20,7 +20,7 @@ use matched repetitions to test its effect. | --- | --- | --- | | Task entry | LoopX modes: loopx-planned | Record planned or seeded for every arm | | Explore | Off in heartbeat-resume; on in heartbeat-explore | Keep resume as the reference | -| Turn envelope | Off; a checkpoint handoff enables capture on fresh_replan execute wakes and receipts record `turn_envelope_source: checkpoint_handoff` | Enable only in its ablation; read the effective receipt, not the opt-in flag | +| Quota input | Current C1 product sources return the complete v1 contract on ordinary and explicit Complete wakes | Historical Short needs a verified v0 product response before model launch | | Native-Goal continuation | Off: a native Goal that declares completion ends the worker; a Goal still active at the budget edge is recorded as `native_goal_outcome: budget_exhausted` with exit 0 | `native_goal_continue: true` / `--native-goal-continue` starts continuation Goals until 300 s before the budget edge (native-goal profile only) | | Replan cadence | 6 settled effective work Turns | Use explicit completed-Todo cadence or another Turn count for ablation | | Iteration context | Harbor: fresh; EdgeBench heartbeat: resume | Freeze the provider and context within a comparison | @@ -48,7 +48,7 @@ These are recommendations, not automatically launched experiments. | Reference | `--worker heartbeat-resume --task-entry loopx-planned --replan-after-turns 6` | Planned entry with effective-turn replanning | | Seeded | Replace only `--task-entry` with `seeded-todo` | Does initial task decomposition help? | | Explore | Replace only `--worker` with `heartbeat-explore` | Are recorded evidence and subsequent route choices useful? | -| Short envelope | Add `--turn-envelope` | Does progressive context loading reduce overhead without losing decisions? | +| Historical Short envelope | Use the recorded product and runner revisions with `--turn-envelope`; the current worker verifies v0 before launch | Does short input with reachable original detail reduce overhead while preserving decisions? | | Todo cadence | Replace `--replan-after-turns 6` with `--replan-after-todos 3` | How do units and thresholds change interruptions? This is a combined ablation. | | No LoopX | `--worker official`; omit LoopX-specific flags | What is the net effect of the whole LoopX treatment? | @@ -93,11 +93,13 @@ configuration unchanged. ## Readback and interpretation -TurnEnvelope projections reuse the validated checkpoint declaration. An explicit -opt-in records `turn_envelope_source=explicit`; a known `fresh_replan` declaration -for heartbeat records `checkpoint_handoff`. Omitting `trigger` means after-wake -capture and does not imply TurnEnvelope. A programmatic guest path without -declaration bytes records capture configuration only: it cannot prove a trigger. +The Short setting in the install profile records the request. Before a model +starts, the worker records the read-only quota preview's expected version, +observed version and eligibility in `quota_input_contract`. The wake records +`turn_envelope_source=explicit` after v0 qualification. A `fresh_replan` checkpoint +preloads a different view and is incompatible with this Short launch. An omitted +trigger means after-wake capture. Interpret saved attempts against their pinned +source and recorded carrier. For the Harbor adapter, pass the same validated `checkpoint_request` bytes with their `checkpoint_request_path` when projecting that known configuration. SForge forwards the exact bytes it stages. The worker's individual wake receipt remains diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index 98e33f6109..72b520d2a8 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -14,7 +14,13 @@ from dataclasses import replace from pathlib import Path -from benchmark.runtime.codex import Execution, explicit_turn_display, prepare_codex_home, process_environment +from benchmark.runtime.codex import ( + Execution, + explicit_turn_display, + prepare_codex_home, + process_environment, + turn_envelope_receipt, +) from benchmark.runtime.scheduler import PHASE_WORK_RESERVE_SECONDS as PHASE_WORK_RESERVE_SECONDS from benchmark.runtime.scheduler import phase_work_reserve @@ -523,6 +529,53 @@ def _phase_timeout(timeout: float | None, deadline: float | None) -> tuple[float return timeout, False +def _qualify_short_envelope(env: dict[str, str], receipt: dict) -> None: + """Check the pinned product's carrier before a Short arm can start a model.""" + from loopx.control_plane.quota.turn_envelope import ( + COMPLETE_TURN_DECISION_VERSION, + TURN_ENVELOPE_SCHEMA_VERSION, + ) + + contract = { + "expected_schema_version": TURN_ENVELOPE_SCHEMA_VERSION, + "observed_schema_version": None, + "eligible": False, + } + receipt.update(quota_input_contract=contract, host_invoked=False) + deadline = env.get("LOOPX_PHASE_DEADLINE_EPOCH") + timeout, _ = _phase_timeout(120, float(deadline) if deadline else None) + # This preview runs the same product projection without a Turn, capture, + # or admission. Keep optional delivery out of the carrier observation. + command = loopx_command(env) + [ + "quota", "should-run", "--dry-run", "--runtime-profile", "generic_cli", + "--goal-id", env["LOOPX_GOAL_ID"], "--agent-id", env["LOOPX_AGENT_ID"], + "--available-capability", "shell", "--available-capability", "filesystem_write", + "--turn-envelope", "--context-delivery", "off", + ] + try: + result = subprocess.run(command, cwd=env["LOOPX_PROJECT"], env=env, + capture_output=True, text=True, timeout=timeout, check=True) + payload = read_json(result.stdout) + except (OSError, ValueError, subprocess.SubprocessError) as exc: + raise ValueError("Short envelope requires a successful v0 quota preview from the pinned product.") from exc + if payload.get("ok") is not True: + raise ValueError("Short envelope requires a successful v0 quota preview from the pinned product.") + observed = payload.get("schema_version") + if observed in (TURN_ENVELOPE_SCHEMA_VERSION, COMPLETE_TURN_DECISION_VERSION): + contract["observed_schema_version"] = observed + if observed != TURN_ENVELOPE_SCHEMA_VERSION: + raise ValueError("Short envelope requires a verified v0 response from the pinned product. " + "Use the recorded historical source pins for that arm.") + if checkpoint_path := env.get("LOOPX_CHECKPOINT_REQUEST"): + from .checkpoint import read_checkpoint_request + + request, _ = read_checkpoint_request(checkpoint_path) + if request.get("trigger") == "fresh_replan": + raise ValueError("Short envelope requires the direct v0 guard. " + "A fresh_replan checkpoint preloads a different view.") + contract["eligible"] = True + + def run_once(env: dict[str, str]) -> dict: if env.get("LOOPX_CONTEXT_DELIVERY", "off") not in {"off", "on"}: raise ValueError("LOOPX_CONTEXT_DELIVERY supports only off or on") @@ -585,10 +638,7 @@ def run_once(env: dict[str, str]) -> dict: "task_entry": execution.task_entry, **({"execution_observation": observation_producer} if observation_producer is not None else {}), - **({"turn_envelope": True, "turn_envelope_source": "explicit"} - if execution.turn_envelope else {}), - **({"complete_turn": True, "complete_turn_source": "explicit"} - if execution.complete_turn else {}), + **turn_envelope_receipt(explicit=False, complete=execution.complete_turn), **({"context_delivery": "on"} if execution.context_delivery != "off" else {}), **({"native_child_limit": execution.native_child_limit} if execution.native_child_limit is not None else {}), @@ -629,6 +679,8 @@ def run_once(env: dict[str, str]) -> dict: execution, timeout_seconds=(remaining - reserve if execution.timeout_seconds is None else min(execution.timeout_seconds, remaining - reserve)) ) + if execution.turn_envelope: + _qualify_short_envelope(env, receipt) prepare_codex_home( home, execution=execution, @@ -678,6 +730,8 @@ def run_once(env: dict[str, str]) -> dict: elif execution.mode in {"heartbeat", "loopx-goal"}: body = heartbeat_body(env, turn_id, native_goal=execution.native_goal, decision_handoff_socket=checkpoint_wake.socket_path) + if execution.turn_envelope: + receipt.update(turn_envelope_receipt(explicit=True)) elif execution.mode == "plain": body = Path(env["LOOPX_TASK_DOC"]).read_text(encoding="utf-8") session_wake = None diff --git a/benchmark/tests/test_shared_codex_runtime.py b/benchmark/tests/test_shared_codex_runtime.py index 2d69b82d6d..b4c8c961bb 100644 --- a/benchmark/tests/test_shared_codex_runtime.py +++ b/benchmark/tests/test_shared_codex_runtime.py @@ -990,18 +990,86 @@ async def task_exec(**kwargs): assert receipt["turn_envelope"] is True +@pytest.mark.parametrize("stage", ["plan", "execute"]) +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_short_wake_rejects_c1_before_host_preparation(tmp_path, monkeypatch, stage, provider): + from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime + from tests.control_plane.test_quota_settlement_cli import ( + AGENT_ID, GOAL_ID, _heartbeat_receipt_count, _write_fixture, + ) + + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path, provider=provider) + state = project / f".codex/goals/{GOAL_ID}/ACTIVE_GOAL_STATE.md" + original_state = state.read_bytes() + wakes = tmp_path / "wakes" + home = tmp_path / "codex-home" + source = Path(__file__).resolve().parents[2] + env = dict( + os.environ, + PYTHONPATH=str(source), LOOPX_PYTHON=sys.executable, LOOPX_USAGE_PING="0", + LOOPX_CLI=str(Path(sys.executable).parent / "loopx"), + LOOPX_PROJECT=str(project), LOOPX_REGISTRY=str(registry), LOOPX_RUNTIME_ROOT=str(runtime), + LOOPX_GOAL_ID=GOAL_ID, LOOPX_AGENT_ID=AGENT_ID, LOOPX_WAKE_LOG_DIR=str(wakes), + LOOPX_CODEX_HOME=str(home), LOOPX_EXECUTION_MODE="heartbeat", + LOOPX_SHARED_SKILLS=str(tmp_path / "shared-skills"), + LOOPX_TASK_STAGE=stage, LOOPX_TURN_ENVELOPE="1", LOOPX_CONTEXT_DELIVERY="off", + MODEL_NAME="fixture", REASONING_EFFORT="low", + ) + + def unexpected_host_preparation(*args, **kwargs): + pytest.fail("A C1 response cannot start the Short envelope arm") + + monkeypatch.setattr(worker, "prepare_codex_home", unexpected_host_preparation) + with pytest.raises(ValueError, match="Short envelope requires"): + run_once(env) + receipt_path, = wakes.glob("*/receipt.json") + receipt = json.loads(receipt_path.read_text()) + assert receipt["ok"] is False and receipt["host_invoked"] is False + assert receipt["quota_input_contract"] == { + "expected_schema_version": "loopx_turn_envelope_v0", + "observed_schema_version": "loopx_turn_envelope_v1", + "eligible": False, + } + assert "turn_envelope_source" not in receipt + assert _heartbeat_receipt_count(runtime, receipt["turn_id"]) == 0 + assert state.read_bytes() == original_state + assert not home.exists() + + @pytest.mark.parametrize("enabled", [False, True]) def test_wake_receipt_preserves_opt_out(tmp_path, monkeypatch, enabled): from benchmark.tests.test_resume_sessions import resume_env env = resume_env(tmp_path, monkeypatch) env["LOOPX_TURN_ENVELOPE"] = "1" if enabled else "0" + original_run = subprocess.run + previews = [] + + def legacy_preview(argv, **kwargs): + if argv[0] != "loopx" or "--dry-run" not in argv: + return original_run(argv, **kwargs) + previews.append(argv) + return subprocess.CompletedProcess(argv, 0, json.dumps({ + "ok": True, "schema_version": "loopx_turn_envelope_v0", + }), "") + + monkeypatch.setattr(worker.subprocess, "run", legacy_preview) receipt = run_once(env) assert receipt["ok"], receipt assert ("turn_envelope" in receipt) is enabled assert ("turn_envelope_source" in receipt) is enabled + assert ("quota_input_contract" in receipt) is enabled + assert len(previews) == int(enabled) if enabled: assert receipt["turn_envelope"] is True assert receipt["turn_envelope_source"] == "explicit" + assert receipt["quota_input_contract"] == { + "expected_schema_version": "loopx_turn_envelope_v0", + "observed_schema_version": "loopx_turn_envelope_v0", + "eligible": True, + } + assert "should-run" in previews[0] and "--turn-envelope" in previews[0] + assert "--turn-instance-id" not in previews[0] and "--decision-output-root" not in previews[0] @pytest.mark.parametrize("observed", [ diff --git a/benchmark/tests/test_task_entry.py b/benchmark/tests/test_task_entry.py index 8e2bdd027f..7bc54676e4 100644 --- a/benchmark/tests/test_task_entry.py +++ b/benchmark/tests/test_task_entry.py @@ -1240,8 +1240,7 @@ def test_execute_wake_keeps_host_clocks_out_of_fresh_and_resumed_input( assert ("--complete-turn" in body) is complete if checkpoint_preflight: original = Path(env["LOOPX_WAKE_LOG_DIR"]) / receipt["turn_id"] / "checkpoint-start-guard.json" - assert json.loads(original.read_bytes()).get("schema_version") == ( - "loopx_turn_envelope_v1" if complete else None) + assert json.loads(original.read_bytes())["schema_version"] == "loopx_turn_envelope_v1" assert "Do not repeat the initial guard below" in body assert current["LOOPX_PHASE_DEADLINE_EPOCH"] == deadline # Runtime clocks remain internal and never change task/completion authority. diff --git a/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.md b/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.md index 650fa11916..71e2fc78b1 100644 --- a/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.md +++ b/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.md @@ -38,6 +38,15 @@ diagnostics stay intact. The crowded regression's pretty JSON changes from the new full carrier. Wire/page limits and the terminal actor's ten-call budget do not change. Frozen raw/v0 model comparison controls remain separate. +The benchmark worker checks an explicit historical Short request before +starting a planning or execution model. It previews the pinned CLI's quota +response without admission and records the expected and observed source versions +and eligibility in the wake receipt. A v1 or unavailable response refuses the +Short treatment. A fresh-replan checkpoint preloads a different input and cannot +qualify as direct Short. Current ordinary and Complete requests continue through +the shared C1 producer. Historical experiments retain their recorded sources +and receipts. + Todo completion, replan-successor return and blocked-completion recovery use the existing TypeScript settlement composer for their current continuation plans. They retain the original identity, route and supplied lease proof. JSON and diff --git a/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.zh-CN.md b/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.zh-CN.md index 59aa0d54ae..e0775360bf 100644 --- a/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.zh-CN.md +++ b/docs/architecture/rfcs/controller-executor-unified-contract-v0/contract-and-context.zh-CN.md @@ -28,6 +28,12 @@ Status、plan 与 scheduler-observation 保留各自用途;本切片不退役 新的完整 carrier。Wire/page 上限和终态 actor 的十次调用预算不变。冻结的 raw/v0 模型对照继续独立保留。 +Benchmark worker 在启动规划或执行模型前核实显式历史 Short 请求。它从已固定 +的 CLI 只读预演 quota 响应,不准入工作,并在 wake 回执记录预期与实际版本及 +资格。v1 或无法核实的响应会拒绝 Short。Fresh-replan checkpoint 会预载另一种 +输入,不能算作直接 Short。当前普通与 Complete 请求继续复用 C1 producer。 +历史实验保留原版本与回执。 + Todo 验收完成、replan-successor 返回及 blocked-completion 恢复的当前续接计划复用 既有 TypeScript 结算 composer,保留原身份、路由与已提供的 lease 证明。JSON 与 Markdown 展示随写回执行的运行时扣额,不再要求模型执行第二条 spend 命令。 diff --git a/docs/reference/protocols/turn-envelope-v0.md b/docs/reference/protocols/turn-envelope-v0.md index f64b42a9bf..45f50f9323 100644 --- a/docs/reference/protocols/turn-envelope-v0.md +++ b/docs/reference/protocols/turn-envelope-v0.md @@ -618,20 +618,15 @@ diagnostic remain unchanged. This is an output qualification, not a full-cost SL 中文:捕获和模型输入投影独立。`--decision-output-root` 仅保存完整决策, `--decision-handoff-socket` 仅保留检查点交接;二者均不隐式开启 TurnEnvelope。 -此前捕获目录同时启用短包的行为已修正。需要短包的调用者须对 -`heartbeat-prompt` 显式传 `--turn-envelope`;省略该开关时仍使用普通输入, -选择、结算、身份与完整私有捕获不变。旧运行的实际输入与设置不得追溯改写。 - -This is Python filesystem/command transport under the existing quota/context -owner, not a new capability or Python decision rule. The shared benchmark -heartbeat worker has a separate default-off `turn_envelope` option documented in -[its runtime guide](../../../benchmark/runtime/RUNTIME.md). Native `/goal`, Turn -driver, installed App automations, frontend and Lark are not switched by this -research-runner setting. Real local worker/renderer/selection/settlement tests -qualify transport; model token/IO costs, decision quality and long-run adoption -still require a separately pinned experiment. Capture grants no additional -read/write/lease authority. Keep captures private; disable the option in the -next run and use ordinary file management to remove unneeded records. +历史 Short 使用曾交付 v0 的产品版本,并保留原请求与捕获。当前 C1 中 +`--turn-envelope` 是完整契约的兼容别名,已有实验按其记录版本解释。 + +The benchmark heartbeat worker verifies a historical `turn_envelope` request +against the pinned CLI's v0 response before launching a model. Its read-only +preview and eligibility are recorded in the wake receipt. See the [runtime +guide](../../../benchmark/runtime/RUNTIME.md) for supported configurations. +Model cost, decision quality and long-run adoption require a separately pinned +experiment. A capture carries its original read and recovery scope. Whole-Goal requirements and selected work have different sources. After final delivery admission, the shared typed interaction owner adds these pre-work reads diff --git a/tests/control_plane/test_codex_context_capture.py b/tests/control_plane/test_codex_context_capture.py index 376fe5dbea..d1f31382cb 100644 --- a/tests/control_plane/test_codex_context_capture.py +++ b/tests/control_plane/test_codex_context_capture.py @@ -23,7 +23,7 @@ from test_quota_settlement_cli import AGENT_ID, GOAL_ID, TODO_ID, _spend_run_count, _write_fixture from tests.control_plane.codex_checkpoint_fixture import ( local_inference_provider, delivered_context as _delivered, last_context_page as _last_page, - reference_metadata_oracle, + native_process_continuation, reference_metadata_oracle, ) from tests.control_plane.test_heartbeat_envelope_transport import private_handoff # noqa: F401 -- shared private socket fixture from tests.control_plane.test_codex_context_boundary import _app_server, _configure_provider, _input_text, _native_fixture @@ -36,9 +36,9 @@ (host, surface, failed, False, False) for host, surface in [("exec", "quota_packet"), ("app_server", "turn_envelope"), ("code_mode", "quota_packet"), ("code_mode", "turn_envelope")] for failed in [False, True] -] + [("benchmark", surface, False, False, False) for surface in ("quota_packet", "turn_envelope")] - + [(host, surface, False, True, False) for host in ("benchmark", "code_mode") - for surface in ("quota_packet", "turn_envelope")] +] + [("benchmark", surface, False, False, False) for surface in ("quota_packet", "complete_turn")] + + [("benchmark", surface, False, True, False) for surface in ("quota_packet", "complete_turn")] + + [("code_mode", surface, False, True, False) for surface in ("quota_packet", "turn_envelope")] + [("code_mode", surface, False, True, True) for surface in ("quota_packet", "turn_envelope")]) def test_native_cli_capture_delta_and_compact_recovery(tmp_path, monkeypatch, host, surface, failed_primary, resumed, adaptive, hook_loss=None, shared_root=False, checkpoint_handoff=False, handoff_socket=None, native_pragma=False, native_read_batch=2, native_history_tokens=None, resume_loss=None, bounded_pages=False, ssh_goal=False, reader_output_bytes=None, recovery_resume=False, working_brief=False): # Real worker qualification follows returned cursors; output budgets can @@ -140,14 +140,14 @@ def test_native_cli_capture_delta_and_compact_recovery(tmp_path, monkeypatch, ho "--decision-output-root", str(root), "--context-delivery", "on", *host_args, context_option, str(lifecycle)] if handoff_socket: command += ["--decision-handoff-socket", handoff_socket] - if surface == "turn_envelope": - command.append("--turn-envelope") + display_args = {"turn_envelope": ["--turn-envelope"], "complete_turn": ["--complete-turn"]}.get(surface, []) + command.extend(display_args) generated_guard = None if shared_root: generated = subprocess.run([*cli, "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", "heartbeat-prompt", "--thin", "--goal-id", GOAL_ID, "--agent-id", AGENT_ID, "--decision-output-root", str(root), - "--context-delivery", "on", *host_args, context_option, str(lifecycle), *(["--turn-envelope"] if surface == "turn_envelope" else [])], + "--context-delivery", "on", *host_args, context_option, str(lifecycle), *display_args], capture_output=True, text=True, timeout=15) assert generated.returncode == 0, generated.stderr or generated.stdout prompt = json.loads(generated.stdout) @@ -245,6 +245,10 @@ def native_state(): def response(request, number, message): nonlocal command, root phase = stages["phase"] + if phase in {"cold_ready", "warm_ready", "cold_read", "warm_read", "resume_read", "recovered"}: + pending = native_process_continuation(request, number, code_mode=host == "code_mode") + if pending is not None: + return pending if bounded_pages and phase in {"cold_read", "warm_read", "resume_read", "recovered"}: page = _last_page(request) assert page and page["ok"], page @@ -549,7 +553,8 @@ def observed_response(request, number, message): project_hooks = project / ".codex/hooks.json" project_hooks.write_text(json.dumps({"hooks": {"SessionStart": [{"hooks": [{ "type": "command", "command": shlex.join(["touch", str(unrelated)])}]}]}})) - env.update(LOOPX_CONTEXT_DELIVERY="on", LOOPX_TURN_ENVELOPE=str(int(surface == "turn_envelope")), + # Current benchmark delivery exercises default and explicit Complete. + env.update(LOOPX_CONTEXT_DELIVERY="on", LOOPX_COMPLETE_TURN=str(int(surface == "complete_turn")), LOOPX_EXECUTION_MODE="heartbeat", LOOPX_ITERATION_CONTEXT="resume", LOOPX_TASK_ENTRY="seeded-todo", LOOPX_TASK_STAGE="execute", LOOPX_CODEX_HOME=str(home), LOOPX_PROJECT=str(project), LOOPX_SHARED_SKILLS=str(skills), @@ -700,8 +705,13 @@ def held(*args, **kwargs): assert not (workspace / "traces").exists() projection = Path(wires["warm"]["recovery"]["path"]) assert hashlib.sha256(projection.read_bytes()).hexdigest() == wires["warm"]["recovery"]["sha256"] - current = prepare_context_delivery(json.loads(projection.read_bytes()), scope={k: wires["warm"]["target"][k] for k in ("goal_id", "agent_id", "session_id", "context_epoch")}, - capture_id=wires["warm"]["target"]["capture_id"], surface=surface) + captured = json.loads(projection.read_bytes()) + complete = captured.get("schema_version") == "loopx_turn_envelope_v1" + if host == "benchmark": + assert complete + delivery_surface = "complete_turn" if complete else surface + current = prepare_context_delivery(captured, scope={k: wires["warm"]["target"][k] for k in ("goal_id", "agent_id", "session_id", "context_epoch")}, + capture_id=wires["warm"]["target"]["capture_id"], surface=delivery_surface) baseline = wires["cold"] if resumed and qualified_resume and not resume_loss: assert wires["warm"]["base"] == baseline["target"] @@ -713,12 +723,12 @@ def held(*args, **kwargs): assert current["target"]["context_epoch"] != baseline["target"]["context_epoch"] baseline = prepare_context_delivery(baseline["context"], scope={ k: current["target"][k] for k in ("goal_id", "agent_id", "session_id", "context_epoch")}, - capture_id=baseline["target"]["capture_id"], surface=surface) + capture_id=baseline["target"]["capture_id"], surface=delivery_surface) assert baseline["target"] == wires["warm"]["base"] restored = restore_context_delivery(wires["warm"], baseline=baseline, current=current) assert restored["context"] == current["context"] and restored["mode"] == wires["warm"]["mode"] for document in (wires["cold"]["context"], current["context"]): - context = (document["work_context"] if surface == "turn_envelope" else + context = (document["work_context"] if delivery_surface == "turn_envelope" else document["interaction_contract"]["agent_channel"]["work_context"]) selected = next(source["content"]["todo"] for source in context["sources"] if source["source"] == "selected_todo") @@ -772,7 +782,7 @@ def test_native_generated_large_page_configuration(tmp_path, monkeypatch, surfac bounded_pages=True, native_pragma=True, native_read_batch=1, recovery_resume=True) -@pytest.mark.parametrize("surface", ["quota_packet", "turn_envelope"]) +@pytest.mark.parametrize("surface", ["quota_packet", "complete_turn"]) @pytest.mark.parametrize("loss", ["replace", "unqualified_profile"]) def test_native_worker_resume_history_change_reads_current_full(tmp_path, monkeypatch, surface, loss): test_native_cli_capture_delta_and_compact_recovery(tmp_path, monkeypatch, @@ -781,7 +791,8 @@ def test_native_worker_resume_history_change_reads_current_full(tmp_path, monkey @pytest.mark.parametrize("host,surface,hook_loss", [ (host, surface, loss) for host in ("code_mode", "benchmark") - for surface, loss in (("quota_packet", "missing"), ("turn_envelope", "killed")) + for surface, loss in (("quota_packet", "missing"), + ("complete_turn" if host == "benchmark" else "turn_envelope", "killed")) ]) def test_native_missing_hook_recovers_from_persistent_cli_route(tmp_path, monkeypatch, host, surface, hook_loss): test_native_cli_capture_delta_and_compact_recovery(tmp_path, monkeypatch, @@ -822,7 +833,7 @@ def test_native_bounded_command_chain_recovers_with_unknown_history_limit(tmp_pa native_read_batch=1, bounded_pages=True) -@pytest.mark.parametrize("surface", ["quota_packet", "turn_envelope"]) +@pytest.mark.parametrize("surface", ["quota_packet", "complete_turn"]) def test_native_benchmark_context_keeps_checkpoint_handoff(tmp_path, monkeypatch, surface): test_native_cli_capture_delta_and_compact_recovery(tmp_path, monkeypatch, "benchmark", surface, False, True, True, "missing", checkpoint_handoff=True) From b1ba1db18eaa7d40a292c7d52746709bc7193ae1 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:50:17 +0800 Subject: [PATCH 2/2] Preserve the phase stop during a Short carrier preview Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/runtime/worker.py | 16 +++++---- benchmark/tests/test_worker_phase_budget.py | 39 +++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index 72b520d2a8..93ebf2c394 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -529,8 +529,8 @@ def _phase_timeout(timeout: float | None, deadline: float | None) -> tuple[float return timeout, False -def _qualify_short_envelope(env: dict[str, str], receipt: dict) -> None: - """Check the pinned product's carrier before a Short arm can start a model.""" +def _qualify_short_envelope(env: dict[str, str], receipt: dict) -> bool: + """Verify the pinned Short carrier, returning False if the phase expires.""" from loopx.control_plane.quota.turn_envelope import ( COMPLETE_TURN_DECISION_VERSION, TURN_ENVELOPE_SCHEMA_VERSION, @@ -542,8 +542,6 @@ def _qualify_short_envelope(env: dict[str, str], receipt: dict) -> None: "eligible": False, } receipt.update(quota_input_contract=contract, host_invoked=False) - deadline = env.get("LOOPX_PHASE_DEADLINE_EPOCH") - timeout, _ = _phase_timeout(120, float(deadline) if deadline else None) # This preview runs the same product projection without a Turn, capture, # or admission. Keep optional delivery out of the carrier observation. command = loopx_command(env) + [ @@ -552,11 +550,16 @@ def _qualify_short_envelope(env: dict[str, str], receipt: dict) -> None: "--available-capability", "shell", "--available-capability", "filesystem_write", "--turn-envelope", "--context-delivery", "off", ] + deadline = env.get("LOOPX_PHASE_DEADLINE_EPOCH") + timeout, phase_limited = _phase_timeout(120, float(deadline) if deadline else None) try: result = subprocess.run(command, cwd=env["LOOPX_PROJECT"], env=env, capture_output=True, text=True, timeout=timeout, check=True) payload = read_json(result.stdout) except (OSError, ValueError, subprocess.SubprocessError) as exc: + if phase_limited and isinstance(exc, subprocess.TimeoutExpired): + receipt.update(ok=True, budget_exhausted=True) + return False raise ValueError("Short envelope requires a successful v0 quota preview from the pinned product.") from exc if payload.get("ok") is not True: raise ValueError("Short envelope requires a successful v0 quota preview from the pinned product.") @@ -574,6 +577,7 @@ def _qualify_short_envelope(env: dict[str, str], receipt: dict) -> None: raise ValueError("Short envelope requires the direct v0 guard. " "A fresh_replan checkpoint preloads a different view.") contract["eligible"] = True + return True def run_once(env: dict[str, str]) -> dict: @@ -679,8 +683,8 @@ def run_once(env: dict[str, str]) -> dict: execution, timeout_seconds=(remaining - reserve if execution.timeout_seconds is None else min(execution.timeout_seconds, remaining - reserve)) ) - if execution.turn_envelope: - _qualify_short_envelope(env, receipt) + if execution.turn_envelope and not _qualify_short_envelope(env, receipt): + return receipt prepare_codex_home( home, execution=execution, diff --git a/benchmark/tests/test_worker_phase_budget.py b/benchmark/tests/test_worker_phase_budget.py index a8895db90f..2efdece8bf 100644 --- a/benchmark/tests/test_worker_phase_budget.py +++ b/benchmark/tests/test_worker_phase_budget.py @@ -259,6 +259,45 @@ def setup_failure(*a, **kw): assert not (project / "native-starts").exists() +@pytest.mark.parametrize("phase_limited", [True, False], ids=["phase", "preview"]) +def test_short_preview_timeout_preserves_its_selected_limit(tmp_path, monkeypatch, phase_limited): + import subprocess + + env, project, runtime, _source, state, clock = _phase_worker( + tmp_path, monkeypatch, work_seconds=1 if phase_limited else 300) + env["LOOPX_TURN_ENVELOPE"] = "1" + before = state.read_bytes() + original_run = subprocess.run + previews = [] + + def time_out_preview(argv, **kwargs): + if argv[0] != env["LOOPX_CLI"] or "--dry-run" not in argv: + return original_run(argv, **kwargs) + assert "--turn-envelope" in argv and "--turn-instance-id" not in argv + previews.append(kwargs["timeout"]) + # The clock can pass the deadline before handling either error. + clock[0] = float(env["LOOPX_PHASE_DEADLINE_EPOCH"]) + 1 + raise subprocess.TimeoutExpired(argv, kwargs["timeout"]) + + monkeypatch.setattr(worker.subprocess, "run", time_out_preview) + if phase_limited: + receipt = worker.run_once(env) + assert receipt["ok"] and receipt["budget_exhausted"] is True + assert "host_failure" not in receipt + else: + with pytest.raises(ValueError, match="Short envelope requires a successful v0 quota preview"): + worker.run_once(env) + receipt = _receipt(env) + assert not receipt["ok"] and not receipt.get("budget_exhausted") + + assert previews == [1 if phase_limited else 120] + assert receipt["host_invoked"] is False and not receipt["quota_input_contract"]["eligible"] + assert not Path(env["LOOPX_CODEX_HOME"]).exists() + assert not (project / "native-starts").exists() + assert fixture._heartbeat_receipt_count(runtime, receipt["turn_id"]) == 0 + assert state.read_bytes() == before + + def test_checkpoint_internal_timeout_is_not_reclassified_after_deadline(tmp_path, monkeypatch): import subprocess from benchmark.runtime import checkpoint_wake