diff --git a/apps/presentation/dashboard/src/data/chat.ts b/apps/presentation/dashboard/src/data/chat.ts index c3db5a2c22..451aaebd1e 100644 --- a/apps/presentation/dashboard/src/data/chat.ts +++ b/apps/presentation/dashboard/src/data/chat.ts @@ -470,6 +470,7 @@ export const typedActionProposalSchema = z.object({ validation_evidence: z.array(z.string().refine((value) => value.trim().length > 0, "Validation evidence must be non-blank text")), available_transitions: z.array(z.enum(["apply", "cancel", "regenerate", "reject", "defer"])), status: z.enum(["preview_ready", "applying", "gated", "failed", "rejected", "deferred", "cancelled", "stale", "applied"]), + task_execution_dispatch_pending: z.literal(true).optional(), receipt: z.record(z.string(), z.unknown()).nullable(), stale: z.record(z.string(), z.unknown()).nullable(), gate: z.record(z.string(), z.unknown()).nullable().optional(), diff --git a/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx b/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx index 1ee6ce52e3..e4364606f6 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/context-drawer.tsx @@ -1231,7 +1231,9 @@ export function ContextDrawer({ agents, attentionHistory = [], onSelectAttention

{selection.item.title}

{selection.item.todoId && (selection.item.actionKind === "gate.resolve" || selection.item.actionKind === "todo.update") ?
: null} - {selection.item.impact ?

{selection.item.impact}

: null} + {selection.item.reviewPlan?.retryOriginal && selection.item.errorMessage + ?

{selection.item.errorMessage}

+ : selection.item.impact ?

{selection.item.impact}

: null} {selection.item.reviewPlan && !selection.item.reviewPlan.creationExecution && !selection.item.reviewPlan.retryOriginal && !taskExecution && selection.item.actionKind !== "team.plan" && !(decisionFrame && ["ready", "applied"].includes(selection.item.status)) ?

{operationUnknown ? t("actionReview.operation_reconcile_original") : selection.item.reviewPlan.operationFrame?.kind === "inactive" diff --git a/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx b/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx index 94e96ece80..66f5c99f70 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/personal-workspace-page.tsx @@ -1832,10 +1832,21 @@ export function PersonalWorkspacePage({ // not have a Turn to revisit. .filter(proposal => proposal.actionKind !== "todo.create" || proposal.status !== "applied" || Boolean(proposal.reviewPlan?.taskExecution)); - const restored = Object.fromEntries(restoreable.map((proposal) => [proposal.previewId, proposal])); const previousIds = restoredProposalIdsRef.current; - restoredProposalIdsRef.current = new Set(Object.keys(restored)); - setProposals((current) => ({...Object.fromEntries(Object.entries(current).filter(([id]) => !previousIds.has(id))), ...restored})); + restoredProposalIdsRef.current = new Set(restoreable.map(proposal => proposal.previewId)); + setProposals((current) => { + const next = Object.fromEntries(Object.entries(current).filter(([id]) => !previousIds.has(id))); + for (const proposal of restoreable) { + const previous = current[proposal.previewId]; + // The stored confirmation outlives a failed dispatch attempt. + // Keep the local reason until readback observes a different confirmation. + const retryError = proposal.reviewPlan?.retryOriginal && previous?.reviewPlan?.retryOriginal + && previous.actionKind === proposal.actionKind && previous.goalId === proposal.goalId + && previous.updatedAt === proposal.updatedAt ? previous.errorMessage : undefined; + next[proposal.previewId] = retryError ? { ...proposal, errorMessage: retryError } : proposal; + } + return next; + }); // The manager conversation shows the cards this channel offered: a team // plan the steward proposed from here is confirmed here, instead of the // owner hunting for the Goal whose workspace happens to hold the card. @@ -2140,7 +2151,7 @@ export function PersonalWorkspacePage({ : null; const lifecycleChange = options.lifecycleProjection ?? inferredLifecycleChange; setActionFeedback(t("feedback.applying", { title: proposal.title })); - const applying = { ...proposal, reviewPlan: proposal.reviewPlan ? { ...proposal.reviewPlan, interaction: "pending" as const, reason: "apply_pending" as const, canApply: false as const } : undefined, status: "applying" as const }; + const applying = { ...proposal, errorMessage: undefined, reviewPlan: proposal.reviewPlan ? { ...proposal.reviewPlan, interaction: "pending" as const, reason: "apply_pending" as const, canApply: false as const } : undefined, status: "applying" as const }; setProposals((current) => ({ ...current, [proposal.previewId]: applying })); if (showDrawer) setSelection({ item: applying, kind: "proposal" }); let applyingSelection = selectionRevision.current; diff --git a/loopx/chat/todo_actions.py b/loopx/chat/todo_actions.py index 4bdc1150f2..026b4d82bf 100644 --- a/loopx/chat/todo_actions.py +++ b/loopx/chat/todo_actions.py @@ -106,8 +106,58 @@ def _read_task_start( raise ValueError("The original Task Session is ambiguous") return (sessions[0] if sessions else None), None + def _pending_applied_task_receipt(self, proposal: dict[str, Any]) -> dict[str, Any] | None: + """Find the exact accepted Turn whose worker has yet to start.""" + parameters = proposal.get("normalized_parameters") or {} + receipt = proposal.get("receipt") or {} + if (proposal.get("status") != "applied" or proposal.get("action_kind") != "todo.create" + or parameters.get("start_execution") is not True + or proposal.get("permission_classification") != "durable_write" + or (proposal.get("context") or {}).get("binding_id") + or receipt.get("outcome") != "task_execution_started" + or receipt.get("projection_verified") is not True): + return None + resources = receipt.get("resource_ids") + canonical = receipt.get("canonical_receipt") + if not isinstance(resources, dict) or not isinstance(canonical, dict): + return None + goal_id = parameters.get("goal_id") + todo_id, sid, tid = (resources.get(key) for key in ("todo_id", "session_id", "turn_id")) + if (not all(isinstance(value, str) and value for value in (goal_id, todo_id, sid, tid)) + or resources.get("goal_id") != goal_id or canonical.get("goal_id") != goal_id + or canonical.get("todo_id") != todo_id or canonical.get("status") != "applied" + or canonical.get("preview_id") != proposal.get("expected_state_fingerprint")): + return None + chat = getattr(self.runtime_controller, "store", None) or self.chat_store + if chat is None: + return None + try: + turn = chat.load_turn(sid, tid) + except (OSError, ValueError): + return None + if (turn and turn.get("session_id") == sid and turn.get("turn_id") == tid + and turn.get("client_turn_id") == f"task-start-{proposal['proposal_id']}" + and turn.get("status") == "queued" and not (turn.get("started_at") or turn.get("upstream_turn_id"))): + return resources + return None + + def project_task_dispatch(self, proposal: dict[str, Any]) -> dict[str, Any]: + if self._pending_applied_task_receipt(proposal) is not None: + return {**proposal, "task_execution_dispatch_pending": True} + return proposal + + def _recover_applied_task(self, proposal: dict[str, Any]) -> dict[str, Any] | None: + execution = self._pending_applied_task_receipt(proposal) + if execution is None: + return self._turn_from_receipt(proposal.get("receipt")) + return self._start_confirmed_task( + proposal["proposal_id"], proposal["normalized_parameters"], execution["todo_id"], {}, + applied_execution=execution, + ) + def _start_confirmed_task( self, proposal_id: str, parameters: dict[str, Any], todo_id: str, steps: dict[str, Any], + *, applied_execution: dict[str, Any] | None = None, ) -> dict[str, Any]: from ..chat_actions import _opaque @@ -122,7 +172,7 @@ def _start_confirmed_task( channel_id = f"task.{todo_id}" client_turn_id = f"task-start-{proposal_id}" opened = steps.get("task_session_opened") - execution = steps.get("execution_started") + execution = applied_execution if applied_execution is not None else steps.get("execution_started") session, turn = self._read_task_start( chat, goal_id=goal_id, agent_id=endpoint, channel_id=channel_id, client_turn_id=client_turn_id, checkpoint=execution if execution is not None else opened, @@ -161,7 +211,7 @@ def _start_confirmed_task( # A past soft claim still needs a read-only current-owner check. if update_goal_todo(**claim_args, dry_run=True)["changed"]: raise ValueError("The reviewed Task assignment changed before launch") - if not isinstance(steps.get("todo_assigned"), dict): + if applied_execution is None and not isinstance(steps.get("todo_assigned"), dict): self.store.save_checkpoint( proposal_id, step="todo_assigned", receipt={"todo_id": todo_id, "agent_id": owner}, ) diff --git a/loopx/chat_actions.py b/loopx/chat_actions.py index 919e074635..c0a09e902c 100644 --- a/loopx/chat_actions.py +++ b/loopx/chat_actions.py @@ -1643,7 +1643,7 @@ def _apply_owner_action(self, proposal_id: str, *, steward_context: dict[str, ob proposal = self.store.load(proposal_id) if proposal is None: raise KeyError("typed Chat action proposal was not found") - if proposal.get("status") == "applied": + if proposal.get("status") == "applied" and self._pending_applied_task_receipt(proposal) is None: return { "proposal": proposal, "turn": self._turn_from_receipt(proposal.get("receipt")), @@ -1680,7 +1680,7 @@ def _apply_owner_action(self, proposal_id: str, *, steward_context: dict[str, ob def _apply_confirmed_action(self, proposal_id: str) -> dict[str, object]: proposal = self.store.start_apply(proposal_id) if proposal["status"] == "applied": - return {"proposal": proposal, "turn": self._turn_from_receipt(proposal.get("receipt"))} + return {"proposal": proposal, "turn": self._recover_applied_task(proposal)} action_kind = str(proposal.get("action_kind") or "") parameters = proposal.get("normalized_parameters") if not isinstance(parameters, dict): diff --git a/loopx/chat_server.py b/loopx/chat_server.py index 947afd3e83..2b0a36a3bc 100644 --- a/loopx/chat_server.py +++ b/loopx/chat_server.py @@ -1412,7 +1412,7 @@ def _action_snapshot(self, proposal_id: str) -> None: { "ok": True, "schema_version": "loopx_chat_action_v1", - "proposal": proposal, + "proposal": self.server.action_service.project_task_dispatch(proposal), } ) @@ -1436,7 +1436,7 @@ def _action_list(self) -> None: { "ok": True, "schema_version": "loopx_chat_action_list_v1", - "proposals": proposals, + "proposals": [self.server.action_service.project_task_dispatch(item) for item in proposals], } ) diff --git a/loopx/control_plane/presentation/action_review_plan.ts b/loopx/control_plane/presentation/action_review_plan.ts index 9bd7518488..214c280ce4 100644 --- a/loopx/control_plane/presentation/action_review_plan.ts +++ b/loopx/control_plane/presentation/action_review_plan.ts @@ -609,6 +609,11 @@ export function compileActionReviewPlan(proposalValue: unknown, nowMs?: number): return held("repair", "readback_unverified"); } const taskExecution = taskExecutionReadback(proposal, receipt, identity.sourceFingerprint); + if (taskExecution && proposal.task_execution_dispatch_pending === true + && proposal.permission_classification === "durable_write" && identity.proposalId + && !objectValue(proposal.context)?.binding_id) { + return {...finish({interaction: "review", canApply: true, reason: "task_start_retry"}), retryOriginal: true}; + } const completed = held("completed", "readback_verified"); return taskExecution ? {...completed, taskExecution} : completed; } diff --git a/tests/control_plane/test_chat_task_start_recovery.py b/tests/control_plane/test_chat_task_start_recovery.py index 56dd2af5cf..f36aa4292a 100644 --- a/tests/control_plane/test_chat_task_start_recovery.py +++ b/tests/control_plane/test_chat_task_start_recovery.py @@ -292,3 +292,80 @@ def after_completion(proposal_id, *, step, receipt): assert calls.count("thread/start") == 1 assert calls.count("turn/start") == (0 if interruption == "accepted-then-stopped" else 1) assert _snapshot(runtime) == before + + +@pytest.mark.parametrize("stopped", [False, True], ids=["active", "stopped"]) +def test_applied_confirmation_retries_the_original_undispatched_turn(task_start, monkeypatch, stopped): + actions, request, chat, controller, reopen, runtime, capture, proposal_id = task_start + path = f"/api/actions/{proposal_id}" + + # The server persisted acceptance and returned it, but its scheduled worker + # never wrote the first durable start fact before the process went away. + with monkeypatch.context() as faults: + faults.setattr(controller, "_start_accepted_turn_worker", lambda **kwargs: True) + code, original = request(f"{path}/apply", {}) + assert code == 202 and original["proposal"]["status"] == "applied", original + receipt = original["proposal"]["receipt"] + ids = receipt["resource_ids"] + sid, tid = ids["session_id"], ids["turn_id"] + queued = chat.load_turn(sid, tid) + assert queued["status"] == "queued" and queued["started_at"] is None + if stopped: + _stop_goal(request) + before = _snapshot(runtime) + chat, controller = reopen() + + # Looking at the original confirmation or its goal's list must never start it. + code, detail = request(path) + assert code == 200 and detail["proposal"]["receipt"] == receipt + code, history = request("/api/actions?goal_id=report") + assert code == 200, history + listed = next(row for row in history["proposals"] if row["proposal_id"] == proposal_id) + assert chat.load_turn(sid, tid) == queued + assert all(json.loads(line)["method"] != "turn/start" for line in capture.read_text().splitlines()) + + if stopped: + code, retried = request(f"{path}/apply", {}) + assert code == 400 and "Goal is no longer active" in retried["error"], retried + assert chat.load_turn(sid, tid) == queued + else: + entered, proceed = Event(), Event() + start = controller._start_accepted_turn_worker + + def pause_ready_dispatch(**kwargs): + entered.set() + assert proceed.wait(20), "the original retry did not resume" + return start(**kwargs) + + monkeypatch.setattr(controller, "_start_accepted_turn_worker", pause_ready_dispatch) + with ThreadPoolExecutor(max_workers=1) as pool: + original_retry = pool.submit(request, f"{path}/apply", {}) + try: + assert entered.wait(10), "the original retry did not reach dispatch" + code, concurrent = request(f"{path}/apply", {}) + assert code == 409 and concurrent["error_code"] == "action_conflict", concurrent + finally: + proceed.set() + code, retried = original_retry.result(timeout=25) + assert code == 202 and retried["proposal"]["receipt"] == receipt, retried + assert retried["turn"]["created"] is False + assert retried["turn"]["turn_id"] == tid + assert controller.wait_for_turn(session_id=sid, turn_id=tid, timeout_sec=3)["status"] == "completed" + code, again = request(f"{path}/apply", {}) + assert code == 202 and again["proposal"]["receipt"] == receipt + assert again["turn"]["status"] == "completed" + code, complete = request(path) + assert code == 200 and "task_execution_dispatch_pending" not in complete["proposal"] + + assert detail["proposal"]["task_execution_dispatch_pending"] is True + assert listed["task_execution_dispatch_pending"] is True + assert "task_execution_dispatch_pending" not in actions.load(proposal_id) + assert actions.load(proposal_id)["receipt"] == receipt + assert _snapshot(runtime) == before + assert [row["session_id"] for row in chat.session_candidates( + goal_id="report", agent_id="codex", channel_id=f"task.{ids['todo_id']}", + )] == [sid] + assert len([row for row in chat.messages(sid) if row.get("role") == "user"]) == 1 + calls = [json.loads(line)["method"] for line in capture.read_text().splitlines()] + assert calls.count("thread/start") == 1 + assert calls.count("turn/start") == (0 if stopped else 1) diff --git a/tests/control_plane/test_new_goal_storage_defaults.py b/tests/control_plane/test_new_goal_storage_defaults.py index b294eec86a..f2bc388873 100644 --- a/tests/control_plane/test_new_goal_storage_defaults.py +++ b/tests/control_plane/test_new_goal_storage_defaults.py @@ -342,10 +342,13 @@ def app(environment): thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() - def request(path, body): + def request(path, body=None): connection = http.client.HTTPConnection(*server.server_address, timeout=30) try: - connection.request("POST", path, json.dumps(body), {"Content-Type": "application/json"}) + if body is None: + connection.request("GET", path) + else: + connection.request("POST", path, json.dumps(body), {"Content-Type": "application/json"}) response = connection.getresponse() return response.status, json.loads(response.read()) finally: diff --git a/tests/control_plane_ts/action_review_plan.test.ts b/tests/control_plane_ts/action_review_plan.test.ts index bc52c22407..219ce826d9 100644 --- a/tests/control_plane_ts/action_review_plan.test.ts +++ b/tests/control_plane_ts/action_review_plan.test.ts @@ -66,6 +66,18 @@ test("confirmed Task dispatch resolves only to the receipt's own execution Turn" ]) { assert.equal(compileActionReviewPlan({...proposal, ...change}).taskExecution, undefined); } + const pending = {...proposal, permission_classification: "durable_write", task_execution_dispatch_pending: true}; + assert.equal(compileActionReviewPlan(pending).reason, "task_start_retry"); + assert.equal(compileActionReviewPlan(pending).canApply, true); + assert.equal(compileActionReviewPlan(pending).retryOriginal, true); + for (const change of [ + {task_execution_dispatch_pending: "true"}, + {context: {binding_id: "bound-source"}}, + {permission_classification: "protected"}, + {receipt: {...receipt, resource_ids: {...resources, turn_id: ""}}}, + ]) { + assert.equal(compileActionReviewPlan({...pending, ...change}).canApply, false); + } // Older saved actions can still report creation while the Session is unknown. const legacy = compileActionReviewPlan({...proposal, receipt: {projection_verified: true, outcome: "todo_added"}}); assert.equal(legacy.interaction, "completed");