From e75e0a42f6a2c443c68c1086d31fbac25dd29817 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:19:55 +0800 Subject: [PATCH 1/9] Use model coverage in the PR review queue Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/pr_review_queue/README.md | 14 ++ .../pr_review_queue/catalog_entry.py | 2 +- .../pr_review_queue/github_source.py | 18 +- .../pr_review_queue/model_coverage.py | 139 +++++++++++++ .../pr_review_queue/selection_execution.py | 11 +- loopx/cli_commands/pr_review.py | 18 +- loopx/pr_review.py | 31 ++- tests/capabilities/test_pr_review_queue.py | 190 ++++++++++++++++++ 8 files changed, 407 insertions(+), 16 deletions(-) create mode 100644 loopx/capabilities/pr_review_queue/model_coverage.py diff --git a/loopx/capabilities/pr_review_queue/README.md b/loopx/capabilities/pr_review_queue/README.md index 21173cc08d..d7fb2eda03 100644 --- a/loopx/capabilities/pr_review_queue/README.md +++ b/loopx/capabilities/pr_review_queue/README.md @@ -661,6 +661,20 @@ published GitHub reviews are recognized (`check_review_body` is unchanged). reference as an error, and a spec-basis gap as an approval blocker. Regenerate results made under revision 13 rather than adding these fields to them. +In a Goal-bound GitHub queue, a recognized model family from the invoking +runtime can reopen an open, non-draft exact head with a declared review from +another family. The queue uses complete provider history. An independent +review from the same family suppresses repeat work even when another account +submitted a later review. A qualifying review published under the shared +author login with a matching runtime declaration also suppresses repeat work +for that model. GitHub independence still requires independent account +evidence. The declarations record their source as published provenance. + +Missing model identity or incomplete history retains the existing queue +behavior. A review with unknown provenance supplies no model-family coverage. +The review action leaves the formal conclusion, dissent, CI policy, approval +and merge-readiness rules in their existing owners. + ### Semantic alignment and CI constraint recovery The semantic triage introduced in policy revision 5 replaces universal detailed diff --git a/loopx/capabilities/pr_review_queue/catalog_entry.py b/loopx/capabilities/pr_review_queue/catalog_entry.py index b8316312da..c0e77f6a98 100644 --- a/loopx/capabilities/pr_review_queue/catalog_entry.py +++ b/loopx/capabilities/pr_review_queue/catalog_entry.py @@ -153,7 +153,7 @@ "A queue is observed only when result_completeness.complete=true; partial or failed reads are not_observed and never count as unchanged.", "Fingerprints cover exact head, formal conclusion, next action, check state, draft state, and mergeability for every open PR.", "Direction resolves from a command override, registered Agent, Goal, then machine. Forward visits other authors first, oldest first within tiers; reverse inverts the complete actionable queue before limiting. Packet and autonomous selection share that order. Inactive heads remain inventory-only. Legacy other-developers-first/owner-first input maps to forward/reverse, including reversed time ordering.", - "Only rows with a non-null review_action_kind enter review_sequence and carry review plans, templates, or evidence commands; valid exact-head conclusions remain artifact-free inventory-only rows, and only --fresh-audit-exact-head NUMBER@HEAD_OID can explicitly reopen one.", + "Only rows with a non-null review_action_kind enter review_sequence and carry review plans, templates, or evidence commands; valid exact-head conclusions retain their verdicts. A bound model-family coverage gap can request a full review of an open ready head. Explicit audits of otherwise concluded heads use --fresh-audit-exact-head NUMBER@HEAD_OID.", "Todo prose, monitor notes, and one-off author filters are not scheduling authority.", "A complete exact-head conclusion requires the five Chinese sections, a state-aligned English verdict, and formal state or the verdict-specific titled author-owned fallback.", "Every merge must rerun the Goal-scoped merge-readiness gate for the reviewed exact head; the resulting compact observation suppresses only unchanged qualification work, and any head, base, review, CI, thread, draft, merge-state, or PR-state change reopens it.", diff --git a/loopx/capabilities/pr_review_queue/github_source.py b/loopx/capabilities/pr_review_queue/github_source.py index f8c0c93210..f1dfcd7ecd 100644 --- a/loopx/capabilities/pr_review_queue/github_source.py +++ b/loopx/capabilities/pr_review_queue/github_source.py @@ -244,12 +244,18 @@ def fetch_complete_pr_reviews(*, repository: str, number: str, cwd: Path | None, def _attach_review_provenance(row: dict[str, Any], details: dict[str, Any], *, repository: str, number: str, cwd: Path | None, - run_gh_json: GitHubJsonRunner) -> bool: + run_gh_json: GitHubJsonRunner, + model_review_coverage: bool = False) -> bool: author = row.get("author") author = author.get("login") if isinstance(author, dict) else None # This only avoids unnecessary provider reads. Eligibility belongs to TS; # no association, state or actor-type policy is duplicated here. - if not author or not any(isinstance(review, dict) + if not author or not details["reviews"]: + return True + model_history_needed = ( + model_review_coverage and row.get("state") == "OPEN" and row.get("isDraft") is not True + ) + if not model_history_needed and not any(isinstance(review, dict) and isinstance(review.get("author"), dict) and review["author"].get("login") and review["author"]["login"].casefold() != author.casefold() for review in details["reviews"]): @@ -292,6 +298,7 @@ def attach_pr_review_details( cwd: Path | None = None, run_gh_json: GitHubJsonRunner = run_gh_json, wait_for_ci: bool = True, + model_review_coverage: bool = False, ) -> bool: """Attach complete per-PR details after the lightweight list scan.""" @@ -348,7 +355,8 @@ def attach_pr_review_details( return False details["files"] = detail_files if not _attach_review_provenance(row, details, repository=repository, number=number, - cwd=cwd, run_gh_json=run_gh_json): + cwd=cwd, run_gh_json=run_gh_json, + model_review_coverage=model_review_coverage): return False for key in detail_fields: row[key] = details[key] @@ -362,6 +370,7 @@ def scan_github_pull_request_targets( cwd: Path | None = None, run_gh_json: GitHubJsonRunner = run_gh_json, wait_for_ci: bool = True, + model_review_coverage: bool = False, ) -> dict[str, Any]: """Read only explicitly requested exact heads, without scanning a queue.""" @@ -402,6 +411,7 @@ def scan_github_pull_request_targets( repository=repository, cwd=cwd, **({"wait_for_ci": False} if not wait_for_ci else {}), + **({"model_review_coverage": True} if model_review_coverage else {}), run_gh_json=run_gh_json, ) if not details_ok: @@ -440,6 +450,7 @@ def attach_pr_review_details_concurrently( attach: Callable[..., bool] = attach_pr_review_details, run_gh_json: GitHubJsonRunner = run_gh_json, wait_for_ci: bool = True, + model_review_coverage: bool = False, ) -> list[bool]: """Read per-PR details concurrently while preserving queue order.""" @@ -454,6 +465,7 @@ def read(row: dict[str, Any]) -> bool: cwd=cwd, run_gh_json=run_gh_json, **({"wait_for_ci": False} if not wait_for_ci else {}), + **({"model_review_coverage": True} if model_review_coverage else {}), ) with ThreadPoolExecutor(max_workers=worker_count) as executor: diff --git a/loopx/capabilities/pr_review_queue/model_coverage.py b/loopx/capabilities/pr_review_queue/model_coverage.py new file mode 100644 index 0000000000..145d46f010 --- /dev/null +++ b/loopx/capabilities/pr_review_queue/model_coverage.py @@ -0,0 +1,139 @@ +"""Read model declarations for queue coverage without changing review authority.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from .approval_closeout import select_review_history +from .review_body import reviewer_declaration_lines + + +# A product alias is evidence only when paired with its declared provider. +_PROVIDERS = { + "gpt": {"openai"}, + "gemini": {"google", "google deepmind"}, + "claude": {"anthropic"}, + "deepseek": {"deepseek"}, + "qwen": {"alibaba", "alibaba cloud", "qwen"}, +} +_MODEL_NAME = re.compile(r"([a-z][a-z0-9]*)(?:[ .-][a-z0-9]+)*") +_COMPACT_DECLARATION = re.compile( + r"(?:model_agent\s*[·|]\s*)?([^·|]+?)\s*[·|]\s*([^()]+?)" + r"\s*\((runtime_reported|self_reported)(?:,\s*[\w -]+)?\)", + re.IGNORECASE, +) +_KEYED_DECLARATION = re.compile( + r"model_agent;\s*model=([^;]+);\s*provider=([^;]+);\s*" + r"declaration_source=(runtime_reported|self_reported)" + r"(?:;\s*reasoning_effort=[\w -]+)?\.?", + re.IGNORECASE, +) +_KEYED_PIPE_DECLARATION = re.compile( + r"model_agent\s*\|\s*model=([^|=]+)\s*\|\s*provider=([^|=]+)" + r"(?:\s*\|\s*reasoning_effort=[\w -]+)?" + r"\s*\|\s*source=(runtime_reported|self_reported)", + re.IGNORECASE, +) +_LEGACY_DECLARATION = re.compile( + r"model_agent\s*\|\s*(runtime_reported|self_reported)\s*\|\s*([^|]+)\s*\|\s*([^|]+)", + re.IGNORECASE, +) + + +def _model_family(model: object, provider: object) -> str | None: + if not isinstance(model, str) or not isinstance(provider, str): + return None + name = _MODEL_NAME.fullmatch(model.strip().casefold()) + if name and provider.strip().casefold() in _PROVIDERS.get(name[1], ()): + return name[1] + return None + + +def current_review_model_family(execution: Mapping[str, Any] | None) -> str | None: + if not execution or execution.get("status") != "runtime_reported": + return None + return _model_family(execution.get("model"), execution.get("provider")) + + +def published_review_model(body: object) -> tuple[str, str] | None: + if not isinstance(body, str): + return None + lines = reviewer_declaration_lines(body) + if len(lines) != 1: + return None + match = ( + _COMPACT_DECLARATION.fullmatch(lines[0]) + or _KEYED_DECLARATION.fullmatch(lines[0]) + or _KEYED_PIPE_DECLARATION.fullmatch(lines[0]) + ) + if match: + model, provider, source = match.groups() + else: + legacy = _LEGACY_DECLARATION.fullmatch(lines[0]) + if not legacy: + return None + source, model, provider = legacy.groups() + family = _model_family(model, provider) + return (family, source.casefold()) if family else None + + +def needs_cross_model_review( + *, family: str, history: dict[str, Any], selection: Mapping[str, Any], + model_reports: Sequence[tuple[str, str] | None], +) -> bool: + """Detect missing model coverage from one complete exact-head snapshot. + + Interpret provenance for queue work only. The existing history owner + still determines whether an account supplies independent review evidence. + """ + author = history["author"].casefold() + head = history["head"].casefold() + by_login: dict[str, list[dict[str, Any]]] = {} + matching_ids: set[int] = set() + other_ids: set[int] = set() + other_author_review = False + for row, declared in zip(history["reviews"], model_reports, strict=True): + login = row["login"].casefold() + by_login.setdefault(login, []).append(row) + if (row["head"].casefold() != head or row["state"] not in + {"APPROVED", "CHANGES_REQUESTED", "COMMENTED"}): + continue + review_id = row["review_id"] + if type(review_id) is not int or review_id <= 0 or declared is None: + continue + reviewed_family, source = declared + if login == author: + if row["state"] == "COMMENTED" and row["conclusion"]["valid"]: + if reviewed_family != family: + other_author_review = True + elif source == "runtime_reported": + # A shared author login cannot establish GitHub independence. + # A complete review already published by this model family + # still prevents repeat assignment to the same model. + return False + continue + (matching_ids if reviewed_family == family else other_ids).add(review_id) + + selected = selection.get("independent_review", {}).get("review_id") + if selected in matching_ids: + return False + other_independent_review = selected in other_ids + if not (other_author_review or other_independent_review or other_ids): + return False + + # Preserve each candidate account's full history, including later comments + # and dismissals. Unrelated accounts cannot overwrite the matching evidence. + candidate_ids = matching_ids | other_ids + for login, reviews in by_login.items(): + if login == author or not login or not any( + row["review_id"] in candidate_ids for row in reviews + ): + continue + evidence = select_review_history({**history, "reviews": reviews}) + review_id = evidence.get("independent_review", {}).get("review_id") + if review_id in matching_ids: + return False + other_independent_review |= review_id in other_ids + return other_author_review or other_independent_review diff --git a/loopx/capabilities/pr_review_queue/selection_execution.py b/loopx/capabilities/pr_review_queue/selection_execution.py index 5ac4636351..1d61eabe59 100644 --- a/loopx/capabilities/pr_review_queue/selection_execution.py +++ b/loopx/capabilities/pr_review_queue/selection_execution.py @@ -26,10 +26,14 @@ def _independently_reviewed(item: Mapping[str, Any]) -> bool: and evidence.get("head_oid") == item.get("head_oid") -def review_action_kind(item: Mapping[str, Any]) -> str | None: +def review_action_kind( + item: Mapping[str, Any], *, cross_model_review_needed: bool = False +) -> str | None: state = str(item.get("state") or "").upper() if item.get("is_draft") is True or state == "CLOSED": return None + if state == "OPEN" and cross_model_review_needed: + return "rereview_pull_request_exact_head" conclusion = item.get("review_conclusion") conclusion = conclusion if isinstance(conclusion, Mapping) else {} if conclusion.get("valid") is True: @@ -82,8 +86,9 @@ def materialize_review_execution( readiness_observations: Mapping[str, Mapping[str, Any]] | None = None, repository: str | None = None, review_threads: Mapping[str, Any] | None = None, + cross_model_review_needed: bool = False, ) -> dict[str, Any]: - action_kind = review_action_kind(item) + action_kind = review_action_kind(item, cross_model_review_needed=cross_model_review_needed) key = exact_head_key(item) readiness_observation: dict[str, Any] | None = None if action_kind == "qualify_pull_request_merge_readiness" and key and repository: @@ -153,6 +158,8 @@ def materialize_review_execution( "review_goal": ( "Run a fresh exact-head audit despite the valid prior conclusion. " if fresh_audit_requested + else "The current head needs review coverage from the invoking model family. " + if cross_model_review_needed else "" ) + "Follow review_execution_contract.decision_procedure: establish the goal and " "specification first, then execute this exact-head review_plan and its evidence commands.", diff --git a/loopx/cli_commands/pr_review.py b/loopx/cli_commands/pr_review.py index 5bd67abc9e..93a65d74ba 100644 --- a/loopx/cli_commands/pr_review.py +++ b/loopx/cli_commands/pr_review.py @@ -18,6 +18,7 @@ build_builtin_machine_configuration_registry, ) from ..capabilities.machine_configuration.store import read_machine_configuration +from ..capabilities.pr_review_queue.model_coverage import current_review_model_family from ..capabilities.pr_review_queue.result_check import check_review_result from ..capabilities.pr_review_queue.readiness_observation import ( observation_key, @@ -247,6 +248,7 @@ def handle_pr_review_command( if args.command != "pr-review": return None checkpoint_path: Path | None = None + execution: dict[str, Any] | None = None resolved_review_priority = DEFAULT_REVIEW_PRIORITY target_exact_heads = list(getattr(args, "target_exact_head", []) or []) readiness_observations: dict[str, dict[str, object]] = {} @@ -491,6 +493,7 @@ def handle_pr_review_command( repository = args.repo source = "github_cli" reviewer_login = None + reviewer_model_family = None if args.fixture: repository_from_fixture, pull_requests, reviewer_login = load_pr_fixture( Path(args.fixture).expanduser() @@ -514,6 +517,10 @@ def handle_pr_review_command( else: repository = repository or resolve_current_github_repository() reviewer_login = resolve_current_github_login() + if goal_id and review_agent_id and reviewer_login: + execution = current_codex_execution_identity() + reviewer_model_family = current_review_model_family(execution) + scan_options = ci_options | ({"model_review_coverage": True} if reviewer_model_family else {}) if args.autonomous_observation and not reviewer_login: raise RuntimeError( "authenticated GitHub reviewer identity is required for " @@ -525,7 +532,7 @@ def handle_pr_review_command( source_scan = scan_github_pull_request_targets( repository=repository, exact_heads=target_exact_heads, - **ci_options, + **scan_options, ) source = "github_cli_exact_targets" else: @@ -534,7 +541,7 @@ def handle_pr_review_command( limit=max(1, args.limit) + 1, state_filter=resolved_state_filter, since=args.since, - **ci_options, + **scan_options, ) pull_requests = source_scan["pull_requests"] if readiness_observations: @@ -599,6 +606,7 @@ def handle_pr_review_command( owner_logins=review_configuration.get("owner_logins", []), wait_for_ci=wait_for_ci, readiness_observations=readiness_observations, + reviewer_model_family=reviewer_model_family, ) payload["request"]["goal_id"] = goal_id payload["request"]["review_configuration"] = review_configuration @@ -685,10 +693,10 @@ def handle_pr_review_command( }, "error": error, } - # Queue inventory does not need execution metadata. Bind the invoking - # session only when this packet actually asks a reviewer to execute work. + # Publish execution provenance only on packets that ask for work. if not args.fixture and any(item.get("review_action_kind") for item in payload.get("pull_requests", [])): - execution = current_codex_execution_identity() + if execution is None: + execution = current_codex_execution_identity() payload["reviewer_execution"] = execution if execution["status"] == "runtime_reported": for item in payload["pull_requests"]: diff --git a/loopx/pr_review.py b/loopx/pr_review.py index 2238eacf4b..919bb1d7f4 100644 --- a/loopx/pr_review.py +++ b/loopx/pr_review.py @@ -221,6 +221,7 @@ def scan_github_pull_requests( state_filter: str = "open", since: str | None = None, wait_for_ci: bool = True, + model_review_coverage: bool = False, ) -> dict[str, Any]: repo_args = ["--repo", repo] if repo else [] api_repository = repo or resolve_current_github_repository(cwd=cwd) @@ -285,6 +286,7 @@ def append_state(state: str) -> None: cwd=cwd, attach=_attach_pr_review_details, **({"wait_for_ci": False} if not wait_for_ci else {}), + **({"model_review_coverage": True} if model_review_coverage else {}), run_gh_json=_run_gh_json, ) detail_read_failures = sum(not result for result in detail_results) @@ -785,6 +787,7 @@ def _review_conclusion( *, reviewer_login: str | None, behavior_bearing: bool = True, + reviewer_model_family: str | None = None, ) -> dict[str, Any]: head_oid = str(pr.get("headRefOid") or pr.get("head_oid") or "").strip() pr_author = str( @@ -859,7 +862,7 @@ def evaluate(review: Mapping[str, Any]) -> dict[str, Any]: if len(reviews) == 1 and pr.get("reviews_complete") is not True: return conclusions[0] from .capabilities.pr_review_queue.approval_closeout import select_review_history - selection = select_review_history({"head": head_oid, "author": pr_author, + history = {"head": head_oid, "author": pr_author, "reviews_complete": pr.get("reviews_complete") is True, "reviews": [{"state": conclusion["state"], "login": str(_as_dict(review.get("author")).get("login") or ""), @@ -870,10 +873,22 @@ def evaluate(review: Mapping[str, Any]) -> dict[str, Any]: "is_bot": _as_dict(review.get("author")).get("is_bot"), "submitted_at": review.get("submittedAt"), "conclusion": conclusion} - for review, conclusion in zip(reviews, conclusions)]}) + for review, conclusion in zip(reviews, conclusions)]} + selection = select_review_history(history) index = cast(int, selection["index"]) - return conclusions[index] | ({"independent_review": selection["independent_review"]} - if "independent_review" in selection else {}) + independent = selection.get("independent_review") + conclusion = conclusions[index] + if reviewer_model_family and history["reviews_complete"] and (conclusion["valid"] or independent): + from .capabilities.pr_review_queue.model_coverage import needs_cross_model_review, published_review_model + + if needs_cross_model_review( + family=reviewer_model_family, + history=history, + selection=selection, + model_reports=[published_review_model(review.get("body")) for review in reviews], + ): + conclusion["cross_model_review_needed"] = True + return conclusion | ({"independent_review": independent} if independent else {}) def _review_sequence_entry(item: dict[str, Any], *, rank: int) -> dict[str, Any]: @@ -914,6 +929,7 @@ def _normalize_pr( readiness_observations: Mapping[str, Mapping[str, object]] | None = None, repository: str | None = None, owner_authored: bool | None = None, + reviewer_model_family: str | None = None, ) -> dict[str, Any]: files = _files(pr) checks = _checks(pr) @@ -937,8 +953,10 @@ def _normalize_pr( else 0.0 ) conclusion = _review_conclusion(pr, reviewer_login=reviewer_login, - behavior_bearing=bool({item["area"] for item in files} & (CODE_AREAS | BEHAVIORAL_POLICY_AREAS))) + behavior_bearing=bool({item["area"] for item in files} & (CODE_AREAS | BEHAVIORAL_POLICY_AREAS)), + reviewer_model_family=reviewer_model_family if raw_state == "OPEN" and not pr.get("isDraft") else None) independent = conclusion.pop("independent_review", None) + cross_model_review_needed = conclusion.pop("cross_model_review_needed", False) item: dict[str, Any] = { "number": number, "title": _redact_text(pr.get("title"), limit=180), @@ -997,6 +1015,7 @@ def _normalize_pr( readiness_observations=readiness_observations or {}, repository=repository, review_threads=_as_dict(pr.get("review_thread_summary")), + cross_model_review_needed=cross_model_review_needed, ) ) if owner_authored is not None: @@ -1030,6 +1049,7 @@ def build_pr_review_packet( owner_logins: Sequence[str] = (), wait_for_ci: bool = True, readiness_observations: Mapping[str, Mapping[str, object]] | None = None, + reviewer_model_family: str | None = None, ) -> dict[str, Any]: normalized_state_filter = normalize_pr_state_filter(state_filter) normalized_priority = normalize_review_priority(review_priority) @@ -1054,6 +1074,7 @@ def build_pr_review_packet( readiness_observations=readiness_observations, repository=repository, owner_authored=owner_identity["owner_authored"][index] if owner_identity else None, + reviewer_model_family=reviewer_model_family, ) for index, item in enumerate(pull_requests) ] diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 94dc8966a6..511f90e7ee 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -696,3 +696,193 @@ def test_projection_ack_must_match_prior_candidate() -> None: assert "prior candidate" in str(exc) else: raise AssertionError("an unselected exact head was acknowledged as projected") + + +def _github_model_review( + number: int, *, login: str, family: str, state: str = "COMMENTED", + sha: str = "1" * 40, source: str = "self_reported", +) -> dict[str, object]: + providers = {"Gemini": "Google DeepMind", "gpt-6.2-sol": "OpenAI", "unknown": "unknown"} + model = f"Reviewer: {family} | {providers[family]} ({source})" + verdict = "REQUEST_CHANGES" if state == "CHANGES_REQUESTED" else "APPROVE" + explanation = ( + "The reviewed change preserves its existing boundaries under the tested caller. " + "The attached observations map the expected behavior and remaining risk. " + ) * 3 + body = ( + f"{model}\n\n" + "Approval conclusion (author-owned PR; GitHub blocks formal self-approval)\n\n" + f"Current reviewed head is {sha}.\n\n" + + "\n\n".join(f"## {section}\n{explanation}" for section in ( + "动机", "改动思路", "具体改动", "对主干的风险", "我的整体评价", + )) + + f"\n\nEnglish verdict: {verdict}" + ) + return { + "id": number, "body": body, "state": state, + "author": {"login": login, "is_bot": False}, "authorAssociation": "MEMBER", + "commit": {"oid": sha}, "submittedAt": f"2026-10-01T00:00:{number:02}Z", + "url": f"https://github.com/example/project/pull/1#pullrequestreview-{number}", + } + + +def _github_model_pr(reviews: list[dict[str, object]], **metadata: object) -> dict[str, object]: + return { + "number": 1, "title": "Keep a verified caller working", + "headRefOid": "1" * 40, "baseRefOid": "a" * 40, + "state": "OPEN", "author": {"login": "shared-review-account"}, + "reviews_complete": True, "reviews": reviews, **metadata, + } + + +def _model_queue_row( + pr: dict[str, object], *, family: str | None = "gpt", + readiness_observations: dict[str, dict] | None = None, +) -> dict[str, object]: + from loopx.pr_review import build_pr_review_packet + + packet = build_pr_review_packet( + pull_requests=[pr], repository="example/project", limit=10, source="fixture", + state_filter="all", reviewer_login="shared-review-account", wait_for_ci=False, + reviewer_model_family=family, + readiness_observations=readiness_observations, + ) + return packet["pull_requests"][0] + + +@pytest.mark.parametrize("author_only", [True, False]) +def test_model_queue_reviews_another_familys_current_head(author_only): + login = "shared-review-account" if author_only else "independent" + pr = _github_model_pr([_github_model_review(10, login=login, family="Gemini")]) + before = _model_queue_row(pr, family=None) + after = _model_queue_row(pr) + assert before["review_action_kind"] == ( + "qualify_pull_request_merge_readiness" if author_only else None + ) + assert after["review_action_kind"] == "rereview_pull_request_exact_head" + assert after["review_plan"] and after["evidence_commands"] + assert after["review_conclusion"] == before["review_conclusion"] + assert after.get("independent_review") == before.get("independent_review") + assert after["wait_for_ci"] is False + assert all("statusCheckRollup" not in command for command in after["evidence_commands"]) + + +@pytest.mark.parametrize("shared_login", [True, False]) +def test_later_other_family_does_not_repeat_a_current_matching_review(shared_login): + matching = "shared-review-account" if shared_login else "independent" + report = _github_model_review(10, login=matching, family="gpt-6.2-sol", source="runtime_reported") + _, separator, rest = report["body"].partition("\n") + report["body"] = ( + "Reviewer: model_agent | model=gpt-6.2-sol | provider=OpenAI " + f"| reasoning_effort=max | source=runtime_reported{separator}{rest}" + ) + pr = _github_model_pr([report, _github_model_review(11, login="second-peer", family="Gemini")]) + before = _model_queue_row(pr, family=None) + after = _model_queue_row(pr) + assert before == after + if shared_login: + assert after["independent_review"]["reviewer"] == "second-peer" + + +@pytest.mark.parametrize("pretend_matching", ["old_head", "dismissed", "unknown", "retired", "authored"]) +def test_non_covering_review_leaves_full_cross_family_review_due(pretend_matching): + matching = _github_model_review(10, login="first-peer", family="gpt-6.2-sol") + if pretend_matching == "old_head": + matching["commit"] = {"oid": "2" * 40} + elif pretend_matching == "dismissed": + matching["state"] = "DISMISSED" + elif pretend_matching == "unknown": + matching = _github_model_review(10, login="first-peer", family="unknown") + elif pretend_matching == "authored": + matching["author"] = {"login": "shared-review-account", "is_bot": False} + reviews = [matching, _github_model_review(11, login="second-peer", family="Gemini")] + if pretend_matching == "retired": + reviews.append(_github_model_review(12, login="first-peer", family="unknown")) + assert _model_queue_row(_github_model_pr(reviews))["review_action_kind"] == ( + "rereview_pull_request_exact_head" + ) + + +def test_cross_family_coverage_preserves_dissent_and_lifecycle_boundaries(): + prior = _github_model_review(10, login="peer", family="Gemini", state="CHANGES_REQUESTED") + pr = _github_model_pr([prior], reviewDecision="CHANGES_REQUESTED") + row = _model_queue_row(pr) + assert row["review_conclusion"] == _model_queue_row(pr, family=None)["review_conclusion"] + assert row["review_conclusion"]["verdict"] == "REQUEST_CHANGES" + assert row["review_action_kind"] == "rereview_pull_request_exact_head" + for changes in ({"isDraft": True}, {"state": "MERGED"}, {"state": "CLOSED"}, {"reviews_complete": False}): + updated = pr | changes + assert _model_queue_row(updated) == _model_queue_row(updated, family=None) + unknown = _github_model_pr([_github_model_review(10, login="peer", family="unknown")]) + assert _model_queue_row(unknown) == _model_queue_row(unknown, family=None) + + +def test_model_queue_reads_complete_history_for_author_only_review_events(): + from loopx.capabilities.pr_review_queue.github_source import _attach_review_provenance + + review = _github_model_review(1, login="shared-review-account", family="Gemini") + pr = _github_model_pr([review]) + pr.pop("reviews_complete") + details = {"reviews": [review]} + calls = [] + + def run(arguments, **kwargs): + calls.append(arguments) + if arguments[0] == "api": + return [[{ + "id": 1, "user": {"login": "shared-review-account", "type": "User"}, + "author_association": "MEMBER", "state": "COMMENTED", + "commit_id": "1" * 40, "html_url": review["url"], + "submitted_at": review["submittedAt"], "body": review["body"], + }]] + return {"headRefOid": pr["headRefOid"], "baseRefOid": pr["baseRefOid"]} + + options = {"repository": "example/project", "number": "1", "cwd": None, "run_gh_json": run} + assert _attach_review_provenance(pr, details, **options) + assert not calls and "reviews_complete" not in pr + assert _attach_review_provenance(pr, details, **options, model_review_coverage=True) + assert pr["reviews_complete"] is True + assert [arguments[0] for arguments in calls] == ["api", "pr"] + + +@pytest.mark.parametrize( + "body, expected", + [ + ("Reviewer: Gemini | Google DeepMind (self_reported)", ("gemini", "self_reported")), + ("Reviewer: model_agent | self_reported | Gemini | Google DeepMind", ("gemini", "self_reported")), + ("Reviewer: model_agent · gpt-6.2-sol · OpenAI (runtime_reported, max)", ("gpt", "runtime_reported")), + ("Reviewer: model_agent; model=GPT-5; provider=OpenAI; declaration_source=self_reported.", ("gpt", "self_reported")), + ("Reviewer: model_agent | model=Gemini | provider=Google DeepMind | source=self_reported", ("gemini", "self_reported")), + ("Reviewer: model_agent | model=Gemini | provider=Google | provider=OpenAI | source=self_reported", None), + ("Reviewer: Gemini | OpenAI (runtime_reported)", None), + ("Reviewer: not-gpt | OpenAI (runtime_reported)", None), + ("Reviewer: GPT-6 | OpenAI (unverified)", None), + ("", None), + ("Reviewer: GPT-5 | OpenAI (self_reported)\nReviewer: GPT-6 | OpenAI (runtime_reported)", None), + ], +) +def test_model_family_requires_an_unambiguous_published_provider_and_grade(body, expected): + from loopx.capabilities.pr_review_queue.model_coverage import published_review_model + + assert published_review_model(body) == expected + + +def test_model_review_is_due_even_when_approval_readiness_was_observed(): + repository = "example/project" + threads = {"complete": True, "total_count": 0, "unresolved_count": 0} + review = _github_model_review(10, login="shared-review-account", family="Gemini") + _, separator, rest = review["body"].partition("\n") + review["body"] = f"Reviewer: model_agent | self_reported | Gemini | Google DeepMind{separator}{rest}" + pr = _github_model_pr([review], review_thread_summary=threads) + before = _model_queue_row(pr, family=None) + material = readiness_material_state( + repository=repository, item=before, review_threads=threads, wait_for_ci=False, + ) + observations = {observation_key(repository, "1@" + "1" * 40): { + "schema_version": MERGE_READINESS_OBSERVATION_SCHEMA_VERSION, + "material_fingerprint": readiness_material_fingerprint(material), + }} + assert _model_queue_row(pr, family=None, readiness_observations=observations)["review_action_kind"] is None + item = _model_queue_row(pr, readiness_observations=observations) + assert item["review_action_kind"] == "rereview_pull_request_exact_head" + assert item["review_conclusion"] == before["review_conclusion"] From 9d85f67d380b6556ad18d21df827f06957bde17b Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:20:01 +0800 Subject: [PATCH 2/9] Recognize dot-separated reviewer declarations in the queue Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../pr_review_queue/model_coverage.py | 6 +++++ tests/capabilities/test_pr_review_queue.py | 22 ++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/model_coverage.py b/loopx/capabilities/pr_review_queue/model_coverage.py index 145d46f010..d8cca35440 100644 --- a/loopx/capabilities/pr_review_queue/model_coverage.py +++ b/loopx/capabilities/pr_review_queue/model_coverage.py @@ -24,6 +24,11 @@ r"\s*\((runtime_reported|self_reported)(?:,\s*[\w -]+)?\)", re.IGNORECASE, ) +_DOTTED_DECLARATION = re.compile( + r"model_agent\s*·\s*([^·]+?)\s*·\s*([^·]+?)\s*·\s*(runtime_reported|self_reported)" + r"(?:\s*·\s*(?:reasoning_effort=[\w -]+|effort_unavailable))?", + re.IGNORECASE, +) _KEYED_DECLARATION = re.compile( r"model_agent;\s*model=([^;]+);\s*provider=([^;]+);\s*" r"declaration_source=(runtime_reported|self_reported)" @@ -65,6 +70,7 @@ def published_review_model(body: object) -> tuple[str, str] | None: return None match = ( _COMPACT_DECLARATION.fullmatch(lines[0]) + or _DOTTED_DECLARATION.fullmatch(lines[0]) or _KEYED_DECLARATION.fullmatch(lines[0]) or _KEYED_PIPE_DECLARATION.fullmatch(lines[0]) ) diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 511f90e7ee..919399ed52 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -768,14 +768,20 @@ def test_model_queue_reviews_another_familys_current_head(author_only): @pytest.mark.parametrize("shared_login", [True, False]) -def test_later_other_family_does_not_repeat_a_current_matching_review(shared_login): +@pytest.mark.parametrize( + "declaration", + [ + "Reviewer: model_agent | model=gpt-6.2-sol | provider=OpenAI " + "| reasoning_effort=max | source=runtime_reported", + "Reviewer: model_agent · gpt-6.2-sol · OpenAI · runtime_reported · reasoning_effort=max", + ], + ids=["keyed-pipe", "middle-dot"], +) +def test_later_other_family_does_not_repeat_a_current_matching_review(shared_login, declaration): matching = "shared-review-account" if shared_login else "independent" report = _github_model_review(10, login=matching, family="gpt-6.2-sol", source="runtime_reported") _, separator, rest = report["body"].partition("\n") - report["body"] = ( - "Reviewer: model_agent | model=gpt-6.2-sol | provider=OpenAI " - f"| reasoning_effort=max | source=runtime_reported{separator}{rest}" - ) + report["body"] = f"{declaration}{separator}{rest}" pr = _github_model_pr([report, _github_model_review(11, login="second-peer", family="Gemini")]) before = _model_queue_row(pr, family=None) after = _model_queue_row(pr) @@ -851,6 +857,12 @@ def run(arguments, **kwargs): ("Reviewer: Gemini | Google DeepMind (self_reported)", ("gemini", "self_reported")), ("Reviewer: model_agent | self_reported | Gemini | Google DeepMind", ("gemini", "self_reported")), ("Reviewer: model_agent · gpt-6.2-sol · OpenAI (runtime_reported, max)", ("gpt", "runtime_reported")), + ( + "Reviewer: model_agent · GPT-6 · OpenAI · runtime_reported · effort_unavailable", + ("gpt", "runtime_reported"), + ), + ("Reviewer: model_agent · Gemini · Google · self_reported", ("gemini", "self_reported")), + ("Reviewer: model_agent · GPT-6 · OpenAI · runtime_reported · self_reported", None), ("Reviewer: model_agent; model=GPT-5; provider=OpenAI; declaration_source=self_reported.", ("gpt", "self_reported")), ("Reviewer: model_agent | model=Gemini | provider=Google DeepMind | source=self_reported", ("gemini", "self_reported")), ("Reviewer: model_agent | model=Gemini | provider=Google | provider=OpenAI | source=self_reported", None), From 672fa8520d391e0cd5314c7c7171d700faebb91c Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:01:39 +0800 Subject: [PATCH 3/9] Recognize bare reasoning effort in review history Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/pr_review_queue/model_coverage.py | 7 ++++++- tests/capabilities/test_pr_review_queue.py | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/model_coverage.py b/loopx/capabilities/pr_review_queue/model_coverage.py index d8cca35440..c8e01a3d7e 100644 --- a/loopx/capabilities/pr_review_queue/model_coverage.py +++ b/loopx/capabilities/pr_review_queue/model_coverage.py @@ -6,6 +6,7 @@ from collections.abc import Mapping, Sequence from typing import Any +from ...reasoning_effort import REASONING_EFFORTS from .approval_closeout import select_review_history from .review_body import reviewer_declaration_lines @@ -24,9 +25,13 @@ r"\s*\((runtime_reported|self_reported)(?:,\s*[\w -]+)?\)", re.IGNORECASE, ) +# Bare effort labels follow the runtime vocabulary and cannot be provenance tags. +_DOTTED_EFFORT_LABELS = "|".join( + re.escape(label) for label in (*REASONING_EFFORTS, "effort_unavailable") +) _DOTTED_DECLARATION = re.compile( r"model_agent\s*·\s*([^·]+?)\s*·\s*([^·]+?)\s*·\s*(runtime_reported|self_reported)" - r"(?:\s*·\s*(?:reasoning_effort=[\w -]+|effort_unavailable))?", + rf"(?:\s*·\s*(?:reasoning_effort=[\w -]+|{_DOTTED_EFFORT_LABELS}))?", re.IGNORECASE, ) _KEYED_DECLARATION = re.compile( diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 919399ed52..5c7b0fd88d 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -774,8 +774,10 @@ def test_model_queue_reviews_another_familys_current_head(author_only): "Reviewer: model_agent | model=gpt-6.2-sol | provider=OpenAI " "| reasoning_effort=max | source=runtime_reported", "Reviewer: model_agent · gpt-6.2-sol · OpenAI · runtime_reported · reasoning_effort=max", + "Reviewer: model_agent · gpt-6.2-sol · OpenAI · runtime_reported · max", + "Reviewer: model_agent · gpt-6.2-sol · OpenAI · runtime_reported · xhigh", ], - ids=["keyed-pipe", "middle-dot"], + ids=["keyed-pipe", "middle-dot", "bare-max", "bare-xhigh"], ) def test_later_other_family_does_not_repeat_a_current_matching_review(shared_login, declaration): matching = "shared-review-account" if shared_login else "independent" From 5da2fc373c8b6b17832acc6eec3a47eb3156a4bd Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Thu, 19 Nov 2026 04:44:37 +0800 Subject: [PATCH 4/9] Share reviewer declaration parsing across publication and the queue Preserve model coverage when an exact-head review uses inline Markdown, reordered fields, or ordinary pipe separators. Validate publication against the same parsed model, provider, source, and reasoning effort while keeping account independence in the review history owner. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/pr_review_queue/README.md | 13 ++- .../pr_review_queue/model_coverage.py | 53 +-------- .../pr_review_queue/result_check.py | 58 +++++----- .../pr_review_queue/review_body.py | 104 ++++++++++++++++++ .../pr_review_queue/review_contract.py | 8 +- tests/capabilities/test_pr_review_queue.py | 48 +++++++- 6 files changed, 202 insertions(+), 82 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/README.md b/loopx/capabilities/pr_review_queue/README.md index d7fb2eda03..b328774c6e 100644 --- a/loopx/capabilities/pr_review_queue/README.md +++ b/loopx/capabilities/pr_review_queue/README.md @@ -639,16 +639,17 @@ evidence and do not pad. Invoking review uses the installed capability policy. ### Reviewer provenance and specification basis -Policy revision 14 makes two publication requirements mandatory for every new -result; neither changes queue selection, merge authority or how already -published GitHub reviews are recognized (`check_review_body` is unchanged). +Policy revision 14 introduced reviewer attribution and specification basis for +new review results. GitHub readback uses the published attribution for model +coverage. Merge authority follows the existing account and review rules. - `result.reviewer` (`reviewer_declaration`) names `actor_kind` (`model_agent` or `human_operator`) and `declaration_source` (`runtime_reported` or `self_reported`); an agent adds `declared_model` and - `declared_provider` in product-family wording. The body repeats them on - exactly one visible `Reviewer:` line, each value as a whole token. This is - provenance, not a credential: it authenticates nothing and adds no weight. + `declared_provider` in product-family wording. Use one visible line such as + `Reviewer: model_agent · MODEL · PROVIDER · self_reported`. Publication and + queue readback share the declaration parser. Model coverage requires an + explicit source. The declaration adds no weight or account authority. - `problem_context.spec_basis` (`spec_basis_assessment`) is `mapped`, `no_spec` or `not_yet_proven`. `mapped` gives text `spec_ref`, `spec_revision` (a full commit id for `accepted_rfc` and diff --git a/loopx/capabilities/pr_review_queue/model_coverage.py b/loopx/capabilities/pr_review_queue/model_coverage.py index c8e01a3d7e..dee96293a3 100644 --- a/loopx/capabilities/pr_review_queue/model_coverage.py +++ b/loopx/capabilities/pr_review_queue/model_coverage.py @@ -6,9 +6,8 @@ from collections.abc import Mapping, Sequence from typing import Any -from ...reasoning_effort import REASONING_EFFORTS from .approval_closeout import select_review_history -from .review_body import reviewer_declaration_lines +from .review_body import parse_model_reviewer_line, reviewer_declaration_lines # A product alias is evidence only when paired with its declared provider. @@ -20,36 +19,6 @@ "qwen": {"alibaba", "alibaba cloud", "qwen"}, } _MODEL_NAME = re.compile(r"([a-z][a-z0-9]*)(?:[ .-][a-z0-9]+)*") -_COMPACT_DECLARATION = re.compile( - r"(?:model_agent\s*[·|]\s*)?([^·|]+?)\s*[·|]\s*([^()]+?)" - r"\s*\((runtime_reported|self_reported)(?:,\s*[\w -]+)?\)", - re.IGNORECASE, -) -# Bare effort labels follow the runtime vocabulary and cannot be provenance tags. -_DOTTED_EFFORT_LABELS = "|".join( - re.escape(label) for label in (*REASONING_EFFORTS, "effort_unavailable") -) -_DOTTED_DECLARATION = re.compile( - r"model_agent\s*·\s*([^·]+?)\s*·\s*([^·]+?)\s*·\s*(runtime_reported|self_reported)" - rf"(?:\s*·\s*(?:reasoning_effort=[\w -]+|{_DOTTED_EFFORT_LABELS}))?", - re.IGNORECASE, -) -_KEYED_DECLARATION = re.compile( - r"model_agent;\s*model=([^;]+);\s*provider=([^;]+);\s*" - r"declaration_source=(runtime_reported|self_reported)" - r"(?:;\s*reasoning_effort=[\w -]+)?\.?", - re.IGNORECASE, -) -_KEYED_PIPE_DECLARATION = re.compile( - r"model_agent\s*\|\s*model=([^|=]+)\s*\|\s*provider=([^|=]+)" - r"(?:\s*\|\s*reasoning_effort=[\w -]+)?" - r"\s*\|\s*source=(runtime_reported|self_reported)", - re.IGNORECASE, -) -_LEGACY_DECLARATION = re.compile( - r"model_agent\s*\|\s*(runtime_reported|self_reported)\s*\|\s*([^|]+)\s*\|\s*([^|]+)", - re.IGNORECASE, -) def _model_family(model: object, provider: object) -> str | None: @@ -73,21 +42,11 @@ def published_review_model(body: object) -> tuple[str, str] | None: lines = reviewer_declaration_lines(body) if len(lines) != 1: return None - match = ( - _COMPACT_DECLARATION.fullmatch(lines[0]) - or _DOTTED_DECLARATION.fullmatch(lines[0]) - or _KEYED_DECLARATION.fullmatch(lines[0]) - or _KEYED_PIPE_DECLARATION.fullmatch(lines[0]) - ) - if match: - model, provider, source = match.groups() - else: - legacy = _LEGACY_DECLARATION.fullmatch(lines[0]) - if not legacy: - return None - source, model, provider = legacy.groups() - family = _model_family(model, provider) - return (family, source.casefold()) if family else None + declaration = parse_model_reviewer_line(lines[0]) + if declaration is None or declaration.source is None: + return None + family = _model_family(declaration.model, declaration.provider) + return (family, declaration.source) if family else None def needs_cross_model_review( diff --git a/loopx/capabilities/pr_review_queue/result_check.py b/loopx/capabilities/pr_review_queue/result_check.py index 8ce8ed54de..f52abc6365 100644 --- a/loopx/capabilities/pr_review_queue/result_check.py +++ b/loopx/capabilities/pr_review_queue/result_check.py @@ -21,6 +21,7 @@ author_owned_review_title_matches, check_review_body, normalized_review_prose, + parse_model_reviewer_line, review_section_text, reviewer_declaration_lines, visible_review_text, @@ -251,7 +252,9 @@ def _published_as_token(reference: str, text: str) -> bool: return re.search(pattern, text) is not None -def _reviewer_errors(value: object, body: str) -> list[str]: +def _reviewer_errors( + value: object, body: str, *, execution_present: bool = False, runtime_reported: bool = False, +) -> list[str]: key = "reviewer" contract = REVIEWER_DECLARATION if not isinstance(value, Mapping): @@ -260,9 +263,9 @@ def _reviewer_errors(value: object, body: str) -> list[str]: if actor_kind not in contract["actor_kinds"]: return [f"{key}:invalid_actor_kind"] errors: list[str] = [] - if value.get("declaration_source") not in contract["declaration_sources"]: + source = value.get("declaration_source") + if source not in contract["declaration_sources"]: errors.append(f"{key}:invalid_declaration_source") - declared: list[str] = [str(actor_kind)] if actor_kind == "model_agent": for field in contract["model_agent_fields"]: text = value.get(field) @@ -270,16 +273,31 @@ def _reviewer_errors(value: object, body: str) -> list[str]: errors.append(f"{key}:missing_field:{field}") elif _DECLARED_NAME_FORBIDDEN.search(text): errors.append(f"{key}:not_a_product_family_name:{field}") - else: - declared.append(text.strip()) lines = reviewer_declaration_lines(body) if len(lines) != 1: - errors.append(f"{key}:body_line_missing_or_ambiguous") - elif _PUBLISHED_LINE_FORBIDDEN.search(lines[0]): - errors.append(f"{key}:body_line_carries_runtime_detail") - elif any(not _published_as_token(token.casefold(), lines[0].casefold()) - for token in declared): + return [*errors, f"{key}:body_line_missing_or_ambiguous"] + if _PUBLISHED_LINE_FORBIDDEN.search(lines[0]): + return [*errors, f"{key}:body_line_carries_runtime_detail"] + if actor_kind == "human_operator": + if not _published_as_token("human_operator", lines[0].casefold()): + errors.append(f"{key}:body_line_disagrees_with_declaration") + return errors + + published = parse_model_reviewer_line(lines[0]) + model, provider = value.get("declared_model"), value.get("declared_provider") + if (published is None or not published.has_model_actor + or isinstance(model, str) and published.model.casefold() != model.strip().casefold() + or isinstance(provider, str) and published.provider.casefold() != provider.strip().casefold() + or published.source is not None and published.source != source + ): errors.append(f"{key}:body_line_disagrees_with_declaration") + if execution_present or runtime_reported or source == "runtime_reported": + expected_effort = value.get("declared_reasoning_effort") or "effort_unavailable" + if (published is None or published.source != source or + runtime_reported and isinstance(expected_effort, str) + and published.reasoning_effort != expected_effort.casefold() + ): + errors.append(f"{key}:runtime_basis_not_published") return errors @@ -612,11 +630,15 @@ def check_review_result( behavior_bearing=applicability.get("behavior_bearing_change") is True) errors.extend(f"review_body:{reason}" for reason in body["invalid_reasons"]) body_text = str(result.get("review_body") or "") - errors.extend(_reviewer_errors(result.get("reviewer"), body_text)) from ...control_plane.effect_runtime import effect_runtime_result execution = packet.get("reviewer_execution") declaration = result.get("reviewer") + observed = current_execution or execution + runtime_reported = isinstance(observed, Mapping) and observed.get("status") == "runtime_reported" + errors.extend(_reviewer_errors( + declaration, body_text, execution_present=execution is not None, runtime_reported=runtime_reported, + )) if execution is not None or current_execution is not None or ( isinstance(declaration, Mapping) and declaration.get("declaration_source") == "runtime_reported" ): @@ -625,20 +647,6 @@ def check_review_result( params["current"] = current_execution matched = effect_runtime_result("runtime.execution_identity.match", params) errors.extend(f"reviewer:{reason}" for reason in matched["errors"]) - if isinstance(declaration, Mapping) and declaration.get("actor_kind") == "model_agent" and ( - execution is not None or declaration.get("declaration_source") == "runtime_reported" - or current_execution is not None and current_execution.get("status") == "runtime_reported" - ): - lines = reviewer_declaration_lines(body_text) - public_fields = [declaration.get("declaration_source")] - observed = current_execution or execution - if isinstance(observed, Mapping) and observed.get("status") == "runtime_reported": - public_fields.append(declaration.get("declared_reasoning_effort") or "effort_unavailable") - if len(lines) == 1 and any( - isinstance(value, str) and not _published_as_token(value.casefold(), lines[0].casefold()) - for value in public_fields - ): - errors.append("reviewer:runtime_basis_not_published") problem_context = evidence.get("problem_context") impact = problem_context.get("outcome_impact") if isinstance(problem_context, Mapping) else None experience = impact.get("user_experience") if isinstance(impact, Mapping) else None diff --git a/loopx/capabilities/pr_review_queue/review_body.py b/loopx/capabilities/pr_review_queue/review_body.py index 48f4b395ed..9a6781c4a0 100644 --- a/loopx/capabilities/pr_review_queue/review_body.py +++ b/loopx/capabilities/pr_review_queue/review_body.py @@ -4,8 +4,11 @@ import re from collections.abc import Iterator +from dataclasses import dataclass from typing import Any +from ...reasoning_effort import REASONING_EFFORTS + # Presentation aliases share the existing canonical section ids. They do not # broaden evidence matching to arbitrary headings that contain a Chinese word. REVIEW_SECTION_TITLES = { @@ -24,6 +27,16 @@ AUTHOR_OWNED_REQUEST_CHANGES_FALLBACK_TITLE = ( "Request changes conclusion (author-owned PR; GitHub blocks formal self-review)" ) +REVIEWER_DECLARATION_SOURCES = ("runtime_reported", "self_reported") + + +@dataclass(frozen=True) +class PublishedModelDeclaration: + model: str + provider: str + source: str | None + reasoning_effort: str | None + has_model_actor: bool def review_body_requirements(*, behavior_bearing: bool) -> dict[str, int]: @@ -106,6 +119,97 @@ def reviewer_declaration_lines(body: str) -> list[str]: return lines +def _reviewer_field(text: str) -> str: + text = text.strip() + if text.startswith("`") and text.endswith("`"): + left = text.lstrip("`") + right = text.rstrip("`") + if len(left) == len(right): + return left.rstrip("`").strip() + elif len(text) > 2 and text[0] in "*_" and text[0] == text[-1]: + return text[1:-1].strip() + return text + + +def _keyed_reviewer_fields(fields: list[str]) -> dict[str, str] | None: + values: dict[str, str] = {} + for field in fields: + name, sep, text = field.partition("=") + key = _reviewer_field(name).casefold() + if key == "declaration_source": + key = "source" + if not sep or key not in ("model", "provider", "source", "reasoning_effort") or key in values: + return None + values[key] = _reviewer_field(text) + return values if all(values.get(key) for key in ("model", "provider", "source")) else None + + +def parse_model_reviewer_line(line: str) -> PublishedModelDeclaration | None: + """Parse the fields after the visible Reviewer label, including historical forms.""" + # The historical compact form puts the source and optional effort in parentheses. + pair, _, attribution = line.rpartition("(") + basis = attribution[:-1].split(",") if pair and attribution.endswith(")") else [] + source_label = _reviewer_field(basis[0]).casefold() if basis else "" + compact = source_label in REVIEWER_DECLARATION_SOURCES + # Old self-reports could include this note without the typed source marker. + legacy_self_report = len(basis) == 1 and source_label == "self-reported family" + identity = pair.rstrip() if compact or legacy_self_report else line + if ";" in identity: + identity = identity.removesuffix(".") + fields = [_reviewer_field(field) for field in re.split(r"[·|;]", identity)] + has_model_actor = fields[0].casefold() == "model_agent" + if has_model_actor: + fields = fields[1:] + elif not compact: + return None + + source: str | None = None + effort: str | None = None + explicit_effort = False + if compact: + if len(fields) != 2 or len(basis) > 2: + return None + model, provider = fields + source = basis[0] + if len(basis) == 2: + effort = basis[1] + explicit_effort = True + elif any("=" in field for field in fields[:2]): + keyed = _keyed_reviewer_fields(fields) + if keyed is None: + return None + model, provider, source = keyed["model"], keyed["provider"], keyed["source"] + effort = keyed.get("reasoning_effort") + explicit_effort = effort is not None + else: + if not 2 <= len(fields) <= 4: + return None + if len(fields) == 3 and fields[0].casefold() in REVIEWER_DECLARATION_SOURCES: + fields = fields[1:] + fields[:1] + model, provider, *basis = fields + source = basis[0] if basis else None + if len(basis) == 2: + name, sep, text = basis[1].partition("=") + explicit_effort = bool(sep) and _reviewer_field(name).casefold() == "reasoning_effort" + effort = text if explicit_effort else basis[1] + + if any(not re.fullmatch(r"[^·|;=`]+", field) or field.casefold() in REVIEWER_DECLARATION_SOURCES + for field in (model, provider)): + return None + if source is not None: + source = _reviewer_field(source).casefold() + if source not in REVIEWER_DECLARATION_SOURCES: + return None + if effort is not None: + effort = _reviewer_field(effort).casefold() + if explicit_effort: + if not re.fullmatch(r"[\w -]+", effort) or effort in REVIEWER_DECLARATION_SOURCES: + return None + elif effort not in (*REASONING_EFFORTS, "effort_unavailable"): + return None + return PublishedModelDeclaration(model, provider, source, effort, has_model_actor) + + def normalized_review_prose(prose: str) -> str: """Compare visible wording while tolerating ordinary Markdown formatting.""" prose = re.sub(r"!?\[([^\]]*)\]\([^)]*\)", r"\1", prose) diff --git a/loopx/capabilities/pr_review_queue/review_contract.py b/loopx/capabilities/pr_review_queue/review_contract.py index 5cdd820a52..122a982cc8 100644 --- a/loopx/capabilities/pr_review_queue/review_contract.py +++ b/loopx/capabilities/pr_review_queue/review_contract.py @@ -8,6 +8,7 @@ AUTHOR_OWNED_APPROVAL_FALLBACK_TITLE, AUTHOR_OWNED_REQUEST_CHANGES_FALLBACK_TITLE, REQUIRED_FINAL_SECTIONS, + REVIEWER_DECLARATION_SOURCES, REVIEW_SECTION_TITLES, review_body_requirements, ) @@ -16,7 +17,7 @@ from .visual_assessment import VISUAL_ASSESSMENT # Increment when review requirements change without changing the packet shape. -REVIEW_POLICY_REVISION = 26 +REVIEW_POLICY_REVISION = 27 # Agent-consumed prose is observable behavior even when JSON shape is stable. # Reuse the existing semantic verdicts; this is evidence within the same review. @@ -271,7 +272,7 @@ # so a gap here makes the result unpublishable rather than blocking the author. REVIEWER_DECLARATION = { "actor_kinds": ["model_agent", "human_operator"], - "declaration_sources": ["runtime_reported", "self_reported"], + "declaration_sources": list(REVIEWER_DECLARATION_SOURCES), "fields": ["actor_kind", "declaration_source"], "model_agent_fields": ["declared_model", "declared_provider"], "body_marker": "Reviewer:", @@ -279,7 +280,8 @@ "Every result names who wrote it. actor_kind is model_agent or human_operator. " "A model_agent gives declared_model and declared_provider in ordinary " "product-family wording, and the published body repeats actor_kind, model and " - "provider on exactly one visible `Reviewer:` line, so a reader on another host " + "provider as separate fields on exactly one visible `Reviewer:` line " + "(for example, `Reviewer: model_agent · MODEL · PROVIDER · self_reported`), so a reader on another host " "or organization can tell a model wrote the review and weigh it without private " "context. declaration_source records whether the host runtime reported the " "identity or the reviewer stated it; a reviewer that cannot observe its exact " diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 5c7b0fd88d..e515e01dac 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -776,14 +776,32 @@ def test_model_queue_reviews_another_familys_current_head(author_only): "Reviewer: model_agent · gpt-6.2-sol · OpenAI · runtime_reported · reasoning_effort=max", "Reviewer: model_agent · gpt-6.2-sol · OpenAI · runtime_reported · max", "Reviewer: model_agent · gpt-6.2-sol · OpenAI · runtime_reported · xhigh", + "Reviewer: model_agent · `gpt-6.2-sol` · OpenAI · runtime_reported · max", + "Reviewer: model_agent | source=runtime_reported | provider=OpenAI " + "| model=gpt-6.2-sol | reasoning_effort=max", + "Reviewer: model_agent; reasoning_effort=max; provider=OpenAI; " + "model=`gpt-6.2-sol`; declaration_source=runtime_reported.", + "Reviewer: model_agent | gpt-6.2-sol | OpenAI | runtime_reported | max", + ], + ids=[ + "keyed-pipe", "middle-dot", "bare-max", "bare-xhigh", "inline-code", + "reordered-pipe", "reordered-semicolon", "positional-pipe", ], - ids=["keyed-pipe", "middle-dot", "bare-max", "bare-xhigh"], ) def test_later_other_family_does_not_repeat_a_current_matching_review(shared_login, declaration): + from loopx.capabilities.pr_review_queue.result_check import _reviewer_errors + matching = "shared-review-account" if shared_login else "independent" report = _github_model_review(10, login=matching, family="gpt-6.2-sol", source="runtime_reported") _, separator, rest = report["body"].partition("\n") report["body"] = f"{declaration}{separator}{rest}" + assert not _reviewer_errors({ + "actor_kind": "model_agent", + "declaration_source": "runtime_reported", + "declared_model": "gpt-6.2-sol", + "declared_provider": "OpenAI", + "declared_reasoning_effort": "xhigh" if declaration.endswith("· xhigh") else "max", + }, report["body"], execution_present=True, runtime_reported=True) pr = _github_model_pr([report, _github_model_review(11, login="second-peer", family="Gemini")]) before = _model_queue_row(pr, family=None) after = _model_queue_row(pr) @@ -792,6 +810,29 @@ def test_later_other_family_does_not_repeat_a_current_matching_review(shared_log assert after["independent_review"]["reviewer"] == "second-peer" +@pytest.mark.parametrize("line,reason", [ + ( + "model_agent | model=gpt-6.2-sol | provider=OpenAI | source=runtime_reported " + "| declaration_source=self_reported | reasoning_effort=max", + "body_line_disagrees_with_declaration", + ), + ("model_agent · gpt-6.2-sol · OpenAI · self_reported · max", "runtime_basis_not_published"), + ("model_agent · gpt-6.2-sol · OpenAI · runtime_reported · xhigh", "runtime_basis_not_published"), + ("model_agent · gpt-6.2-sol · OpenAI (runtime_reported)", "runtime_basis_not_published"), +]) +def test_runtime_publication_compares_provenance_and_effort_fields(line, reason): + from loopx.capabilities.pr_review_queue.result_check import _reviewer_errors + + errors = _reviewer_errors({ + "actor_kind": "model_agent", + "declaration_source": "runtime_reported", + "declared_model": "gpt-6.2-sol", + "declared_provider": "OpenAI", + "declared_reasoning_effort": "max", + }, f"Reviewer: {line}", execution_present=True, runtime_reported=True) + assert f"reviewer:{reason}" in errors + + @pytest.mark.parametrize("pretend_matching", ["old_head", "dismissed", "unknown", "retired", "authored"]) def test_non_covering_review_leaves_full_cross_family_review_due(pretend_matching): matching = _github_model_review(10, login="first-peer", family="gpt-6.2-sol") @@ -867,7 +908,12 @@ def run(arguments, **kwargs): ("Reviewer: model_agent · GPT-6 · OpenAI · runtime_reported · self_reported", None), ("Reviewer: model_agent; model=GPT-5; provider=OpenAI; declaration_source=self_reported.", ("gpt", "self_reported")), ("Reviewer: model_agent | model=Gemini | provider=Google DeepMind | source=self_reported", ("gemini", "self_reported")), + ("Reviewer: model_agent | Gemini | Google | self_reported", ("gemini", "self_reported")), ("Reviewer: model_agent | model=Gemini | provider=Google | provider=OpenAI | source=self_reported", None), + ("Reviewer: model_agent | source=self_reported | model=Gemini | provider=Google | declaration_source=runtime_reported", None), + ("Reviewer: model_agent | Gemini | Google | self_reported | runtime_reported", None), + ("Reviewer: model_agent · Gemini · Google", None), + ("Reviewer: model_agent · Gemini · Google (self-reported family)", None), ("Reviewer: Gemini | OpenAI (runtime_reported)", None), ("Reviewer: not-gpt | OpenAI (runtime_reported)", None), ("Reviewer: GPT-6 | OpenAI (unverified)", None), From 8ba5248c02a3e447319eea11694c6e682c1a15af Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:51:10 +0800 Subject: [PATCH 5/9] Keep model coverage for shared independent review accounts Retain valid same-head reviews while a shared account maintains its formal opinion. Reuse the typed history evidence in the queue and preserve the formal review conclusion. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/pr_review_queue/README.md | 9 ++- .../pr_review_queue/model_coverage.py | 43 +++--------- .../pr_review_approval_closeout.ts | 67 ++++++++++++++----- loopx/pr_review.py | 7 +- tests/capabilities/test_pr_review_queue.py | 67 +++++++++++++++++++ 5 files changed, 138 insertions(+), 55 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/README.md b/loopx/capabilities/pr_review_queue/README.md index b328774c6e..e1ee3fe509 100644 --- a/loopx/capabilities/pr_review_queue/README.md +++ b/loopx/capabilities/pr_review_queue/README.md @@ -666,9 +666,12 @@ In a Goal-bound GitHub queue, a recognized model family from the invoking runtime can reopen an open, non-draft exact head with a declared review from another family. The queue uses complete provider history. An independent review from the same family suppresses repeat work even when another account -submitted a later review. A qualifying review published under the shared -author login with a matching runtime declaration also suppresses repeat work -for that model. GitHub independence still requires independent account +submitted a later review. If different models share an independent GitHub +account, each valid, attributable formal review on the current head supplies coverage +while that account maintains the same formal opinion. Follow-up review comments +preserve that history. A dismissal, a different head or a changed formal +opinion ends it. A qualifying review published under the shared author login +with a matching runtime declaration also suppresses repeat work for that model. GitHub independence still requires independent account evidence. The declarations record their source as published provenance. Missing model identity or incomplete history retains the existing queue diff --git a/loopx/capabilities/pr_review_queue/model_coverage.py b/loopx/capabilities/pr_review_queue/model_coverage.py index dee96293a3..d0a356f71f 100644 --- a/loopx/capabilities/pr_review_queue/model_coverage.py +++ b/loopx/capabilities/pr_review_queue/model_coverage.py @@ -6,7 +6,6 @@ from collections.abc import Mapping, Sequence from typing import Any -from .approval_closeout import select_review_history from .review_body import parse_model_reviewer_line, reviewer_declaration_lines @@ -50,7 +49,7 @@ def published_review_model(body: object) -> tuple[str, str] | None: def needs_cross_model_review( - *, family: str, history: dict[str, Any], selection: Mapping[str, Any], + *, family: str, history: dict[str, Any], independent_review_ids: Sequence[int], model_reports: Sequence[tuple[str, str] | None], ) -> bool: """Detect missing model coverage from one complete exact-head snapshot. @@ -60,13 +59,9 @@ def needs_cross_model_review( """ author = history["author"].casefold() head = history["head"].casefold() - by_login: dict[str, list[dict[str, Any]]] = {} - matching_ids: set[int] = set() - other_ids: set[int] = set() - other_author_review = False + independent_ids = set(independent_review_ids) + other_family_review = False for row, declared in zip(history["reviews"], model_reports, strict=True): - login = row["login"].casefold() - by_login.setdefault(login, []).append(row) if (row["head"].casefold() != head or row["state"] not in {"APPROVED", "CHANGES_REQUESTED", "COMMENTED"}): continue @@ -74,36 +69,18 @@ def needs_cross_model_review( if type(review_id) is not int or review_id <= 0 or declared is None: continue reviewed_family, source = declared - if login == author: + if row["login"].casefold() == author: if row["state"] == "COMMENTED" and row["conclusion"]["valid"]: if reviewed_family != family: - other_author_review = True + other_family_review = True elif source == "runtime_reported": # A shared author login cannot establish GitHub independence. # A complete review already published by this model family # still prevents repeat assignment to the same model. return False continue - (matching_ids if reviewed_family == family else other_ids).add(review_id) - - selected = selection.get("independent_review", {}).get("review_id") - if selected in matching_ids: - return False - other_independent_review = selected in other_ids - if not (other_author_review or other_independent_review or other_ids): - return False - - # Preserve each candidate account's full history, including later comments - # and dismissals. Unrelated accounts cannot overwrite the matching evidence. - candidate_ids = matching_ids | other_ids - for login, reviews in by_login.items(): - if login == author or not login or not any( - row["review_id"] in candidate_ids for row in reviews - ): - continue - evidence = select_review_history({**history, "reviews": reviews}) - review_id = evidence.get("independent_review", {}).get("review_id") - if review_id in matching_ids: - return False - other_independent_review |= review_id in other_ids - return other_author_review or other_independent_review + if review_id in independent_ids: + if reviewed_family == family: + return False + other_family_review = True + return other_family_review diff --git a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts index a168684624..d3e16f07d9 100644 --- a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts +++ b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts @@ -9,6 +9,7 @@ import {parseIsoTimestamp} from "../runtime_timestamp.ts"; type ReviewState = "PENDING" | "COMMENTED" | "APPROVED" | "CHANGES_REQUESTED" | "DISMISSED"; type Opinion = {id: number; login: string; state: ReviewState; head: string; time: number}; type Review = Opinion & {submittedAt: string | null; url: string}; +type ProviderOpinion = Opinion & {source: JsonObject; conclusion: JsonObject}; const states = new Set(["PENDING", "COMMENTED", "APPROVED", "CHANGES_REQUESTED", "DISMISSED"]); // GitHub associations, not a reviewer-name allowlist or evidence of approval. const repositoryReviewers = new Set(["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]); @@ -35,35 +36,65 @@ function effectiveOpinions(reviews: readonly T[], author: str /** Submitted review activity covers duplicate work, independently of whether * its body earns a qualified approval. Comments here are formal review events; * issue comments do not enter this history. Dismissal retires only that actor. */ -function independentReview(reviews: readonly (Opinion & {source: JsonObject})[], - complete: boolean, author: string | null, head: string): JsonObject | undefined { - if (!complete || !author) return; - const latest = new Map(); +function independentReviewEvidence(reviews: readonly ProviderOpinion[], + complete: boolean, author: string | null, head: string, includeCoverage: boolean): JsonObject { + if (!complete || !author) return {}; + const latest = new Map(); + const submitted: ProviderOpinion[] | null = includeCoverage ? [] : null; const seen = new Set(); for (const row of reviews) { const source = row.source; const id = source.review_id; if (typeof id !== "number" || !Number.isSafeInteger(id) || id <= 0 || seen.has(id) - || !row.login || row.login !== row.login.trim()) return; + || !row.login || row.login !== row.login.trim()) return {}; seen.add(id); if (row.state === "PENDING") continue; - const submitted = typeof source.submitted_at === "string" ? parseIsoTimestamp(source.submitted_at) : null; - if (!submitted || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(row.head)) return; - const candidate = {...row, id, time: submitted.getTime()}; + const submittedAt = typeof source.submitted_at === "string" ? parseIsoTimestamp(source.submitted_at) : null; + if (!submittedAt || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(row.head)) return {}; + const candidate = {...row, id, time: submittedAt.getTime()}; + submitted?.push(candidate); const previous = latest.get(row.login); if (!previous || candidate.time > previous.time || candidate.time === previous.time && id > previous.id) { latest.set(row.login, candidate); } } - const selected = [...latest.values()].filter(row => row.login !== author && row.head === head + const qualifies = (row: ProviderOpinion) => row.login !== author && row.head === head && row.state !== "DISMISSED" && row.source.is_bot === false && typeof row.source.association === "string" && repositoryReviewers.has(row.source.association) - && typeof row.source.url === "string" - && row.source.url.trim()).sort((a, b) => b.time - a.time || b.id - a.id)[0]; - if (selected) return {reviewer: selected.login, association: selected.source.association, + && typeof row.source.url === "string" && Boolean(row.source.url.trim()); + const active = [...latest.values()].filter(qualifies).sort((a, b) => b.time - a.time || b.id - a.id); + const selected = active[0]; + const evidence: JsonObject = {}; + if (selected) evidence.independent_review = { + reviewer: selected.login, association: selected.source.association, state: selected.state, head_oid: head, review_id: selected.id, url: selected.source.url, submitted_at: selected.source.submitted_at, source_complete: true, - grants_approval: false, grants_merge_authority: false}; + grants_approval: false, grants_merge_authority: false, + }; + if (submitted) { + const coverage = new Set(active.map(row => row.id)); + const logins = new Set(active.map(row => row.login)); + const formalStates = new Map(); + // Keep attributable formal reviews while the same account and head still + // carry that opinion. A follow-up comment leaves the formal history intact. + for (const row of submitted.sort((a, b) => b.time - a.time || b.id - a.id)) { + if (!logins.has(row.login)) continue; + if (row.head !== head || row.state === "DISMISSED") { + logins.delete(row.login); + continue; + } + if (row.state === "COMMENTED") continue; + const formalState = formalStates.get(row.login); + if (formalState && row.state !== formalState) { + logins.delete(row.login); + continue; + } + formalStates.set(row.login, row.state); + if (qualifies(row) && row.conclusion.valid === true) coverage.add(row.id); + } + evidence.independent_coverage_review_ids = [...coverage]; + } + return evidence; } /** Bodies are qualified by the adapter; history precedence has one typed owner. @@ -81,8 +112,9 @@ export function selectPrReviewConclusion(value: unknown): JsonObject { if (typeof row.head !== "string" || typeof row.login !== "string") return fail("review identity must be strings"); // The adapter supplies newest-first rows; equal-time ties retain its order. const id = history.length - index; - conclusions.set(id, requireJsonObject(row.conclusion, "qualified review conclusion")); - return {id, login: row.login.toLowerCase(), state, head: row.head.toLowerCase(), time: row.time, source: row}; + const conclusion = requireJsonObject(row.conclusion, "qualified review conclusion"); + conclusions.set(id, conclusion); + return {id, login: row.login.toLowerCase(), state, head: row.head.toLowerCase(), time: row.time, source: row, conclusion}; }); const author = typeof request.author === "string" ? request.author.toLowerCase() : null; const head = requireNonEmptyString(request.head, "current head").toLowerCase(); @@ -94,8 +126,9 @@ export function selectPrReviewConclusion(value: unknown): JsonObject { || conclusions.get(row.id)!.valid === true && conclusions.get(row.id)!.verdict === "REQUEST_CHANGES")) ?? effective.find(row => row.head === head && conclusions.get(row.id)!.valid === true) ?? effective[0] ?? reviews[0]; - const independent = independentReview(reviews, request.reviews_complete === true, author, head); - return {index: history.length - selected.id, ...(independent ? {independent_review: independent} : {})}; + return {index: history.length - selected.id, + ...independentReviewEvidence(reviews, request.reviews_complete === true, author, head, + request.include_independent_coverage === true)}; } export function planPrReviewApprovalCloseout(value: unknown): JsonObject { diff --git a/loopx/pr_review.py b/loopx/pr_review.py index 919bb1d7f4..21e35afa77 100644 --- a/loopx/pr_review.py +++ b/loopx/pr_review.py @@ -874,17 +874,20 @@ def evaluate(review: Mapping[str, Any]) -> dict[str, Any]: "submitted_at": review.get("submittedAt"), "conclusion": conclusion} for review, conclusion in zip(reviews, conclusions)]} + if reviewer_model_family and history["reviews_complete"]: + history["include_independent_coverage"] = True selection = select_review_history(history) index = cast(int, selection["index"]) independent = selection.get("independent_review") conclusion = conclusions[index] - if reviewer_model_family and history["reviews_complete"] and (conclusion["valid"] or independent): + coverage_ids = selection.get("independent_coverage_review_ids") + if reviewer_model_family and isinstance(coverage_ids, list) and (conclusion["valid"] or independent): from .capabilities.pr_review_queue.model_coverage import needs_cross_model_review, published_review_model if needs_cross_model_review( family=reviewer_model_family, history=history, - selection=selection, + independent_review_ids=coverage_ids, model_reports=[published_review_model(review.get("body")) for review in reviews], ): conclusion["cross_model_review_needed"] = True diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index e515e01dac..3b9910ae11 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -810,6 +810,73 @@ def test_later_other_family_does_not_repeat_a_current_matching_review(shared_log assert after["independent_review"]["reviewer"] == "second-peer" +@pytest.mark.parametrize( + "first_family,second_family", + [("Gemini", "gpt-6.2-sol"), ("gpt-6.2-sol", "Gemini")], +) +def test_shared_independent_login_keeps_both_model_families( + first_family: str, second_family: str, +) -> None: + login = "shared-peer" + first = _github_model_review(10, login=login, family=first_family, state="APPROVED") + second = _github_model_review(11, login=login, family=second_family, state="APPROVED") + repeat = _github_model_review(12, login=login, family=first_family, state="APPROVED") + comment = _github_model_review(13, login=login, family=first_family) + comment["body"] = str(comment["body"]).splitlines()[0] + "\n\nOne follow-up question." + + missing = "gemini" if second_family == "Gemini" else "gpt" + one_family = _github_model_pr([first], reviewDecision="APPROVED") + assert _model_queue_row(one_family, family=missing)["review_action_kind"] == ( + "rereview_pull_request_exact_head" + ) + for reviews in ([first, second], [first, second, repeat], [first, second, repeat, comment]): + pr = _github_model_pr(reviews, reviewDecision="APPROVED") + ordinary = _model_queue_row(pr, family=None) + for family in ("gpt", "gemini"): + assert _model_queue_row(pr, family=family) == ordinary + + +@pytest.mark.parametrize("intervening", ["dismissal", "opposite_opinion", "different_head"]) +def test_shared_independent_login_retires_earlier_formal_evidence(intervening: str) -> None: + login = "shared-peer" + prior = _github_model_review(10, login=login, family="gpt-6.2-sol", state="APPROVED") + if intervening == "dismissal": + boundary = _github_model_review(11, login=login, family="Gemini", state="DISMISSED") + elif intervening == "opposite_opinion": + boundary = _github_model_review(11, login=login, family="Gemini", state="CHANGES_REQUESTED") + else: + boundary = _github_model_review( + 11, login=login, family="Gemini", state="APPROVED", sha="2" * 40, + ) + current = _github_model_review(12, login=login, family="Gemini", state="APPROVED") + pr = _github_model_pr([prior, boundary, current], reviewDecision="APPROVED") + row = _model_queue_row(pr) + ordinary = _model_queue_row(pr, family=None) + assert row["review_action_kind"] == "rereview_pull_request_exact_head" + assert row["review_conclusion"] == ordinary["review_conclusion"] + assert row["independent_review"] == ordinary["independent_review"] + + +@pytest.mark.parametrize( + "invalid_evidence", + ["association", "bot", "body"], +) +def test_shared_login_history_needs_valid_evidence_for_each_formal_review( + invalid_evidence: str, +) -> None: + login = "shared-peer" + prior = _github_model_review(10, login=login, family="gpt-6.2-sol", state="APPROVED") + if invalid_evidence == "association": + prior["authorAssociation"] = "NONE" + elif invalid_evidence == "bot": + prior["author"] = {"login": login, "is_bot": True} + else: + prior["body"] = str(prior["body"]).splitlines()[0] + "\n\nLooks good." + current = _github_model_review(11, login=login, family="Gemini", state="APPROVED") + pr = _github_model_pr([prior, current], reviewDecision="APPROVED") + assert _model_queue_row(pr)["review_action_kind"] == "rereview_pull_request_exact_head" + + @pytest.mark.parametrize("line,reason", [ ( "model_agent | model=gpt-6.2-sol | provider=OpenAI | source=runtime_reported " From b5bb9edd2d2b40379802c8e7df6f1b3d546e5f8e Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:20:16 +0800 Subject: [PATCH 6/9] Require qualified conclusions for latest model coverage Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../pr_review_queue/model_coverage.py | 16 ++++++---- .../pr_review_approval_closeout.ts | 12 +++++--- loopx/pr_review.py | 29 ++++++++++++------- tests/capabilities/test_pr_review_queue.py | 19 ++++++++++++ 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/model_coverage.py b/loopx/capabilities/pr_review_queue/model_coverage.py index d0a356f71f..6efe31de38 100644 --- a/loopx/capabilities/pr_review_queue/model_coverage.py +++ b/loopx/capabilities/pr_review_queue/model_coverage.py @@ -49,17 +49,20 @@ def published_review_model(body: object) -> tuple[str, str] | None: def needs_cross_model_review( - *, family: str, history: dict[str, Any], independent_review_ids: Sequence[int], + *, family: str, history: dict[str, Any], + independent_coverage_ids: Sequence[int], independent_activity_ids: Sequence[int], model_reports: Sequence[tuple[str, str] | None], ) -> bool: """Detect missing model coverage from one complete exact-head snapshot. - Interpret provenance for queue work only. The existing history owner - still determines whether an account supplies independent review evidence. + The history owner supplies current account activity and qualified + conclusions. Another family's submission can trigger a second opinion. + Suppressing it requires qualified coverage from the invoking family. """ author = history["author"].casefold() head = history["head"].casefold() - independent_ids = set(independent_review_ids) + coverage_ids = set(independent_coverage_ids) + activity_ids = set(independent_activity_ids) other_family_review = False for row, declared in zip(history["reviews"], model_reports, strict=True): if (row["head"].casefold() != head or row["state"] not in @@ -79,8 +82,9 @@ def needs_cross_model_review( # still prevents repeat assignment to the same model. return False continue - if review_id in independent_ids: - if reviewed_family == family: + if reviewed_family == family: + if review_id in coverage_ids: return False + elif review_id in coverage_ids or review_id in activity_ids: other_family_review = True return other_family_review diff --git a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts index d3e16f07d9..acb21a61a9 100644 --- a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts +++ b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts @@ -33,8 +33,8 @@ function effectiveOpinions(reviews: readonly T[], author: str return [...latest.values()]; } -/** Submitted review activity covers duplicate work, independently of whether - * its body earns a qualified approval. Comments here are formal review events; +/** Submitted review activity establishes account-level evidence. Model coverage + * also requires a qualified conclusion. Comments here are formal review events; * issue comments do not enter this history. Dismissal retires only that actor. */ function independentReviewEvidence(reviews: readonly ProviderOpinion[], complete: boolean, author: string | null, head: string, includeCoverage: boolean): JsonObject { @@ -72,7 +72,10 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], grants_approval: false, grants_merge_authority: false, }; if (submitted) { - const coverage = new Set(active.map(row => row.id)); + // A complete COMMENTED review can cover a model without a formal opinion. + const validCoverageConclusion = (row: ProviderOpinion) => row.conclusion.valid === true + || row.state === "COMMENTED" && row.source.review_body_valid === true; + const coverage = new Set(active.filter(validCoverageConclusion).map(row => row.id)); const logins = new Set(active.map(row => row.login)); const formalStates = new Map(); // Keep attributable formal reviews while the same account and head still @@ -90,8 +93,9 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], continue; } formalStates.set(row.login, row.state); - if (qualifies(row) && row.conclusion.valid === true) coverage.add(row.id); + if (qualifies(row) && validCoverageConclusion(row)) coverage.add(row.id); } + evidence.independent_review_activity_ids = active.map(row => row.id); evidence.independent_coverage_review_ids = [...coverage]; } return evidence; diff --git a/loopx/pr_review.py b/loopx/pr_review.py index 21e35afa77..85dc96dd0a 100644 --- a/loopx/pr_review.py +++ b/loopx/pr_review.py @@ -815,7 +815,7 @@ def _review_conclusion( "invalid_reasons": ["no_review_conclusion_available"], } - def evaluate(review: Mapping[str, Any]) -> dict[str, Any]: + def evaluate(review: Mapping[str, Any]) -> tuple[dict[str, Any], bool]: state = str(review.get("state") or "").strip().upper() body = str(review.get("body") or "") english_verdict = _english_review_verdict(body) @@ -856,12 +856,13 @@ def evaluate(review: Mapping[str, Any]) -> dict[str, Any]: "reviewer": review_author or reviewer_login, "submitted_at": review.get("submittedAt"), "invalid_reasons": reasons, - } + }, body_check["valid"] is True - conclusions = [evaluate(review) for review in reviews] + evaluations = [evaluate(review) for review in reviews] if len(reviews) == 1 and pr.get("reviews_complete") is not True: - return conclusions[0] + return evaluations[0][0] from .capabilities.pr_review_queue.approval_closeout import select_review_history + include_model_coverage = bool(reviewer_model_family and pr.get("reviews_complete") is True) history = {"head": head_oid, "author": pr_author, "reviews_complete": pr.get("reviews_complete") is True, "reviews": [{"state": conclusion["state"], @@ -872,22 +873,30 @@ def evaluate(review: Mapping[str, Any]) -> dict[str, Any]: "association": review.get("authorAssociation"), "is_bot": _as_dict(review.get("author")).get("is_bot"), "submitted_at": review.get("submittedAt"), - "conclusion": conclusion} - for review, conclusion in zip(reviews, conclusions)]} - if reviewer_model_family and history["reviews_complete"]: + "conclusion": conclusion, + **({"review_body_valid": body_valid} if include_model_coverage else {})} + for review, (conclusion, body_valid) in zip(reviews, evaluations, strict=True)]} + if include_model_coverage: history["include_independent_coverage"] = True selection = select_review_history(history) index = cast(int, selection["index"]) independent = selection.get("independent_review") - conclusion = conclusions[index] + conclusion = evaluations[index][0] coverage_ids = selection.get("independent_coverage_review_ids") - if reviewer_model_family and isinstance(coverage_ids, list) and (conclusion["valid"] or independent): + activity_ids = selection.get("independent_review_activity_ids") + if ( + reviewer_model_family + and isinstance(coverage_ids, list) + and isinstance(activity_ids, list) + and (conclusion["valid"] or independent) + ): from .capabilities.pr_review_queue.model_coverage import needs_cross_model_review, published_review_model if needs_cross_model_review( family=reviewer_model_family, history=history, - independent_review_ids=coverage_ids, + independent_coverage_ids=coverage_ids, + independent_activity_ids=activity_ids, model_reports=[published_review_model(review.get("body")) for review in reviews], ): conclusion["cross_model_review_needed"] = True diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 3b9910ae11..2bfa82a2ec 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -877,6 +877,25 @@ def test_shared_login_history_needs_valid_evidence_for_each_formal_review( assert _model_queue_row(pr)["review_action_kind"] == "rereview_pull_request_exact_head" +@pytest.mark.parametrize("state", ["COMMENTED", "APPROVED"]) +def test_latest_review_with_unqualified_body_does_not_cover_its_model(state: str) -> None: + login = "shared-peer" + prior = _github_model_review(10, login=login, family="Gemini", state="APPROVED") + latest = _github_model_review(11, login=login, family="gpt-6.2-sol", state=state) + latest["body"] = str(latest["body"]).splitlines()[0] + "\n\nLooks good." + pr = _github_model_pr([prior, latest], reviewDecision="APPROVED") + + ordinary = _model_queue_row(pr, family=None) + gpt = _model_queue_row(pr) + assert gpt["review_action_kind"] == "rereview_pull_request_exact_head" + assert gpt["review_plan"] and gpt["evidence_commands"] + assert gpt["review_conclusion"] == ordinary["review_conclusion"] + assert gpt["independent_review"] == ordinary["independent_review"] + assert gpt["independent_review"]["review_id"] == 11 + assert gpt["independent_review"]["state"] == state + assert _model_queue_row(pr, family="gemini") == ordinary + + @pytest.mark.parametrize("line,reason", [ ( "model_agent | model=gpt-6.2-sol | provider=OpenAI | source=runtime_reported " From 7105188c949d2a048cc2ca9a4af40b6e75715fa8 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:24:02 +0800 Subject: [PATCH 7/9] Keep qualified review coverage after follow-ups Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/pr_review_queue/README.md | 8 ++- .../pr_review_approval_closeout.ts | 43 +++++++------ tests/capabilities/test_pr_review_queue.py | 61 +++++++++++++++++++ 3 files changed, 92 insertions(+), 20 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/README.md b/loopx/capabilities/pr_review_queue/README.md index a13ea54544..044a843ed7 100644 --- a/loopx/capabilities/pr_review_queue/README.md +++ b/loopx/capabilities/pr_review_queue/README.md @@ -660,9 +660,11 @@ another family. The queue uses complete provider history. An independent review from the same family suppresses repeat work even when another account submitted a later review. If different models share an independent GitHub account, each valid, attributable formal review on the current head supplies -coverage while that account maintains the same formal opinion. Follow-up review -comments preserve that history. A dismissal, a different head or a changed -formal opinion ends it. A qualifying review published under the shared author +coverage while that account maintains the same formal opinion. The most recent +qualified `COMMENTED` review in the same history can supply coverage as well. +An ordinary follow-up preserves both kinds of evidence. A dismissal, a different +head or a changed formal opinion resets the account's coverage history. +A qualifying review published under the shared author login with a matching runtime declaration also suppresses repeat work for that model. GitHub independence still requires independent account evidence. diff --git a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts index acb21a61a9..b64dfe05fe 100644 --- a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts +++ b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts @@ -72,31 +72,40 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], grants_approval: false, grants_merge_authority: false, }; if (submitted) { - // A complete COMMENTED review can cover a model without a formal opinion. - const validCoverageConclusion = (row: ProviderOpinion) => row.conclusion.valid === true - || row.state === "COMMENTED" && row.source.review_body_valid === true; - const coverage = new Set(active.filter(validCoverageConclusion).map(row => row.id)); const logins = new Set(active.map(row => row.login)); - const formalStates = new Map(); - // Keep attributable formal reviews while the same account and head still - // carry that opinion. A follow-up comment leaves the formal history intact. - for (const row of submitted.sort((a, b) => b.time - a.time || b.id - a.id)) { + const coverage = new Map(); + // A follow-up preserves the last complete comment and the current formal + // opinion. Process oldest first so an opinion change can retire comments + // between the old and new formal reviews as well as the old formal reviews. + for (const row of submitted.sort((a, b) => a.time - b.time || a.id - b.id)) { if (!logins.has(row.login)) continue; if (row.head !== head || row.state === "DISMISSED") { - logins.delete(row.login); + coverage.delete(row.login); continue; } - if (row.state === "COMMENTED") continue; - const formalState = formalStates.get(row.login); - if (formalState && row.state !== formalState) { - logins.delete(row.login); - continue; + const formal = row.state !== "COMMENTED"; + let account = coverage.get(row.login); + if (!account || (formal && account.formalState !== undefined && account.formalState !== row.state)) { + account = {formalReviewIds: []}; + coverage.set(row.login, account); + } + if (formal) account.formalState = row.state; + if (!qualifies(row)) continue; + if (formal && row.conclusion.valid === true) { + account.formalReviewIds.push(row.id); + } else if (!formal && (row.conclusion.valid === true || row.source.review_body_valid === true)) { + account.commentedReviewId = row.id; } - formalStates.set(row.login, row.state); - if (qualifies(row) && validCoverageConclusion(row)) coverage.add(row.id); } evidence.independent_review_activity_ids = active.map(row => row.id); - evidence.independent_coverage_review_ids = [...coverage]; + evidence.independent_coverage_review_ids = [...coverage.values()].flatMap( + ({formalReviewIds, commentedReviewId}) => commentedReviewId === undefined + ? formalReviewIds : [...formalReviewIds, commentedReviewId], + ); } return evidence; } diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 2bfa82a2ec..1816c54ae5 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -857,6 +857,67 @@ def test_shared_independent_login_retires_earlier_formal_evidence(intervening: s assert row["independent_review"] == ordinary["independent_review"] +@pytest.mark.parametrize( + "with_formal_opinion", [True, False], + ids=["keep-gpt-coverage", "keep-gemini-review-trigger"], +) +def test_follow_up_preserves_qualified_commented_review(with_formal_opinion: bool) -> None: + login = "shared-peer" + if with_formal_opinion: + reviews = [ + _github_model_review(10, login=login, family="Gemini", state="APPROVED"), + _github_model_review(11, login=login, family="gpt-6.2-sol"), + ] + decision = "APPROVED" + action = "qualify_pull_request_merge_readiness" + else: + reviews = [_github_model_review(11, login=login, family="Gemini")] + decision = "REVIEW_REQUIRED" + action = "rereview_pull_request_exact_head" + + before = _github_model_pr(reviews, reviewDecision=decision) + assert _model_queue_row(before)["review_action_kind"] == action + follow_up = _github_model_review(12, login=login, family="gpt-6.2-sol") + follow_up["body"] = "Thanks for the update. I will check the next revision." + after = _github_model_pr([*reviews, follow_up], reviewDecision=decision) + row = _model_queue_row(after) + ordinary = _model_queue_row(after, family=None) + assert row["review_conclusion"] == ordinary["review_conclusion"] + assert row["independent_review"] == ordinary["independent_review"] + assert row["independent_review"]["review_id"] == 12 + assert row["review_action_kind"] == action + + +@pytest.mark.parametrize( + "intervening", ["dismissal", "changed-formal-opinion", "different-head"], +) +def test_commented_review_coverage_retires_at_account_boundary(intervening: str) -> None: + login = "shared-peer" + approval = _github_model_review(9, login=login, family="Gemini", state="APPROVED") + prior = _github_model_review(10, login=login, family="gpt-6.2-sol") + if intervening == "changed-formal-opinion": + replacement = _github_model_review(11, login=login, family="Gemini", state="CHANGES_REQUESTED") + reviews = [approval, prior, replacement] + decision = "CHANGES_REQUESTED" + else: + boundary = ( + _github_model_review(11, login=login, family="Gemini", state="DISMISSED") + if intervening == "dismissal" + else _github_model_review(11, login=login, family="Gemini", sha="2" * 40) + ) + replacement = _github_model_review(12, login=login, family="Gemini", state="APPROVED") + reviews = [approval, prior, boundary, replacement] + decision = "APPROVED" + follow_up = _github_model_review(13, login=login, family="Gemini") + follow_up["body"] = "I have added a reference to the discussion." + pr = _github_model_pr([*reviews, follow_up], reviewDecision=decision) + row = _model_queue_row(pr) + ordinary = _model_queue_row(pr, family=None) + assert row["review_action_kind"] == "rereview_pull_request_exact_head" + assert row["review_conclusion"] == ordinary["review_conclusion"] + assert row["independent_review"] == ordinary["independent_review"] + + @pytest.mark.parametrize( "invalid_evidence", ["association", "bot", "body"], From 33f46790e283ac2685f8800fdd66034c8dc84588 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:30:33 +0800 Subject: [PATCH 8/9] Preserve review coverage for each model family on shared accounts Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/pr_review_queue/README.md | 10 +++++---- .../pr_review_approval_closeout.ts | 21 ++++++++++++------- loopx/pr_review.py | 20 ++++++++++++------ tests/capabilities/test_pr_review_queue.py | 21 ++++++++++++------- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/README.md b/loopx/capabilities/pr_review_queue/README.md index 044a843ed7..398645ab24 100644 --- a/loopx/capabilities/pr_review_queue/README.md +++ b/loopx/capabilities/pr_review_queue/README.md @@ -660,10 +660,12 @@ another family. The queue uses complete provider history. An independent review from the same family suppresses repeat work even when another account submitted a later review. If different models share an independent GitHub account, each valid, attributable formal review on the current head supplies -coverage while that account maintains the same formal opinion. The most recent -qualified `COMMENTED` review in the same history can supply coverage as well. -An ordinary follow-up preserves both kinds of evidence. A dismissal, a different -head or a changed formal opinion resets the account's coverage history. +coverage while that account maintains the same formal opinion. The latest +qualified `COMMENTED` review from each known family in that history can also +supply coverage. A qualified comment with unknown model provenance resets the +commented coverage for that account. An ordinary follow-up preserves both kinds +of evidence. A dismissal, a different head or a changed formal opinion resets +the account's coverage history. A qualifying review published under the shared author login with a matching runtime declaration also suppresses repeat work for that model. GitHub independence still requires independent account evidence. diff --git a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts index b64dfe05fe..b6fcbf0b5c 100644 --- a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts +++ b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts @@ -76,11 +76,10 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], const coverage = new Map; }>(); - // A follow-up preserves the last complete comment and the current formal - // opinion. Process oldest first so an opinion change can retire comments - // between the old and new formal reviews as well as the old formal reviews. + // A follow-up preserves the latest qualified comment for each model family. + // Process oldest first so an opinion change retires the prior coverage. for (const row of submitted.sort((a, b) => a.time - b.time || a.id - b.id)) { if (!logins.has(row.login)) continue; if (row.head !== head || row.state === "DISMISSED") { @@ -90,7 +89,7 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], const formal = row.state !== "COMMENTED"; let account = coverage.get(row.login); if (!account || (formal && account.formalState !== undefined && account.formalState !== row.state)) { - account = {formalReviewIds: []}; + account = {formalReviewIds: [], commentedReviewIds: new Map()}; coverage.set(row.login, account); } if (formal) account.formalState = row.state; @@ -98,13 +97,19 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], if (formal && row.conclusion.valid === true) { account.formalReviewIds.push(row.id); } else if (!formal && (row.conclusion.valid === true || row.source.review_body_valid === true)) { - account.commentedReviewId = row.id; + const family = row.source.review_model_family; + if (typeof family === "string" && family) { + account.commentedReviewIds.set(family, row.id); + } else { + // A qualified review with unknown model provenance still replaces + // the prior commented evidence, as it did for a single comment. + account.commentedReviewIds.clear(); + } } } evidence.independent_review_activity_ids = active.map(row => row.id); evidence.independent_coverage_review_ids = [...coverage.values()].flatMap( - ({formalReviewIds, commentedReviewId}) => commentedReviewId === undefined - ? formalReviewIds : [...formalReviewIds, commentedReviewId], + ({formalReviewIds, commentedReviewIds}) => [...formalReviewIds, ...commentedReviewIds.values()], ); } return evidence; diff --git a/loopx/pr_review.py b/loopx/pr_review.py index 85dc96dd0a..d97d2abb51 100644 --- a/loopx/pr_review.py +++ b/loopx/pr_review.py @@ -863,9 +863,8 @@ def evaluate(review: Mapping[str, Any]) -> tuple[dict[str, Any], bool]: return evaluations[0][0] from .capabilities.pr_review_queue.approval_closeout import select_review_history include_model_coverage = bool(reviewer_model_family and pr.get("reviews_complete") is True) - history = {"head": head_oid, "author": pr_author, - "reviews_complete": pr.get("reviews_complete") is True, - "reviews": [{"state": conclusion["state"], + review_rows: list[dict[str, object]] = [ + {"state": conclusion["state"], "login": str(_as_dict(review.get("author")).get("login") or ""), "head": str(_as_dict(review.get("commit")).get("oid") or ""), "time": _parse_updated_epoch(review.get("submittedAt")), @@ -875,9 +874,18 @@ def evaluate(review: Mapping[str, Any]) -> tuple[dict[str, Any], bool]: "submitted_at": review.get("submittedAt"), "conclusion": conclusion, **({"review_body_valid": body_valid} if include_model_coverage else {})} - for review, (conclusion, body_valid) in zip(reviews, evaluations, strict=True)]} + for review, (conclusion, body_valid) in zip(reviews, evaluations, strict=True) + ] + history = {"head": head_oid, "author": pr_author, + "reviews_complete": pr.get("reviews_complete") is True, "reviews": review_rows} + model_reports: list[tuple[str, str] | None] = [] if include_model_coverage: + from .capabilities.pr_review_queue.model_coverage import published_review_model + + model_reports = [published_review_model(review.get("body")) for review in reviews] history["include_independent_coverage"] = True + for row, report in zip(review_rows, model_reports, strict=True): + row["review_model_family"] = report[0] if report else None selection = select_review_history(history) index = cast(int, selection["index"]) independent = selection.get("independent_review") @@ -890,14 +898,14 @@ def evaluate(review: Mapping[str, Any]) -> tuple[dict[str, Any], bool]: and isinstance(activity_ids, list) and (conclusion["valid"] or independent) ): - from .capabilities.pr_review_queue.model_coverage import needs_cross_model_review, published_review_model + from .capabilities.pr_review_queue.model_coverage import needs_cross_model_review if needs_cross_model_review( family=reviewer_model_family, history=history, independent_coverage_ids=coverage_ids, independent_activity_ids=activity_ids, - model_reports=[published_review_model(review.get("body")) for review in reviews], + model_reports=model_reports, ): conclusion["cross_model_review_needed"] = True return conclusion | ({"independent_review": independent} if independent else {}) diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 1816c54ae5..52171b3488 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -814,24 +814,31 @@ def test_later_other_family_does_not_repeat_a_current_matching_review(shared_log "first_family,second_family", [("Gemini", "gpt-6.2-sol"), ("gpt-6.2-sol", "Gemini")], ) +@pytest.mark.parametrize("state", ["APPROVED", "COMMENTED"]) def test_shared_independent_login_keeps_both_model_families( - first_family: str, second_family: str, + first_family: str, second_family: str, state: str, ) -> None: login = "shared-peer" - first = _github_model_review(10, login=login, family=first_family, state="APPROVED") - second = _github_model_review(11, login=login, family=second_family, state="APPROVED") - repeat = _github_model_review(12, login=login, family=first_family, state="APPROVED") + prior_reviews = [] + if state == "COMMENTED": + prior_reviews.append( + _github_model_review(9, login=login, family=first_family, state="APPROVED") + ) + first = _github_model_review(10, login=login, family=first_family, state=state) + second = _github_model_review(11, login=login, family=second_family, state=state) + repeat = _github_model_review(12, login=login, family=first_family, state=state) comment = _github_model_review(13, login=login, family=first_family) comment["body"] = str(comment["body"]).splitlines()[0] + "\n\nOne follow-up question." missing = "gemini" if second_family == "Gemini" else "gpt" - one_family = _github_model_pr([first], reviewDecision="APPROVED") + one_family = _github_model_pr([*prior_reviews, first], reviewDecision="APPROVED") assert _model_queue_row(one_family, family=missing)["review_action_kind"] == ( "rereview_pull_request_exact_head" ) - for reviews in ([first, second], [first, second, repeat], [first, second, repeat, comment]): - pr = _github_model_pr(reviews, reviewDecision="APPROVED") + for later in ([], [repeat], [repeat, comment]): + pr = _github_model_pr([*prior_reviews, first, second, *later], reviewDecision="APPROVED") ordinary = _model_queue_row(pr, family=None) + assert ordinary["review_conclusion"]["state"] == "APPROVED" for family in ("gpt", "gemini"): assert _model_queue_row(pr, family=family) == ordinary From 2e61725e8e024814acccc89a1e73e6a36e946d87 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:31:14 +0800 Subject: [PATCH 9/9] Retain model coverage across unattributed review activity Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- loopx/capabilities/pr_review_queue/README.md | 7 +++---- .../capabilities/pr_review_approval_closeout.ts | 6 +----- tests/capabilities/test_pr_review_queue.py | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/loopx/capabilities/pr_review_queue/README.md b/loopx/capabilities/pr_review_queue/README.md index 398645ab24..6a3d80e15d 100644 --- a/loopx/capabilities/pr_review_queue/README.md +++ b/loopx/capabilities/pr_review_queue/README.md @@ -662,10 +662,9 @@ submitted a later review. If different models share an independent GitHub account, each valid, attributable formal review on the current head supplies coverage while that account maintains the same formal opinion. The latest qualified `COMMENTED` review from each known family in that history can also -supply coverage. A qualified comment with unknown model provenance resets the -commented coverage for that account. An ordinary follow-up preserves both kinds -of evidence. A dismissal, a different head or a changed formal opinion resets -the account's coverage history. +supply coverage. A comment with unknown model provenance can record account +activity and preserves existing family coverage. A dismissal, a different head +or a changed formal opinion resets the account's coverage history. A qualifying review published under the shared author login with a matching runtime declaration also suppresses repeat work for that model. GitHub independence still requires independent account evidence. diff --git a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts index b6fcbf0b5c..78340fe75c 100644 --- a/loopx/control_plane/capabilities/pr_review_approval_closeout.ts +++ b/loopx/control_plane/capabilities/pr_review_approval_closeout.ts @@ -78,7 +78,7 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], formalReviewIds: number[]; commentedReviewIds: Map; }>(); - // A follow-up preserves the latest qualified comment for each model family. + // Keep the latest qualified comment for each attributable family. // Process oldest first so an opinion change retires the prior coverage. for (const row of submitted.sort((a, b) => a.time - b.time || a.id - b.id)) { if (!logins.has(row.login)) continue; @@ -100,10 +100,6 @@ function independentReviewEvidence(reviews: readonly ProviderOpinion[], const family = row.source.review_model_family; if (typeof family === "string" && family) { account.commentedReviewIds.set(family, row.id); - } else { - // A qualified review with unknown model provenance still replaces - // the prior commented evidence, as it did for a single comment. - account.commentedReviewIds.clear(); } } } diff --git a/tests/capabilities/test_pr_review_queue.py b/tests/capabilities/test_pr_review_queue.py index 52171b3488..a1f1db6323 100644 --- a/tests/capabilities/test_pr_review_queue.py +++ b/tests/capabilities/test_pr_review_queue.py @@ -829,13 +829,15 @@ def test_shared_independent_login_keeps_both_model_families( repeat = _github_model_review(12, login=login, family=first_family, state=state) comment = _github_model_review(13, login=login, family=first_family) comment["body"] = str(comment["body"]).splitlines()[0] + "\n\nOne follow-up question." + unattributed = _github_model_review(14, login=login, family="unknown") + unattributed["body"] = str(unattributed["body"]).split("\n\n", 1)[1] missing = "gemini" if second_family == "Gemini" else "gpt" one_family = _github_model_pr([*prior_reviews, first], reviewDecision="APPROVED") assert _model_queue_row(one_family, family=missing)["review_action_kind"] == ( "rereview_pull_request_exact_head" ) - for later in ([], [repeat], [repeat, comment]): + for later in ([], [repeat], [repeat, comment], [repeat, comment, unattributed]): pr = _github_model_pr([*prior_reviews, first, second, *later], reviewDecision="APPROVED") ordinary = _model_queue_row(pr, family=None) assert ordinary["review_conclusion"]["state"] == "APPROVED" @@ -868,7 +870,10 @@ def test_shared_independent_login_retires_earlier_formal_evidence(intervening: s "with_formal_opinion", [True, False], ids=["keep-gpt-coverage", "keep-gemini-review-trigger"], ) -def test_follow_up_preserves_qualified_commented_review(with_formal_opinion: bool) -> None: +@pytest.mark.parametrize("complete_comment", [False, True], ids=["brief", "complete-unattributed"]) +def test_follow_up_preserves_qualified_commented_review( + with_formal_opinion: bool, complete_comment: bool, +) -> None: login = "shared-peer" if with_formal_opinion: reviews = [ @@ -885,7 +890,10 @@ def test_follow_up_preserves_qualified_commented_review(with_formal_opinion: boo before = _github_model_pr(reviews, reviewDecision=decision) assert _model_queue_row(before)["review_action_kind"] == action follow_up = _github_model_review(12, login=login, family="gpt-6.2-sol") - follow_up["body"] = "Thanks for the update. I will check the next revision." + if complete_comment: + follow_up["body"] = str(follow_up["body"]).split("\n\n", 1)[1] + else: + follow_up["body"] = "Thanks for the update. I will check the next revision." after = _github_model_pr([*reviews, follow_up], reviewDecision=decision) row = _model_queue_row(after) ordinary = _model_queue_row(after, family=None) @@ -1000,7 +1008,7 @@ def test_non_covering_review_leaves_full_cross_family_review_due(pretend_matchin matching["author"] = {"login": "shared-review-account", "is_bot": False} reviews = [matching, _github_model_review(11, login="second-peer", family="Gemini")] if pretend_matching == "retired": - reviews.append(_github_model_review(12, login="first-peer", family="unknown")) + reviews.append(_github_model_review(12, login="first-peer", family="gpt-6.2-sol", sha="2" * 40)) assert _model_queue_row(_github_model_pr(reviews))["review_action_kind"] == ( "rereview_pull_request_exact_head" )