Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
# make release SHA=<sha> plan, then apply at once (no prompt), read back
# make release-rule-test RULE=<id> SHA=<sha> Console-only edit of a scratch rule
# rollback: make release SHA=<previous release sha>
# make self-review-plan read-only: what the R-self setup would create
# make self-review-setup create the R-self workspace and rule, write its ID into analyze.py

TOOL = cd review && uv run --frozen python tools/release.py

.PHONY: release-plan release release-rule-test
.PHONY: release-plan release release-rule-test self-review-plan self-review-setup

release-plan:
$(TOOL) plan --sha "$(SHA)" $(if $(NO_CONSOLE),--no-console)
Expand All @@ -20,3 +22,9 @@ release:

release-rule-test:
$(TOOL) rule-test --rule "$(RULE)" --sha "$(SHA)"

self-review-plan:
cd review && uv run --frozen python tools/self_review_setup.py plan

self-review-setup:
cd review && uv run --frozen python tools/self_review_setup.py setup
39 changes: 36 additions & 3 deletions docs/claude-review-design.md

Large diffs are not rendered by default.

22 changes: 21 additions & 1 deletion review/src/oc_review/analyze.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,15 @@
ANTHROPIC_ORGANIZATION_ID = "2bc920f9-977b-43b2-870a-f840e4ce809c"
ANTHROPIC_SERVICE_ACCOUNT_ID = "svac_01PiMgfb6qJgNpLZUV8hprCC"

# Self-review rule R-self (spec "Auth and cost", "Self-review rule"): only for a
# `pull_request_target` run in OperationCode/.github, which runs this file from
# `main`, so R1 (bound to the release SHA) refuses its token. Own workspace,
# same service account. The release tool never edits it. All zeros until the
# owner creates it: such a run fails with "federation not configured".
SELF_REVIEW_FEDERATION_RULE_ID = "fdrl_000000000000000000000000"
SELF_REVIEW_REPOSITORY_ID = "1432342572" # OperationCode/.github
SELF_REVIEW_EVENT_NAME = "pull_request_target"

FEDERATION_RULE_ID_RE = re.compile(r"^fdrl_[A-Za-z0-9]{24}$")
ORGANIZATION_ID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
SERVICE_ACCOUNT_ID_RE = re.compile(r"^svac_[A-Za-z0-9]{24}$")
Expand Down Expand Up @@ -453,6 +462,15 @@ def federation_ids() -> FederationIds:
return FederationIds(ANTHROPIC_FEDERATION_RULE_ID, ANTHROPIC_ORGANIZATION_ID, ANTHROPIC_SERVICE_ACCOUNT_ID)


def self_review_federation(env: Mapping[str, str]) -> FederationIds | None:
"""R-self for a `pull_request_target` run in `.github`, else None (R1).
Only runner-set variables decide; a workflow cannot override `GITHUB_*`."""
if (env.get("GITHUB_REPOSITORY_ID") == SELF_REVIEW_REPOSITORY_ID
and env.get("GITHUB_EVENT_NAME") == SELF_REVIEW_EVENT_NAME):
return FederationIds(SELF_REVIEW_FEDERATION_RULE_ID, ANTHROPIC_ORGANIZATION_ID, ANTHROPIC_SERVICE_ACCOUNT_ID)
return None


def _all_zero(s: str) -> bool:
return set(s) <= {"0", "-"}

Expand Down Expand Up @@ -796,7 +814,8 @@ def analyze(
pass problem; every path gives three results.

`federation` is the Anthropic federation IDs. None (the review path) means
the production constants of this module. The eval runner passes its own.
the production constants of this module (R1). `main` passes R-self for a
`.github` self-review run; the eval runner passes its own.

`github` is None without `GITHUB_TOKEN`, and `runner_temp` is None without
`RUNNER_TEMP`. Only a bundle with status `ok` needs them; without them its
Expand Down Expand Up @@ -1016,6 +1035,7 @@ def main(
client_factory=client_factory,
clock=clock,
deadline_seconds=deadline_seconds,
federation=self_review_federation(env),
)
except Exception as exc: # noqa: BLE001 - fail closed with three records
print(f"analyze: internal error: {type(exc).__name__}", file=sys.stderr)
Expand Down
34 changes: 34 additions & 0 deletions review/tests/unit/fixtures/oidc/v3-direct-dispatch-claims.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"_source": "decoded OIDC claims printed (no token) by V3 run 40259047420, OperationCode/claude-review-p0-source, a direct workflow_dispatch run (not ruleset-required, not reusable); actor fields removed",
"claims": {
"aud": "https://api.anthropic.com",
"base_ref": "",
"check_run_id": "121367205481",
"event_name": "workflow_dispatch",
"exp": 1792981857,
"head_ref": "",
"iat": 1792981557,
"iss": "https://token.actions.githubusercontent.com",
"job_workflow_ref": "OperationCode/claude-review-p0-source/.github/workflows/p0-required.yml@refs/heads/main",
"job_workflow_sha": "c2390aef5f2f9ae9501ad1365c549150a2488e4a",
"jti": "92171d42-e134-4465-b831-7df9240a43e1",
"nbf": 1792981257,
"ref": "refs/heads/main",
"ref_protected": "false",
"ref_type": "branch",
"repository": "OperationCode/claude-review-p0-source",
"repository_id": "1433591286",
"repository_owner": "OperationCode",
"repository_owner_id": "10479364",
"repository_visibility": "public",
"run_attempt": "1",
"run_id": "40259047420",
"run_number": "1",
"runner_environment": "github-hosted",
"sha": "c2390aef5f2f9ae9501ad1365c549150a2488e4a",
"sub": "repo:OperationCode@10479364/claude-review-p0-source@1433591286:ref:refs/heads/main",
"workflow": "p0-required",
"workflow_ref": "OperationCode/claude-review-p0-source/.github/workflows/p0-required.yml@refs/heads/main",
"workflow_sha": "c2390aef5f2f9ae9501ad1365c549150a2488e4a"
}
}
97 changes: 94 additions & 3 deletions review/tests/unit/test_analyze.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ class Run:
oidc: FakeOidc
out_dir: Path
token_files: list
factory_kwargs: list = field(default_factory=list)


def run_main(tmp_path, *, client=None, bundle=None, digest=None, github=None, oidc=None, env=None,
Expand Down Expand Up @@ -219,11 +220,12 @@ def run_main(tmp_path, *, client=None, bundle=None, digest=None, github=None, oi
client = client or FakeClient()
github = github or FakeGitHub()
oidc = oidc or FakeOidc()
token_files = []
token_files, factory_kwargs = [], []

def factory(token_file, *, deadline, clock):
def factory(token_file, *, deadline, clock, **kw):
mode = stat.S_IMODE(token_file.stat().st_mode)
token_files.append((token_file, mode, token_file.read_text()))
factory_kwargs.append(kw)
if factory_exc is not None:
raise factory_exc
if real_factory is not None:
Expand Down Expand Up @@ -251,7 +253,7 @@ def factory(token_file, *, deadline, clock):
for line in output.read_text().splitlines():
k, _, v = line.partition("=")
outputs[k] = v
return Run(code, records, raw, outputs, client, github, oidc, out_dir, token_files)
return Run(code, records, raw, outputs, client, github, oidc, out_dir, token_files, factory_kwargs)


def statuses(run):
Expand Down Expand Up @@ -1429,6 +1431,95 @@ def factory(token_file, *, deadline, clock): # no `federation` kwarg: the defau
assert calls == [1] and {r.status for r in results.values()} == {"ok"}


# --- the self-review rule R-self (PRs to .github) ------------------------------------

SELF_ENV = {"GITHUB_REPOSITORY_ID": analyze.SELF_REVIEW_REPOSITORY_ID, "GITHUB_EVENT_NAME": "pull_request_target"}
SELF_RULE = "fdrl_01SelfReviewRuleIdAbcdef"


@pytest.mark.parametrize("env, chosen", [
(SELF_ENV, True),
({**SELF_ENV, "GITHUB_EVENT_NAME": "workflow_dispatch"}, False),
({**SELF_ENV, "GITHUB_EVENT_NAME": "push"}, False),
({**SELF_ENV, "GITHUB_EVENT_NAME": "pull_request"}, False),
({**SELF_ENV, "GITHUB_EVENT_NAME": "PULL_REQUEST_TARGET"}, False),
({**SELF_ENV, "GITHUB_REPOSITORY_ID": "1432457406"}, False), # claude-review-test
({**SELF_ENV, "GITHUB_REPOSITORY_ID": " 1432342572"}, False),
({"GITHUB_EVENT_NAME": "pull_request_target"}, False),
({"GITHUB_REPOSITORY_ID": analyze.SELF_REVIEW_REPOSITORY_ID}, False),
({}, False),
])
def test_r_self_only_for_pull_request_target_in_dotgithub(env, chosen):
ids = analyze.self_review_federation(env)
if chosen:
assert ids == analyze.FederationIds(analyze.SELF_REVIEW_FEDERATION_RULE_ID, analyze.ANTHROPIC_ORGANIZATION_ID,
analyze.ANTHROPIC_SERVICE_ACCOUNT_ID)
else:
assert ids is None


def test_r_self_is_its_own_rule_and_r1_is_the_release_rule():
assert analyze.SELF_REVIEW_FEDERATION_RULE_ID != analyze.ANTHROPIC_FEDERATION_RULE_ID
assert analyze.FEDERATION_RULE_ID_RE.match(analyze.SELF_REVIEW_FEDERATION_RULE_ID)


def test_placeholder_r_self_fails_every_pass_and_never_mints(tmp_path, configured, monkeypatch):
monkeypatch.setattr(analyze, "SELF_REVIEW_FEDERATION_RULE_ID", "fdrl_000000000000000000000000")
run = run_main(tmp_path, env=SELF_ENV)
assert_three_records(run)
assert set(statuses(run).values()) == {"failed"}
assert set(reasons(run).values()) == {analyze.R_FEDERATION_NOT_CONFIGURED}
assert run.client.calls == {} and run.oidc.calls == [] and run.token_files == []


def test_configured_r_self_reaches_the_client(tmp_path, configured, monkeypatch):
monkeypatch.setattr(analyze, "SELF_REVIEW_FEDERATION_RULE_ID", SELF_RULE)
run = run_main(tmp_path, env=SELF_ENV)
assert set(statuses(run).values()) == {"ok"}
assert run.factory_kwargs == [{"federation": analyze.FederationIds(
SELF_RULE, CONFIGURED_IDS["ANTHROPIC_ORGANIZATION_ID"], CONFIGURED_IDS["ANTHROPIC_SERVICE_ACCOUNT_ID"])}]


V3_DIRECT = json.loads((Path(__file__).parent / "fixtures" / "oidc" / "v3-direct-dispatch-claims.json").read_text())


def dotgithub_direct_claims():
"""The recorded direct-run claim set (V3 run 40259047420), moved to a
`pull_request_target` run of claude-review.yml in .github on main."""
ref = "OperationCode/.github/.github/workflows/claude-review.yml@refs/heads/main"
claims = {**V3_DIRECT["claims"], "event_name": "pull_request_target", "repository": "OperationCode/.github",
"repository_id": analyze.SELF_REVIEW_REPOSITORY_ID, "job_workflow_ref": ref, "workflow_ref": ref,
"job_workflow_sha": JOB_SHA, "workflow_sha": JOB_SHA,
"sub": f"repo:OperationCode@10479364/.github@{analyze.SELF_REVIEW_REPOSITORY_ID}:pull_request"}
return claims


def test_a_recorded_direct_run_token_has_the_job_workflow_claims():
"""A direct run (not reusable, not ruleset-required) carries job_workflow_ref
in branch form and job_workflow_sha, equal to workflow_ref / workflow_sha.
So the local check and R-self's job_workflow_ref claim hold for it."""
c = V3_DIRECT["claims"]
assert c["job_workflow_ref"] == c["workflow_ref"] and c["job_workflow_ref"].endswith("@refs/heads/main")
assert c["job_workflow_sha"] == c["workflow_sha"]
assert analyze.claims_match(c, c["workflow_sha"])


def test_configured_r_self_accepts_a_direct_run_claim_set(tmp_path, configured, monkeypatch):
monkeypatch.setattr(analyze, "SELF_REVIEW_FEDERATION_RULE_ID", SELF_RULE)
run = run_main(tmp_path, env=SELF_ENV, oidc=FakeOidc(make_jwt(**dotgithub_direct_claims())))
assert set(statuses(run).values()) == {"ok"}
assert run.factory_kwargs[0]["federation"].rule_id == SELF_RULE


@pytest.mark.parametrize("env", [{}, {**SELF_ENV, "GITHUB_EVENT_NAME": "workflow_dispatch"},
{**SELF_ENV, "GITHUB_REPOSITORY_ID": "107068150"}])
def test_every_other_run_keeps_r1(tmp_path, configured, monkeypatch, env):
monkeypatch.setattr(analyze, "SELF_REVIEW_FEDERATION_RULE_ID", SELF_RULE)
run = run_main(tmp_path, env=env)
assert set(statuses(run).values()) == {"ok"}
assert run.factory_kwargs == [{}] # no `federation`: the module constants (R1)


def test_default_client_factory_uses_the_federation_parameter(tmp_path, configured):
token_file = tmp_path / "tok"
token_file.write_text(GOOD_JWT)
Expand Down
19 changes: 19 additions & 0 deletions review/tests/unit/test_release_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -566,13 +566,32 @@ def test_rule_test_refuses_the_production_rule():
assert fake.calls == []


def test_release_targets_r1_only_never_r_self():
from oc_review import analyze
assert R1 == analyze.ANTHROPIC_FEDERATION_RULE_ID
assert R1 != analyze.SELF_REVIEW_FEDERATION_RULE_ID
fake = Fake()
assert release.main(["release", "--sha", SHA_NEW], run=fake) == 0
touched = {c[c.index("--federation-rule-id") + 1] for c, _ in fake.calls if "--federation-rule-id" in c}
assert touched == {R1}


def test_rule_test_refuses_a_rule_bound_to_a_branch_ref_such_as_r_self():
fake = Fake()
r_self = "fdrl_01SelfReviewRuleIdAbcdef"
fake.rules[r_self] = rule(r_self, sha="refs/heads/main")
assert release.main(["rule-test", "--rule", r_self, "--sha", SHA_NEW], run=fake) == 2
assert fake.writes() == [] and fake.rules[r_self] == rule(r_self, sha="refs/heads/main")


# ---------------------------------------------------------------- never in CI


def test_no_workflow_runs_the_release_tool():
for path in (ROOT / ".github" / "workflows").glob("*.y*ml"):
text = path.read_text(encoding="utf-8")
assert "release.py" not in text and "make release" not in text and "ant " not in text, path.name
assert "self_review_setup" not in text and "make self-review" not in text, path.name


def test_default_runner_is_guarded_in_tests():
Expand Down
Loading
Loading