Reproduce v2 post-hoc candidates and verify one BM25 repair

.github/workflows/evaluate-v2-posthoc.yml

@@ -0,0 +1,67 @@
+name: evaluate-v2-posthoc
+on:
+  pull_request:
+    paths:
+      - 'evaluation/t4-e2b-nf4-v2/**'
+      - 'benchmarks/grade_t4_posthoc_case.py'
+      - '.github/workflows/evaluate-v2-posthoc.yml'
+permissions:
+  contents: read
+jobs:
+  revalidate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m pip install -e . pyarrow==26.0.1
+      - run: python benchmarks/prepare_repair_inputs.py --output-dir evaluation-inputs
+      - run: python benchmarks/prepare_edit_snapshots.py --inputs evaluation-inputs/inputs.jsonl --manifest evaluation-inputs/manifest.json --output-dir evaluation-sources
+      - run: python benchmarks/revalidate_structured_run.py --run-dir evaluation/t4-e2b-nf4-v2/original --inputs evaluation-inputs/inputs.jsonl --manifest evaluation-inputs/manifest.json --source-dir evaluation-sources --output-dir posthoc-results
+      - name: Verify frozen candidate set
+        run: |
+          python - <<'PY'
+          import json
+          from pathlib import Path
+          report = json.loads(Path("posthoc-results/report.json").read_text())
+          candidates = {(r["instance_id"],r["arm"]) for r in report["records"] if r["final_status"] == "applicable"}
+          assert candidates == {("pydicom__pydicom-1694","bm25"),("sqlfluff__sqlfluff-2419","bm25")}, candidates
+          print("LCG_V2_REVALIDATION_JSON=" + json.dumps(report))
+          PY
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: v2-posthoc-revalidation
+          path: |
+            posthoc-results/
+            evaluation-inputs/manifest.json
+            evaluation-sources/manifest.json
+  controlled-grading:
+    needs: revalidate
+    runs-on: ubuntu-latest
+    timeout-minutes: 55
+    strategy:
+      fail-fast: false
+      matrix:
+        instance: ['pydicom__pydicom-1694', 'sqlfluff__sqlfluff-2419']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: actions/download-artifact@v4
+        with:
+          name: v2-posthoc-revalidation
+          path: replay-artifact
+      - run: python -m pip install swebench==4.1.0 pyarrow==26.0.1
+      - run: docker version
+      - run: python benchmarks/grade_t4_posthoc_case.py --instance ${{ matrix.instance }} --input-dir replay-artifact/posthoc-results --output-dir grading-results
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: v2-controls-${{ matrix.instance }}
+          path: |
+            grading-results/
+            logs/

benchmarks/grade_t4_posthoc_case.py

@@ -0,0 +1,112 @@
+"""Official test-based grading of the only strict-applicable, source-changing export."""
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+from run_swebench_lite import obtain_tasks
+
+INSTANCE = None
+MARKER_PATCH = (
+    "diff --git a/.lcg_unfixed_control b/.lcg_unfixed_control\n"
+    "new file mode 100644\n--- /dev/null\n+++ b/.lcg_unfixed_control\n"
+    "@@ -0,0 +1 @@\n+Unmodified program-source control.\n"
+)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input-dir", required=True)
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--instance", required=True)
+    args = parser.parse_args()
+    global INSTANCE
+    INSTANCE = args.instance
+    if INSTANCE not in {"pydicom__pydicom-1694", "sqlfluff__sqlfluff-2419"}:
+        raise ValueError("instance is outside the frozen post-hoc candidate set")
+    output = Path(args.output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory() as directory:
+        tasks, dataset = obtain_tasks(Path(directory), "6ec7bb89b9342f664a54a6e0a6ea6501d3437cc2")
+    task = next(t for t in tasks if t["instance_id"] == INSTANCE)
+    (output / "task.json").write_text(json.dumps([task]))
+    rows = [json.loads(line) for line in (Path(args.input_dir) / "predictions.bm25.posthoc.jsonl").read_text().splitlines()]
+    candidate = next(r["model_patch"] for r in rows if r["instance_id"] == INSTANCE)
+    cases = {"gold": task["patch"], "unfixed": MARKER_PATCH, "bm25_posthoc": candidate}
+    # This pre-fix project uses nose-style setup/teardown, removed in pytest 8.
+    # Pin only the test dependency; do not change project code, labels or patches.
+    wrapper = (
+        "from swebench.harness.constants.python import SPECS_PYDICOM\n"
+        "import runpy\n"
+        "for spec in SPECS_PYDICOM.values():\n"
+        "    spec['pip_packages'] = ['pytest==7.4.4' if p == 'pytest' else p "
+        "for p in spec.get('pip_packages', [])]\n"
+        "runpy.run_module('swebench.harness.run_evaluation', run_name='__main__')\n"
+    )
+    statuses = {}
+    for label, patch in cases.items():
+        model_name = "lcg-control-" + label
+        prediction = output / f"{label}.jsonl"
+        prediction.write_text(json.dumps({"instance_id": INSTANCE,
+            "model_name_or_path": model_name, "model_patch": patch}) + "\n")
+        run_id = "lcg-v2-" + INSTANCE + "-" + label + "-" + os.environ.get("GITHUB_RUN_ID", "local")
+        command = [sys.executable, "-c", wrapper,
+            "--dataset_name", str(output / "task.json"), "--split", "dev",
+            "--predictions_path", str(prediction), "--instance_ids", INSTANCE,
+            "--max_workers", "1", "--namespace", "none",
+            "--cache_level", "instance", "--timeout", "600", "--run_id", run_id]
+        print("Grading", label, "with swebench 4.1.0", flush=True)
+        log_path = output / f"{label}.log"
+        try:
+            with log_path.open("w") as log:
+                process = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT,
+                                         text=True, timeout=1500)
+            matches = [p for p in Path("logs").rglob("report.json")
+                       if run_id in str(p) and INSTANCE in str(p)]
+            report = matches[0] if len(matches) == 1 else output / "missing-report.json"
+            test_log = report.parent / "test_output.txt"
+            statuses[label] = {"exit_code": process.returncode,
+                              "report": json.loads(report.read_text()) if report.exists() else None,
+                              "test_failure_excerpt": (test_log.read_text()[max(0, test_log.read_text().find("FAILURES")):][:12000]
+                                  if test_log.exists() else None),
+                              "test_output_head": test_log.read_text()[:10000] if test_log.exists() else None,
+                              "test_output_tail": test_log.read_text()[-10000:] if test_log.exists() else None,
+                              "log_tail": log_path.read_text()[-4000:]}
+        except Exception as error:
+            statuses[label] = {"error": str(error)}
+        (output / "status.json").write_text(json.dumps(statuses, indent=2))
+        case_report = statuses[label].get("report") or {}
+        resolved = case_report.get(INSTANCE, {}).get("resolved")
+        detail = case_report.get(INSTANCE, {})
+        status = detail.get("tests_status", {})
+        f2p = status.get("FAIL_TO_PASS", {})
+        p2p = status.get("PASS_TO_PASS", {})
+        expected_f2p = task["FAIL_TO_PASS"]
+        expected_p2p = task["PASS_TO_PASS"]
+        if isinstance(expected_f2p, str):
+            expected_f2p = json.loads(expected_f2p)
+        if isinstance(expected_p2p, str):
+            expected_p2p = json.loads(expected_p2p)
+        observed_f2p = set(f2p.get("success", []) + f2p.get("failure", []))
+        observed_p2p = set(p2p.get("success", []) + p2p.get("failure", []))
+        complete = (detail.get("patch_successfully_applied") is True
+            and observed_f2p == set(expected_f2p) and observed_p2p == set(expected_p2p))
+        qualified = complete and not p2p.get("failure") and (
+            (resolved is True and not f2p.get("failure")) if label == "gold"
+            else (resolved is False and bool(f2p.get("failure"))) if label == "unfixed" else True)
+        statuses[label]["control_qualified"] = qualified if label in {"gold", "unfixed"} else None
+        (output / "status.json").write_text(json.dumps(statuses, indent=2))
+        if label in {"gold", "unfixed"} and not qualified:
+            print("Control validation failed; stopping before model grading.", flush=True)
+            break
+    result = {"instance_id": INSTANCE, "dataset": dataset, "harness": "swebench==4.1.0",
+              "environment_override": {"pydicom_pytest": "7.4.4",
+                  "reason": "Restore legacy nose-style setup/teardown support; applied to pydicom specs only"},
+              "scope": "Controlled grading of a post-hoc final-step candidate, not a complete official repair score",
+              "cases": statuses}
+    (output / "results.json").write_text(json.dumps(result, indent=2) + "\n")
+    print("LCG_GRADE_JSON=" + json.dumps(result), flush=True)
+
+if __name__ == "__main__":
+    main()

evaluation/t4-e2b-nf4-v2/PROVENANCE.md

@@ -0,0 +1 @@
+Original v2 attempt records and empty prediction exports supplied by the author. Original run commit: 48fa23825b2a1cfbb124a086673ae97fe1aad02d. Runtime package environment dumps and private transfer URLs are omitted. Model responses may quote upstream source; those quotes retain upstream terms. Regenerate gold-free inputs at the pinned dataset revision and verify the manifest checksum before post-hoc revalidation. Post-hoc predictions are distinct derived artifacts, never replacements for original v2 results. No test outcome is inferred from application checks.

evaluation/t4-e2b-nf4-v2/original/attempts/marshmallow-code__marshmallow-1343.bm25.json

@@ -0,0 +1 @@
+{"instance_id": "marshmallow-code__marshmallow-1343", "arm": "bm25", "patch": "", "source_snapshot_sha256": "6f3afba4a09b3550be1e09217781cc8c804e2849e012dc9960c57be4ac79a929", "status": "error", "error": "return one JSON object without fences or prose", "steps": [{"input_tokens": 2808, "context_characters": 7430, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "c4f202c2eb74a75c9b8ddd6b32f4365c27511111fec0c02f1e4e6f953ba6fa9a", "output_tokens": 132, "response": "```json\n{\n  \"edits\": [\n    {\n      \"path\": \"package/file.py\",\n      \"old\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')  # <- issue here\\n    def validate_value(self, value):\\n        pass\",\n      \"new\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')\\n    def validate_value(self, value):\\n        pass\"\n    }\n  ]\n}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7510396416, "peak_gpu_reserved_bytes": 7738490880, "seconds": 14.698119918999964, "validation_error": "return one JSON object without fences or prose"}, {"input_tokens": 2969, "context_characters": 7430, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "e7f8b1e84e1aa130bb3688165363c9d0638888ac5f61cfb4a18194f00a446faf", "output_tokens": 132, "response": "```json\n{\n  \"edits\": [\n    {\n      \"path\": \"package/file.py\",\n      \"old\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')  # <- issue here\\n    def validate_value(self, value):\\n        pass\",\n      \"new\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')\\n    def validate_value(self, value):\\n        pass\"\n    }\n  ]\n}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7585712128, "peak_gpu_reserved_bytes": 7811891200, "seconds": 14.817595222999898, "validation_error": "return one JSON object without fences or prose"}], "seconds": 29.64976290200002}

evaluation/t4-e2b-nf4-v2/original/attempts/marshmallow-code__marshmallow-1343.graph.json

@@ -0,0 +1 @@
+{"instance_id": "marshmallow-code__marshmallow-1343", "arm": "graph", "patch": "", "source_snapshot_sha256": "6f3afba4a09b3550be1e09217781cc8c804e2849e012dc9960c57be4ac79a929", "status": "error", "error": "return one JSON object without fences or prose", "steps": [{"input_tokens": 2961, "context_characters": 7960, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "e601028dd603837e63e316a758267e5d54cf5dfceed7fdbf383b8e646c7aff9d", "output_tokens": 132, "response": "```json\n{\n  \"edits\": [\n    {\n      \"path\": \"package/file.py\",\n      \"old\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')  # <- issue here\\n    def validate_value(self, value):\\n        pass\",\n      \"new\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')\\n    def validate_value(self, value):\\n        pass\"\n    }\n  ]\n}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7582035456, "peak_gpu_reserved_bytes": 7805599744, "seconds": 14.753714582000157, "validation_error": "return one JSON object without fences or prose"}, {"input_tokens": 3122, "context_characters": 7960, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "a6cea9495706044028b1db862c94b79f2417d9d95a64604f36b2584339a6a993", "output_tokens": 132, "response": "```json\n{\n  \"edits\": [\n    {\n      \"path\": \"package/file.py\",\n      \"old\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')  # <- issue here\\n    def validate_value(self, value):\\n        pass\",\n      \"new\": \"class Bar(Schema):\\n    value = fields.String()\\n\\n    @validates('value')\\n    def validate_value(self, value):\\n        pass\"\n    }\n  ]\n}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7661223936, "peak_gpu_reserved_bytes": 7927234560, "seconds": 14.973738913000034, "validation_error": "return one JSON object without fences or prose"}], "seconds": 29.778565179999987}

evaluation/t4-e2b-nf4-v2/original/attempts/marshmallow-code__marshmallow-1359.bm25.json

@@ -0,0 +1 @@
+{"instance_id": "marshmallow-code__marshmallow-1359", "arm": "bm25", "patch": "", "source_snapshot_sha256": "cbf79693388595a3dd0138a89ad71e6085f29b732a920633f5d204e87df38464", "status": "error", "error": "return one JSON object without fences or prose", "steps": [{"input_tokens": 2228, "context_characters": 6008, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "f4a4da9b90dcb9462227bd83dbe71c2e76118e53dcc8f5531b8d42ee07cec34f", "output_tokens": 11, "response": "```json\n{\"edits\":[]}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7267818496, "peak_gpu_reserved_bytes": 7375683584, "seconds": 1.8191050690002157, "validation_error": "return one JSON object without fences or prose"}, {"input_tokens": 2268, "context_characters": 6008, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "4d52956cc2148ed1664d85f91685d127448671b183f9d6e64fa5e946f88f86c5", "output_tokens": 11, "response": "```json\n{\"edits\":[]}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7282323456, "peak_gpu_reserved_bytes": 7470055424, "seconds": 2.0032523979998587, "validation_error": "return one JSON object without fences or prose"}], "seconds": 3.865513715999896}

evaluation/t4-e2b-nf4-v2/original/attempts/marshmallow-code__marshmallow-1359.graph.json

@@ -0,0 +1 @@
+{"instance_id": "marshmallow-code__marshmallow-1359", "arm": "graph", "patch": "", "source_snapshot_sha256": "cbf79693388595a3dd0138a89ad71e6085f29b732a920633f5d204e87df38464", "status": "error", "error": "return one JSON object without fences or prose", "steps": [{"input_tokens": 2787, "context_characters": 7852, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "f5493612246f831a6c50fec43858c24ae7a05ad5804a30692cd3232daad954ce", "output_tokens": 90, "response": "```json\n{\n  \"edits\": [\n    {\n      \"path\": \"src/marshmallow/schema.py\",\n      \"old\": \"class MySchema(Schema):\n    times = fields.List(fields.DateTime())\",\n      \"new\": \"class MySchema(Schema):\n    times = fields.List(fields.DateTime())\"\n    }\n  ]\n}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7502244352, "peak_gpu_reserved_bytes": 7730102272, "seconds": 9.994519429999855, "validation_error": "return one JSON object without fences or prose"}, {"input_tokens": 2906, "context_characters": 7852, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "a0d8f46376e35d96bc12279c1ceea7bbb90cc3809fb86acac5662802db30b7c4", "output_tokens": 90, "response": "```json\n{\n  \"edits\": [\n    {\n      \"path\": \"src/marshmallow/schema.py\",\n      \"old\": \"class MySchema(Schema):\n    times = fields.List(fields.DateTime())\",\n      \"new\": \"class MySchema(Schema):\n    times = fields.List(fields.DateTime())\"\n    }\n  ]\n}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7558277632, "peak_gpu_reserved_bytes": 7776239616, "seconds": 10.233021708999786, "validation_error": "return one JSON object without fences or prose"}], "seconds": 20.283444058999976}

evaluation/t4-e2b-nf4-v2/original/attempts/pvlib__pvlib-python-1072.bm25.json

@@ -0,0 +1 @@
+{"instance_id": "pvlib__pvlib-python-1072", "arm": "bm25", "patch": "", "source_snapshot_sha256": "af82a118ad86eb2d77eae234e4e26962116573c28329eff1829aa7de67152482", "status": "error", "error": "return one JSON object without fences or prose", "steps": [{"input_tokens": 1625, "context_characters": 1920, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "ce2f8fb3e275f351ff4d1b96dde4bc5279602cdc4fc1e1b49205f97772b6a8a9", "output_tokens": 11, "response": "```json\n{\"edits\":[]}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7060618240, "peak_gpu_reserved_bytes": 7216300032, "seconds": 1.9533041120002963, "validation_error": "return one JSON object without fences or prose"}, {"input_tokens": 1665, "context_characters": 1920, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "db80c820912bfa9c73275ce0b33ffd32235a833cfc1115559424dbbafc76da37", "output_tokens": 11, "response": "```json\n{\"edits\":[]}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7072102912, "peak_gpu_reserved_bytes": 7260340224, "seconds": 1.668533213000046, "validation_error": "return one JSON object without fences or prose"}], "seconds": 3.677587344999665}

evaluation/t4-e2b-nf4-v2/original/attempts/pvlib__pvlib-python-1072.graph.json

@@ -0,0 +1 @@
+{"instance_id": "pvlib__pvlib-python-1072", "arm": "graph", "patch": "", "source_snapshot_sha256": "af82a118ad86eb2d77eae234e4e26962116573c28329eff1829aa7de67152482", "status": "error", "error": "return one JSON object without fences or prose", "steps": [{"input_tokens": 3410, "context_characters": 6838, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "5587fe1339973bdf944cd5aed83b41c7a7b226c13da57eba2fac58b55c781363", "output_tokens": 11, "response": "```json\n{\"edits\":[]}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7811358208, "peak_gpu_reserved_bytes": 8092909568, "seconds": 2.4603071569999884, "validation_error": "return one JSON object without fences or prose"}, {"input_tokens": 3450, "context_characters": 6838, "issue_truncated": false, "context_truncated": false, "prompt_sha256": "fa5756f23beaa1d8994d92ff545b91f866065ee7ae84f2386ba0af1f1bce6843", "output_tokens": 11, "response": "```json\n{\"edits\":[]}\n```", "output_limit": false, "peak_gpu_allocated_bytes": 7832422912, "peak_gpu_reserved_bytes": 8139046912, "seconds": 2.4737165509995975, "validation_error": "return one JSON object without fences or prose"}], "seconds": 4.994577687999936}