diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d48d27..0010147 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,48 @@ on: branches: [main, master] jobs: + supply-chain: + # SDK-3 (security audit 2026-11-06). This job is the check that keeps + # the CI itself honest, and it runs first so a mutable action ref or a + # lockless dependency install fails in seconds instead of after the + # whole test matrix. + # + # The two steps: + # 1. every ``uses:`` points at a 40-hex commit SHA, and every + # dependency install comes from uv.lock; + # 2. uv.lock still matches pyproject.toml, so ``--frozen`` installs + # the versions the commit was reviewed with. + # + # The lint needs no third-party package. It reads the workflows with + # PyYAML when PyYAML is importable and with a text scanner when it is + # not, and a test proves the two scanners agree. Installing PyYAML here + # would be the very thing the lint forbids. + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + # An exact uv version. "latest" would let a new uv change how the + # lock is read. + version: "0.11.28" + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Check the action pins and the lock-based installs + run: python scripts/check_supply_chain.py + + - name: Check that uv.lock matches pyproject.toml + run: uv lock --check + commit-message: # The commits in this repository are authored solely by the repo owner. # That rule is in CLAUDE.md and was still broken twice, so it is checked @@ -31,7 +73,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: # Full history, because the range check needs the merge base with # origin/master, and `fetch-depth: 0` is what makes that base @@ -73,29 +115,34 @@ jobs: python: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python }} - # Cache pip's download cache keyed on the lock-relevant - # surfaces of pyproject.toml. Skips the ~60-90s cold - # install on warm caches; the action also reuses the - # cache across matrix legs when the key matches. - cache: "pip" - cache-dependency-path: pyproject.toml - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # xdist ships in the dev tree already; pin it explicitly so - # a future deps churn can't drop it without breaking CI. - # ``pytest-rerunfailures`` is used by Sprint 0 (coverage) on - # a single rare-flaky test under pytest-xdist on linux - # (thread-scheduling race in the approval-wait fixture); - # pin it for the same reason. - pip install -e ".[dev]" "pytest-xdist>=3.6" "pytest-rerunfailures>=14.0,<16.0" + + - name: Set up uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + # uv caches the resolved packages, not pip's download cache. + # The key is uv.lock, so a lock change invalidates the cache. + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Point uv at the interpreter setup-python installed + # Without this, uv picks its own managed interpreter and the matrix + # leg would not test the Python version it claims to test. + run: echo "UV_PYTHON=${{ steps.setup-python.outputs.python-path }}" >> "$GITHUB_ENV" + + - name: Install the dependencies from the lock + # --frozen: the lock file is the contract. The job must never + # rewrite it, and it must never resolve a version at run time. + # --extra dev: the dev extra carries pytest, pytest-xdist, + # pytest-rerunfailures, pytest-cov, ruff, mypy and the rest. + run: uv sync --frozen --extra dev - name: Run tests # `-n auto` lets xdist pick a worker count from the runner's @@ -105,26 +152,37 @@ jobs: # near single-thread baseline (no shared state between # workers — ``reset_runtime`` autouse fixture in conftest # is per-process by construction under xdist). - run: pytest -n auto --durations=20 + # + # `uv run` puts the command inside the environment the lock + # installed, so pytest is the pinned pytest. + run: uv run --frozen --extra dev pytest -n auto --durations=20 - name: Run ruff - run: ruff check src/ + run: uv run --frozen --extra dev ruff check src/ - name: Run mypy - run: mypy src/ + run: uv run --frozen --extra dev mypy src/ coverage: runs-on: ubuntu-latest permissions: contents: read steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Set up Python + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - cache: "pip" - cache-dependency-path: pyproject.toml - - run: pip install -e ".[dev]" "pytest-xdist>=3.6" "pytest-cov>=5.0" "pytest-rerunfailures>=14.0,<16.0" + - name: Set up uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + enable-cache: true + cache-dependency-glob: uv.lock + - name: Point uv at the interpreter setup-python installed + run: echo "UV_PYTHON=${{ steps.setup-python.outputs.python-path }}" >> "$GITHUB_ENV" + - run: uv sync --frozen --extra dev # Single Python leg for coverage — multi-version coverage # reports don't add signal and double the runner time. 3.12 # is the modern floor for typing-only changes. @@ -132,8 +190,8 @@ jobs: # the data before producing the report. ``coverage run`` only # traced the coordinator process, so every parallel run uploaded # 0 hits even though all tests passed. - - run: pytest -n auto --cov=src/nullrun --cov-branch --cov-report=xml:coverage.xml --cov-report=term - - uses: codecov/codecov-action@v4 + - run: uv run --frozen --extra dev pytest -n auto --cov=src/nullrun --cov-branch --cov-report=xml:coverage.xml --cov-report=term + - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 if: always() with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/publish-test.yml b/.github/workflows/publish-test.yml index a25657e..f27dbc3 100644 --- a/.github/workflows/publish-test.yml +++ b/.github/workflows/publish-test.yml @@ -14,19 +14,29 @@ jobs: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-python@v5 + - name: Set up Python ${{ matrix.python-version }} + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: pyproject.toml + + - name: Set up uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Point uv at the interpreter setup-python installed + run: echo "UV_PYTHON=${{ steps.setup-python.outputs.python-path }}" >> "$GITHUB_ENV" - name: Install dependencies - run: pip install -e ".[dev]" "pytest-xdist>=3.6" + run: uv sync --frozen --extra dev - name: Run tests - run: pytest tests/ -v -n auto + run: uv run --frozen --extra dev pytest tests/ -v -n auto publish: name: Build and publish to TestPyPI @@ -39,24 +49,42 @@ jobs: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-python@v5 + - name: Set up Python + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" + - name: Set up uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Point uv at the interpreter setup-python installed + run: echo "UV_PYTHON=${{ steps.setup-python.outputs.python-path }}" >> "$GITHUB_ENV" + + # This job holds the TestPyPI Trusted Publishing token, so the + # supply-chain check runs here, before the build and the upload. + - name: Check the action pins and the lock-based installs + run: python scripts/check_supply_chain.py + + - name: Check that uv.lock matches pyproject.toml + run: uv lock --check + - name: Build - run: | - pip install --upgrade build - python -m build + # ``build`` and ``twine`` come from the ``publish`` dependency + # group in uv.lock, not from a run-time pip resolution. + run: uv run --frozen --only-group publish python -m build - name: Check dist - run: | - pip install twine - twine check dist/* + run: uv run --frozen --only-group publish twine check dist/* - name: Publish to TestPyPI (Trusted Publishing) - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 with: repository-url: https://test.pypi.org/legacy/ # TestPyPI rejects re-uploads of the same wheel hash with diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 30e70b2..c3d5f92 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,26 +13,36 @@ jobs: permissions: contents: read # 2026-07-08: parallel matrix kept (PyPI publish is a one-shot - # event and the runner is already paid for) but pip cache + # event and the runner is already paid for) but the uv cache # brought in for parity with ci.yml. strategy: matrix: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-python@v5 + - name: Set up Python ${{ matrix.python-version }} + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: pyproject.toml + + - name: Set up uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Point uv at the interpreter setup-python installed + run: echo "UV_PYTHON=${{ steps.setup-python.outputs.python-path }}" >> "$GITHUB_ENV" - name: Install dependencies - run: pip install -e ".[dev]" "pytest-xdist>=3.6" + run: uv sync --frozen --extra dev - name: Run tests - run: pytest tests/ -v -n auto + run: uv run --frozen --extra dev pytest tests/ -v -n auto publish: name: Build and publish @@ -50,24 +60,46 @@ jobs: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-python@v5 + - name: Set up Python + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" + - name: Set up uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Point uv at the interpreter setup-python installed + run: echo "UV_PYTHON=${{ steps.setup-python.outputs.python-path }}" >> "$GITHUB_ENV" + + # This is the job that holds the PyPI Trusted Publishing token + # (``id-token: write`` above), so the supply-chain check belongs + # HERE and not only in ci.yml. A tag push that carries a mutable + # action ref or a lockless install must not reach the upload step. + - name: Check the action pins and the lock-based installs + run: python scripts/check_supply_chain.py + + - name: Check that uv.lock matches pyproject.toml + run: uv lock --check + - name: Build - run: | - pip install --upgrade build - python -m build + # ``build`` and ``twine`` come from the ``publish`` dependency + # group in uv.lock. They used to be installed with + # ``pip install --upgrade build`` and ``pip install twine``, + # which resolved inside the trusted-publishing job. + run: uv run --frozen --only-group publish python -m build - name: Check dist - run: | - pip install twine - twine check dist/* + run: uv run --frozen --only-group publish twine check dist/* - name: Publish to PyPI (Trusted Publishing) - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 # GitHub Release creation — gated on tag pushes so the manual # workflow_dispatch path (used for hotfix re-runs) doesn't create @@ -80,7 +112,7 @@ jobs: # for environments where pip isn't available. - name: Create GitHub Release if: startsWith(github.ref, 'refs/tags/v') - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: tag_name: ${{ github.ref_name }} generate_release_notes: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9f0d2de..4ac954b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,11 +14,18 @@ repos: args: [--fix] - id: ruff-format - - repo: https://github.com/pre-commit/mypy - rev: v1.8.0 - hooks: - - id: mypy - additional_dependencies: [types-all] + # The mypy hook used to come from ``https://github.com/pre-commit/mypy``. + # That mirror no longer exists: GitHub answers 404 and ``git fetch`` + # answers "Repository not found". The consequence was not a dead hook + # but a dead CHAIN — pre-commit installs every repo in the file before + # it runs any hook, so the missing mirror aborted the whole run and + # NO hook in this file executed, including the supply-chain gate + # below. A gate that cannot start is not a gate. + # + # The check is not lost: CI runs ``mypy src/`` (``.github/workflows/ + # ci.yml``) and ``make type-check`` runs ``mypy src/nullrun --strict``. + # The old ``types-all`` extra is dropped with the hook: CI never + # installed it and the strict run is green without it. # Project rule, not a third-party check. Lives here rather than in # .git/hooks so it is versioned, shared, and runs in CI — a hook installed @@ -30,4 +37,16 @@ repos: entry: scripts/reject_coauthor_trailer.sh language: script stages: [commit-msg] + always_run: true + # SDK-3 (security audit 2026-11-06). The same script CI runs: a + # workflow that pins an action to a mutable ref, or installs a + # dependency outside uv.lock, fails here before it reaches a runner. + # ``always_run`` because the check reads every workflow, not only + # the ones the commit touched. + - id: check-supply-chain + name: check the supply chain pins + entry: python scripts/check_supply_chain.py + language: system + pass_filenames: false + stages: [pre-commit] always_run: true \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 219edc1..53217e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,765 @@ +## [0.24.0] - 2026-11-08 + +Minor release. The SDK had a shape it could not keep: the enforcement +path swallowed refusals. A gate that said "no" could be turned into a +`None`, and `None` is read by every caller as "yes"; a `/track` the +backend refused could be logged at DEBUG and returned as an ordinary +call; a LangChain callback dropped the refusal on the floor so the +agent got the call for free. Nine of the defects below are this one +defect seen from nine angles — an indeterminate gate answer, an +unbindable reservation id, a refused `/track`, a swallowed callback +refusal, an SDK-internal fault in the decorator — and each of them +ended the same way: the agent ran the call, the money was never +booked, and nobody was told. + +The second theme is the control plane. The approval push read the +UNSIGNED outer body — the bytes the HMAC never covered — and handed +them to the handler that releases a gate parked waiting for a human. +The WebSocket receiver verified a signature only when all four of +`signature`, `timestamp`, `api_key` and `secret_key` were present, so +a frame missing any one of them skipped verification entirely. Both +are closed: the privileged branch now requires an AUTHENTICATED frame +and reads the verified payload only. + +The third theme is honesty about the surface. The SDK's error-code +catalogue lagged the backend's registry by 34 codes; the blocked-family +dispatch arm tested the class by IDENTITY, so every typed SUBCLASS fell +through to a generic fallback that raised `TypeError` — nineteen codes +were flattened into a synthetic `{"decision": "block"}` dict. The +surface also advertised `DecisionSource.CACHED`, `cache_ttl` and +`cache_max_size` for a cache that no longer existed. All of it is now +either true or pinned. + +The release is 33 commits, 15 295 insertions and 1 213 deletions across +77 files, +644 tests. There is no wire-format change: every new error +code the SDK names is a code the backend already sends, and the one +place the SDK speaks a NEW code (`APPROVAL_TOOL_MISMATCH`, HTTP 409) +landed together with the backend half. + +### Migration + +Nine things differ from 0.23.0. The first is breaking; the second and +third change what an operator's configuration does; the rest are +behavioural changes that are visible only when something goes wrong, +which is precisely when they are visible. + +1. **`check_workflow_budget` lost its `use_gate_cache` parameter.** + The in-process gate cache is gone (ADR: SDK-1/2/3). A caller that + passed `use_gate_cache=False` now gets a `TypeError` — remove the + argument. There is no replacement: every call is its own `/gate` + call with its own reservation, and the backend's own idempotency + store (keyed by the SDK's `operation_id`) is the correct retry + dedup. A client-side cache that spans calls also spans principals, + which is why it was deleted rather than re-keyed. +2. **`NULLRUN_SENSITIVE_FAIL_OPEN` opens nothing on its own.** The + switch is still a whole-system switch, but what it may open is now + decided per tool by an explicit, code-level classification. The set + is EMPTY by default, so the switch alone opens nothing and every + tool stays fail-CLOSED: + + ```python + runtime.register_fail_open_tools(["charge_card"]) + runtime.remove_fail_open_tool("charge_card") + runtime.get_fail_open_tools() + runtime.is_fail_open_allowed("charge_card") + ``` + + Before this release, enabling the switch for one tool disabled + enforcement for EVERY tool. The classification is deliberately NOT + the SDK's built-in sensitivity registry: reusing that list would + make the SDK's own list of destructive operations an allowlist for + running without a policy decision. +3. **A gate refusal now escapes the LangChain / LangGraph callback.** + The callback used to classify refusals with a three-class + allow-list and drop everything else into a blanket `except + Exception` — so `TOOL_BLOCKED` and `RATE_LIMIT_EXCEEDED` were + swallowed and the agent carried on. The family is now named by BASE + class (`NullRunDecision`, `RateLimitError`, `WorkflowKilledInterrupt` + — `RateLimitError` is named explicitly because it is NOT a + `NullRunDecision`), the refusal is re-raised, and the handler sets + `raise_error = True` so langchain-core does not swallow it. A + cosmetic failure is still swallowed at DEBUG — only a refusal may + escape. ADR-008's fail-OPEN for a transport failure (the gate could + not be REACHED) is unchanged; "the gate could not be reached" and + "the gate said no" stay different categories. The canonical family + now lives in `nullrun.breaker.exceptions.ENFORCEMENT_EXCEPTIONS`; + the langchain module's private copy is gone. +4. **An indeterminate gate decision is now a REFUSAL, not a + permission.** When the gate call fails for a reason that is NOT a + transport failure — the SDK's own code faulted while asking — the + SDK raises `NullRunGateIndeterminateError` (NR-P004) instead of + failing open and returning `None`. `None` meant "no decision, carry + on", and every caller read it as ALLOW. The new class is an + infrastructure error, NOT a transport error, so ADR-126's two-step + opt-in cannot convert it into "the body runs". A new counter, + `gate_indeterminate_total`, records it — counting it as + `gate_fail_open_total` would send an operator to the backend + dashboard when the fault is in the SDK. +5. **The write-ahead log moved.** The default path is now + `/nullrun--.wal` — both scope tokens reduced to + SHA-256 digests, so the API key never reaches a path every user on + the host can list. The legacy `/nullrun.wal` is reported + and left alone: it is never read, never migrated and never deleted, + because its rows may belong to another tenant. The operator's CLI + lists every scoped WAL this user owns, and `replay` refuses to guess + between them. `NULLRUN_WAL_PATH` still wins outright, and it is + documented as the operator's responsibility to keep unique per + runtime. +6. **`api_key_prefix` now renders as a fingerprint.** The public field + keeps its legacy name; its VALUE changes. It is now the first eight + hex characters of the SHA-256 digest of the key, and the log label + changed from `api_key_prefix=` to `api_key_fp=`. A prefix of a + credential is the part an attacker brute-forces the rest from, so + three sites that each printed a different length of it are now one + helper. The eight characters are a prefix of the backend's own + `compute_fingerprint` (32 characters), so a client log line and a + backend row correlate on both sides without either side holding a + fragment of the secret. **Log parsers that matched the old raw + prefix must change.** +7. **The MCP tool-annotation cache window is 30 seconds, not 300.** + The annotations the MCP adapter stamps on `/check` are a POLICY + INPUT — the gate's `mcp_destructive_policy` and + `mcp_readonly_policy` umbrella policies are evaluated against them — + so a stale snapshot is a stale policy. The window is now the 30 + second floor the constructor already enforced, and an operator who + cannot accept any staleness passes `cache_seconds=0`. A FAILED + refresh no longer presents the previous snapshot to the gate at + all: the gate gets the unknown shape the adapter already uses for a + tool it has never seen, and falls back to its own classifier. The + deliberate cost: the default window buys ten `tools/list` calls + instead of one per five minutes. +8. **Three new public exceptions.** `NullRunGateIndeterminateError` + (NR-P004), `NullRunUnknownActionError` (NR-P005) and + `NullRunApprovalToolMismatchError` (NR-A017) are exported from the + package root and carry an end-user message in the catalogue. + `NullRunChainError` now covers the backend's `CHAIN_KEY_MISMATCH` + (403) instead of the refusal landing in the generic 403 auth + bucket. +9. **Six new metric counters.** `gate_indeterminate_total`, + `unknown_action_total`, `policy_gate_indeterminate_total`, + `ws_unverified_approval_total`, `ws_unverified_frame_total`, and + `dropped_llm_call_no_reservation` are exported in `to_dict()` — a + counter the operator cannot read is invisible. + +Two members of the public surface deliberately did NOT move: +`DecisionSource.CACHED`, `ExecuteConfig.cache_ttl` and +`ExecuteConfig.cache_max_size` stay, because removing them would break +importers for no safety gain. What was missing was a check that they +stay producer-less, so an AST walk over the package now proves that +nothing references `DecisionSource.CACHED`, that the string `"cached"` +appears exactly twice and both times as surface, and that nothing +reads `cache_ttl` or `cache_max_size`. The pin is textual and honest +about that: it proves the absence of a producer in THIS tree, not the +absence of a decision cache in some future version. + +### Fixed + +#### Money that was never booked + +- **SDK-1 (High) — the money path asked the wrong question.** `/gate` + RESERVEs the budget, the tool call runs and costs real money, then + `POST /api/v1/track` carries the CONSUME. `_route_track` wrapped + `track_single` in `except Exception`; a `NullRunDecision` was + re-raised, everything else was logged "event dropped" and the + function returned normally — so the caller returned `{"allowed": + True}` to the agent. The CONSUME never ran, the period-bound + counter never moved and no `cost_events` row existed: the spend was + free and the agent was told `allowed`. The taxonomy was the sharp + edge — the SDK decided open-vs-closed by exception CLASS and the + backend by error CODE, and `NullRunDecision` and + `NullRunInfrastructureError` are SIBLINGS, so + `isinstance(exc, NullRunDecision)` could never catch a backend + refusal. A later narrowing to three classes still left nine codes + outside it, among them the three `RATE_LIMIT_*` codes, and the + backend rate-limits `/track`. The rule now asks the WIRE one + question: did the backend answer? A request that never REACHED the + backend keeps ADR-008's fail-OPEN row, unchanged; a refusal raises, + and the auto-instrumentation layer stops swallowing it. The payload + is written to the v3 WAL before the raise, so a refusal is raised + AND recoverable. A determinate verdict is not persisted — it is the + backend's answer, and re-sending it produces the same answer. + (`a4fbed2`, +1090/-19; new test file `tests/test_v3_track_recovery.py` + — two existing tests changed contract: a 500 that used to be + swallowed now raises, and the "transport failure" mock became a real + `ConnectError`) +- **SDK-1/2/3 — the in-process gate cache made chain steps 2..N free.** + `runtime.py` kept a module-level `_GATE_CACHE` keyed + `(workflow_id, chain_id, call_model, estimated_tokens)` — a key with + no org id, no api-key id and no tool set, so the cached answer + belonged to no particular tenant or principal. It re-captured call + #1's server-minted `reservation_id`, so every chain step ran under + ONE execution id; `consume_v3.lua` answers `IDEMPOTENT_REPLAY` for + an already-consumed execution and books nothing — steps 2..N were + never billed. And the tool-block check never ran for a cached step. + The cache is deleted; every step is its own `/gate` call with its + own reservation. The same commit closed the edge branch of + `Transport.execute`, which returned a client-side `{"decision": + "allow"}` without ever asking the box: `EdgeTransport` now records + the box's answer per event id, `take_answer()` POPS it, and + `/execute` with no recorded answer returns `not_enforced()` — a + block carrying `EDGE_NOT_ENFORCED`. The third defect was the edge's + unreachable answer, which used `decision_source=FALLBACK` and was + therefore read as a degraded transport and failed OPEN — but ADR-067 + §3 requires deny, because the edge has no cloud to book the spend + through. The new `EDGE_UNREACHABLE` is not a fallback source and the + runtime honours the block. (`87a4c2b`, +846/-648) +- **SDK-9 — the box's answer cache was read and written without a + lock.** `EdgeTransport._answers` is shared by every thread of the + agent's tool calls, and the eviction walked it LIVE: + `self._answers[event_id] = answer` then `while len(...) > cap: + self._answers.pop(next(iter(self._answers)))`. `iter(d)` and `next()` + are two bytecodes apart, so a thread that resizes the dict in + between makes CPython raise `RuntimeError: dictionary changed size + during iteration`. That is more than a crash: the exception escapes + `enforce`, the gate's fail-OPEN arm catches it and returns "no + decision" — and "no decision" is read by the caller as "no block". + The shared dict is now read and written under a lock, and the + eviction picks its victim from a snapshot of the key list rather + than from a live iteration. The cap is now an EXACT promise: before, + two threads could both decide to evict and drop two entries for one + insert — and an answer evicted too early is a refusal, so an + over-eager eviction turns a paid-for call into a spurious block. + (`0964705`, +387/-4; new test file `tests/test_sdk9_shared_state.py`, + 6 cells — mutation-proved, and the first version of the file was + VACUOUS: 6/6 green against the unfixed code until the + `sys.setswitchinterval(1e-5)` fixture made the race land) +- **SDK-10 — a reservation id the SDK cannot bind is a REFUSAL.** The + capture dropped a `reservation_id` that is not a UUID with a warning + and returned `None`. That reads "the server reserved nothing" when + the truth is "the server reserved something I cannot bind" — so the + call proceeded with NO reservation, the `/track` event had no + `execution_id` to carry, the cost event was DROPPED, and the agent + got the call for free. The capture now refuses a present-but- + unusable id with `NullRunMalformedGateResponseError`; an ABSENT id + stays clear-and-`None`, because a block reserves nothing and that is + a fact, not a fault. The refusal also had to REACH the caller — the + instrumentation's blanket arms re-raise the enforcement family and + swallow everything else, so the class joins + `ENFORCEMENT_EXCEPTIONS`. ADR-008's fail-OPEN posture is untouched: + an unreachable gate still fails open; an ANSWER the SDK cannot use is + the opposite case. (`dc197a9`, +721/-44; new test file + `tests/test_sdk10_malformed_reservation.py`) +- **SDK-11 — an indeterminate gate decision is a REFUSAL, not a + permission.** The audit's words: "the same check is implemented twice + with different handlers... the fail-open path is the simpler one, so + it is the one users hit first." The two arms were the CACHED + branch's narrow arm and the UNCACHED branch's broad one, inside ONE + handler — so the narrow twin is gone and what is left is the BROAD + arm: a bare `except Exception` that logged, counted + `gate_fail_open_total`, and returned `None`. Three facts were + conflated and are now separated by TYPE: the gate NAMED a refusal → + honour it (the enforcement family is re-raised FIRST, and the + ordering is load-bearing — `RateLimitError` IS a + `NullRunTransportError`, so a later transport arm would swallow the + 429 and fail OPEN on it); the gate could not be REACHED → ADR-008's + fail-OPEN, and only that; anything else → the SDK's OWN code + faulted while asking, so the decision is INDETERMINATE, and an + unanswered question is not a yes → `NullRunGateIndeterminateError` + (NR-P004). (`9dcb22d`, +884/-10; 21 new tests) +- **SDK-6 — a refused `/track` must reach the application.** The + runtime already re-raised a `NullRunDecision` when the backend + refused to book the money (CONSUME_OVERBUDGET 422, a budget block + 402). The auto-instrumentation layer then caught that refusal with a + blanket `except Exception` and logged it at DEBUG. The application + saw a perfectly ordinary LLM call while the backend had just refused + to book it: a refused track looked like a SUCCESS. One family, named + by BASE class, now lives in `nullrun.breaker.exceptions`, and every + billable emission site the layer installs re-raises that family + BEFORE the broad arm — the httpx sync and async transports, the + OpenAI-Agents tracer, the requests transport, the crewai / autogen / + llama_index integrations, and the LangChain callback's `on_llm_end` + (whose inner blanket arm had defeated the SDK-4 decorator, so the + refusal never even reached it). The fail-OPEN posture is UNCHANGED + for everything else. (`22f8d77`, +876/-12; 20 tests) +- **SDK-4 (callback) — a gate refusal leaves the callback; only a + refusal may escape.** The LangChain callback classified a gate + refusal with a three-class allow-list, so every OTHER refusal — + `TOOL_BLOCKED`, `RATE_LIMIT_EXCEEDED` — fell into the broad arm, was + logged at DEBUG and dropped. The family is named by BASE class so a + refusal subclass added later cannot rot back into the fail-OPEN arm, + the refusal is re-raised, the handler sets `raise_error = True` + (langchain-core swallows a handler's exception by default and + re-raises it only when the flag is set — in BOTH the sync and the + async dispatch path), and `_only_a_gate_refusal_escapes` guards + every callback method. The deferred stash stays as the SECONDARY + channel: the one that survives a host framework which ignores + `raise_error`, and the one that can abort on the `@protect` + boundary's own thread. ADR-122. (`28f1b00`, +490/-44) +- **SDK-16 — the callback obeys the same fail-OPEN/CLOSED table as + `@protect`.** The LangChain/LangGraph callback swallowed every + refusal the runtime raises on purpose: + `NullRunAuthenticationError`, `NullRunUnclassifiedRefusalError` and + `NullRunRateLimitRedisError` fell into its broad arm. The same + refusal stopped the call through the `@protect` boundary and + vanished through the callback, so the LLM call ran with no + reservation and its cost event was dropped: the agent got the call + for free. The callback now classifies like the `@protect` boundary — + the family raises, a transport failure fails OPEN (ADR-008), and + every other fault fails CLOSED. The transport arm comes FIRST, + because a transport failure is also an infrastructure error and the + order decides whether a dead backend freezes the agent. The + `/track` path is untouched. (`1bb76dd`, +522/-13) +- **SDK-13 — an SDK-internal fault in the `@protect` gate is a + REFUSAL.** The decorator's catch-all turned a fault in the SDK's OWN + code into a refusal that wore the network code NR-B001 and the + policy-block type; the FastAPI integration mapped that through the + decision table and rendered HTTP 429 / category "decision" — the + operator was told the policy refused when the truth is that the SDK + has a bug. Under the two-step opt-out the same arm RETURNED, so the + body ran with no policy decision at all. The catch-all now raises + `NullRunGateIndeterminateError` (NR-P004) — the `/execute` twin of + ADR-129's `/gate` class — and the two-step opt-out no longer opens + an SDK-internal fault: what reaches the catch-all is by construction + NOT a transport failure, because `/execute` converts every transport + failure into `NullRunTransportError`, which the arm above owns. The + refusal is counted (`policy_gate_indeterminate_total`, a counter of + its own — it is an SDK incident, not a backend outage), logged at + ERROR with the tool name, and carries the original exception as its + cause. Three counters that were counted but never exported in + `to_dict()` are now exported. (`ac62e10`, +561/-20; 17 new tests) +- **SDK-8 — the fail-open switch is bounded per tool.** + `NULLRUN_SENSITIVE_FAIL_OPEN` was one global boolean, read in + exactly ONE place, and turning it on opened EVERY protected tool: + "enabling it for one tool disables enforcement for every tool". The + switch is still a whole-system switch, but what it may open is now + decided per tool by an explicit, code-level classification — an env + var can be inherited by a deploy pipeline or a CI environment, a + code-level classification cannot. The set is EMPTY by default. The + authoritative fail-OPEN/CLOSED table also named a gate + (`_enforce_sensitive_tool`) that no longer exists, and claimed the + env switch was the opt-out for the permissive fallback — false, the + switch is never read there. The table is split into the `@protect` + path and the `/execute` path, and the README's "What does fail open" + section moves with it. The MCP path is unchanged and still fails + CLOSED through `fallback_mode=STRICT`. (`b519481`, +635/-28) +- **P2-18 — every MCP tool call runs its own `/gate` pre-flight.** + `MCPAdapter.call_tool` ran `runtime.check_workflow_budget()` only + when `get_server_minted_execution_id()` was `None`. That contextvar + is set by `/gate` and by `/execute` and is NEVER reset in `src/`, so + the second and later MCP tool calls in one scope ran NO `/gate`: no + budget check, no reservation, no spend reserved for those calls. Both + `/execute` bodies also carried the FIRST call's execution id, so the + second call settled against a reservation that was already spent. + The condition cannot simply become "always pre-flight" — two cases + legitimately skip it, and both are about THIS action: the approval + re-entry (the backend's `require_approval` arm echoes the approval + row's execution id into `reservation_id`, so the ambient id IS the + approval's execution) and the `@protect` scope (which reserved for + itself, and a second `/gate` inside it would reserve twice and + orphan the first, NR-059). So the pre-flight runs unless the ambient + reservation is known to cover THIS action: `approval_id is not None` + or an envelope was already in scope. The envelope marker is read + BEFORE the adapter installs its own envelope — otherwise every call + would read as "already gated". (`1b9ef6d`, +445/-17; new test file + `tests/test_p2_18_mcp_preflight_scope.py`) +- **SDK-7 — the MCP adapter derives ONE envelope from the tool + call.** The MCP path went to `/execute` with NO `business_impact` and + NO `action_digest`. That is not "the approval binds the tool name but + not the arguments" — on the MCP path the approval could not be spent + at all: `/execute` with an `approval_id` runs Step 9.5, which + SERVER-DERIVES the action digest from the live `business_impact`, and + with no envelope the derive returns `Ok(None)` and the orchestrator + fails CLOSED with BUSINESS_IMPACT_INVALID (NR-010). The row was + created, the operator approved it, and the re-execute was refused. + The adapter now derives one envelope from the call it is about to + make — a tool_call impact carrying the tool name and the EXACT + argument dict it forwards as the canonical bag — plus that envelope's + digest, and puts both on the wire. The params MUST be that same + dict: the backend compares `BusinessImpact::ToolCall.params` with the + live `tool_arguments` canonically, so a masked or reshaped copy + would bind the operator's approval to arguments nobody approved. The + envelope is installed on the call context BEFORE the pre-flight + `/gate` runs, so the two hops hash the SAME bytes (ADR-065), and it + is released in a `finally` so it cannot leak into the NEXT action. + When no envelope can be built (an f64 argument, a non-ASCII tool + name) the digest still goes on the wire over the `no_impact` sentinel + and the envelope stays ABSENT, so the backend takes its fail-CLOSED + arm ("cannot recompute") instead of the legacy "no digest at all" + arm, which is a grant bound to nothing. (`776eb01`, +686/-7; 20 new + behavioural tests, the digest pinned against the BACKEND's golden + hex so the two repos cannot drift apart here without this going red) +- **P2-17 — the edge refusal reports its OWN `last_source`.** + `EdgeTransport.last_source` was the single constant `EDGE_LEASE` on + BOTH the allow path and the refusal path, so the two collapsed into + one string and a caller reading the attribute could not tell which + way the box went: it said "the lease spoke" when the truth was "the + lease said no". The new named marker `EDGE_LEASE_REFUSED` is a + `last_source` marker only and never reaches the wire — the refusal's + `decision_source` stays `EDGE_LEASE`, because the lease was the + authority that spoke either way and `decision` says which way it + went. Changing it would need a new name in the runtime's known-source + set, and until the runtime knows it, a body carrying it is a + MALFORMED response: a clean refusal would become a crash. The new + marker is also NOT a fallback source — a box that said no is a + VERDICT, not a degraded transport. (`946a3a2`, +261/-1; new test file + `tests/test_p2_17_edge_last_source.py`, 9 cells, mutation-proved) +- **SDK-4 (WAL) — the write-ahead log is scoped per user and per + key.** The WAL path was `os.path.join(tempfile.gettempdir(), + "nullrun.wal")` — the same string for every user and every org on + the host — and `_persist_to_wal` writes with a tmp file, an fsync + and an `os.replace` that puts only this runtime's buffer in place of + the whole file. Two runtimes on one file do not queue: the second one + destroys the first one's undelivered events, so spend that already + happened is never booked, and the flush path's availability is + coupled across tenants. The default is now + `/nullrun--.wal`, both scope tokens reduced to + SHA-256 digests. The path is resolved once, in the constructor, and + never moves: the org id is only known after authentication, and a + path that changed there would orphan the file written before it. One + API key belongs to exactly one org, so a key-scoped WAL is an + org-scoped WAL in money terms. (`90bc05a`, +787/-20; new test file + `tests/test_wal_scope.py`) + +#### The control plane + +- **SDK-5 (approval push) — the approval push reads the VERIFIED + payload, not the outer body.** The control-plane WebSocket HMAC + covers the bytes inside `signed_payload`, NOT the flattened outer wire + body. The outer body is therefore writable by anyone who can inject + bytes into the socket: they rewrite `type`, `outcome` and + `execution_id` while keeping a captured `(signed_payload, + signature)` pair — and the signature check still PASSES, because it + verifies bytes the attacker never touched. The `approval_resolved` + branch was the one dispatch branch that read the outer body, and it + hands that dict to `on_approval_resolved`, which pops the pending + approval and sets its threading.Event — releasing a gate that was + parked waiting for a human. The branch type and every field are now + read from the verified payload; the outer body is a fallback only + for frames that could not be authenticated at all. A signed frame + whose payload does not parse is DROPPED, because the outer body is + not what was signed. The privileged branch requires an AUTHENTICATED + frame: an unauthenticated approval is refused, counted + (`ws_unverified_approval_total`) and logged at ERROR, and the waiter + stays parked and fails closed on the approval timeout. The safety + direction is unchanged: an unsigned state_change (kill / pause) is + still delivered, so the kill switch is not weakened. (`c4e8c4a`, + +698/-44; 13 tests, five mutations each red for its own reason) +- **P2-19 — a control-plane frame the client COULD have verified is + refused.** The WebSocket receiver verified the signature only when + all four of `signature`, `timestamp`, `api_key` and `secret_key` were + present. A frame missing any one of them skipped the verification + block entirely and was then read from its outer body — the body the + server never signed. A client that holds the API key and the HMAC + secret can verify EVERY frame the server sends it: the server signs + every frame it sends for such a key. A frame that arrives at such a + client unsigned — or with a signature but no timestamp — did not come + from the server, so it is refused before any branch reads a field out + of it. The refusal is counted (`ws_unverified_frame_total`) and + logged. An approval frame keeps bumping + `ws_unverified_approval_total`, so the operator still sees that + somebody pushed an approval this client cannot verify. A client + WITHOUT a secret (a legacy key) can verify nothing, so the unsigned + path stays open for it — the kill switch must keep working there. + A refused frame also cannot poison the per-workflow version dedup, so + a forged frame cannot silence the next real kill. (`2a79524`, +312; + 7 new cells driving the real `_handle_message`) +- **P2-19 (log) — the refusal log names the refused frame's code.** + The server's own refusal frame is unsigned BY DESIGN: it is sent + before the connection is authenticated, so there is no secret to sign + it with yet. The refusal dropped such a frame, so the operator saw + the refusal but not the server's reason. The log now names the + frame's code as well as its type, read from the UNSIGNED outer body + — so it is a hint, not a source of truth: the log line says the frame + was refused, and the code only tells the operator what the server + claimed. (`b0f5645`, +30/-1) +- **SDK-12 — an unknown control-plane action is REFUSED, not silently + dropped.** The action dispatcher logged an unknown action type at + ERROR, recorded a forensic history event, and RETURNED. The action + was then neither executed nor refused, while the caller's own + bookkeeping said it had been handled: a control-plane instruction + that is silently dropped is a control-plane instruction that does not + exist. An unknown action type now RAISES + `NullRunUnknownActionError` (NR-P005) — an infrastructure error, NOT + a block, so B14's anti-DoS property survives: the workflow keeps + running, the action does not. The refusal is not a transport error, + so ADR-126's two-step opt-in cannot convert it into "the body runs". + `except BaseException` was narrowed to `except Exception`, so a + Ctrl-C or a SystemExit reaches the caller again. The handler's own + refusal (kill / pause / block) is now logged at INFO, not ERROR: a + delivered kill is the action working, not a fault (every server-side + kill used to raise a false alarm). (`5d1e5e0`, +756/-57; 22 + new/rewritten tests) +- **DEC-5 / DEC-7 — a money approval must bind a tool.** The backend + now refuses an approval consume whose live tool is not a member of the + tool set the approval covers. Before this, a money approval bound + only the AMOUNT — `BusinessImpact::Money` carries no tool name and + `tool_digest` is NULL for non-MCP tools — so an approval for tool A + authorised tool B. The new wire code `APPROVAL_TOOL_MISMATCH` + (HTTP 409) dispatches to `NullRunApprovalToolMismatchError` + (NR-A017). 409, not 403: the approval is valid and unconsumed, it + just covers a different tool, so the remedy is a FRESH approval + rather than a retry. (`5adc309`, +124/-4) +- **ADR-180 — the chain-key refusal carries the chain class.** The + backend refuses a chain presented by an API key of the same + organization with `CHAIN_KEY_MISMATCH` (403). The SDK had no entry + for it: the dispatcher fell through to the 403 auth bucket, so a + caller branching on `NullRunChainError` never caught it. The code is + now in the typed catalog and in the envelope parser, and the + frozen-catalog test carries it so the entry cannot be dropped again. + (`2d19f45`, +35) + +#### The surface says what the SDK does + +- **SDK-15 — the error-code catalog names a class the dispatcher can + build.** The SDK's v3 error-code catalog lagged the backend's + registry by 34 codes, and the parity guard in the backend's + `nr007_sdk_error_code_parity.rs` could not see the drift because it + was wired into no workflow. Every missing code fell through to the + status-based fallback, so the refusal arrived as a generic + `NullRunBackendError` (or, on a 4xx, as an authentication error) with + no diagnostic class at all. Two defects, one symptom: the catalog + was incomplete (34 codes added, each mapped by its WIRE BUCKET — the + bucket the backend's own registry gives the code — so the SDK class + and the backend status cannot disagree), and the catalog named + classes the dispatcher could not build. The blocked-family arm + tested the class by IDENTITY, so every typed SUBCLASS of + `NullRunBlockedException` fell through to the generic fallback, which + calls `catalog(full_message, **forwarded)` against a constructor that + requires `workflow_id` and `reason`: a `TypeError`, which escaped + the parser into the transport's catch-all and surfaced as a + synthetic `{"decision": "block"}` dict. Nineteen codes were + flattened that way (eleven approval, eight MCP). The arm now + matches by SUBCLASS, and two dispatch arms were added for the + families whose constructors the generic fallback cannot satisfy: + `RateLimitError` (requires source + endpoint) and + `NullRunChainError` (drops `chain_id` / `backend_code`). The new + test holds the property over the WHOLE catalog: for every code, the + real dispatcher returns an instance of the catalog's class, for every + status — and it pins the 34 codes by NAME, because a removal is + invisible to a test parametrized over the map's own keys. (`e1ec87c`, + +518/-65; mutation-proved — reverting the arm to the identity test + turns 35 cells red) +- **SDK minor 2 — the SDK has no decision cache, and its surface says + it has one.** `DecisionSource.CACHED` and `ExecuteConfig.cache_ttl` / + `cache_max_size` sat in the public surface with no producer anywhere + in `src/`. The surface reads as a live decision cache, and that is + the shape of NR-116, which was closed. A future "restore the cache" + change would land green, and a reader would assume a decision can be + reused. The members stay — they are public and removing them breaks + importers for no safety gain — but what was missing was a check that + they stay producer-less, so an AST walk over the package now proves + it. (`c78ef32`, +216/-2; new test file + `tests/test_decision_cache_surface_has_no_producer.py`) +- **SDK-14 — the suite must drive the PRODUCTION enforcement family.** + The SDK's own tests re-implemented the production predicate: + `tests/test_langchain_enforcement.py` spelled out its own + `ENFORCEMENT` tuple and asserted `issubclass` against the production + family — one direction only. Measured on the pristine tree: with the + production family WIDENED by one member, the whole suite stayed + GREEN. The production predicate changed and the suite noticed + nothing — the hole, measured, not argued. The hand-written tuple is + gone; `_CASE_BUILDERS` holds CONSTRUCTORS; and + `TestTheFamilyAndTheCasesCorrespond` compares the production family + with the cases actually driven, in BOTH directions — a family member + nobody drives is red, and a case the family no longer covers is red. + The instrumentation's alias must BE the production tuple, so its + `except ImportError` fallback (an empty tuple = every refusal + downgraded to a transport error and swallowed) can never be in + effect. (`4d386bb`, +265/-31; mutation-proved across five mutations) +- **SDK-9 (test) — pin the money direction: a refusal is never a + failed-open.** The SDK-9 fix was committed with a docstring that + named the wrong consequence. The measurement says the opposite: the + crash lands in the gate's bare `except Exception` fail-OPEN arm, so a + REFUSED box becomes "nothing was decided" and the caller PROCEEDS. + Measured on the real LLM-span path (96 threads × 6 calls, a box that + refuses every time): 35–43 of 576 LLM calls proceeded on a refused + lease before the fix, 0 after. Two tests added — one pinning the + money invariant directly, one pinning the eviction's ORDER, because + evicting the newest instead of the oldest passed every other test in + the file and the order decides whether a live answer is dropped + before its `/execute` half consumes it. (`0d1186e`, +134/-5) + +#### Secrets, environment, and the supply chain + +- **SDK minor 3 — redaction follows the credential, not only the + argument name.** The SDK redacted arguments by NAME. + `SENSITIVE_ARG_KEYS` knows which argument names someone listed; it + cannot catch a secret that arrives under a name nobody listed: + `{"note": ""}` printed the key in full. A value-shaped pass + now runs beside it. `scrub_secret_values` matches the credential + itself, whatever key it travels under: this product's own `nr_live_` / + `nr_test_` keys, a JSON Web Token, an AWS access key id, an + OpenAI-style `sk-` key, a PEM private key block, and an HTTP bearer + credential. The patterns are narrow on purpose — a shape that cannot + be told apart from ordinary data is a guess, and a guess that scrubs + real data is a bug of its own, so the near misses are test cases: + `risk-assessment-report` and `task-abcdef0123456789abcdef0123456789` + survive, and so does ordinary SQL. Most of the shapes carry no + leading word boundary, because a boundary would make the pattern miss + a credential that sits right after a letter — the one exception is + `sk-`, whose boundary keeps a hyphenated word out of the scrub. The + pass runs BEFORE the truncation, so a credential that straddles the + cut is caught whole. (`1ad95d0`, +154; new test file + `tests/test_value_level_redaction.py`) +- **SDK-5 (fingerprint) — no character of the API key reaches a log + line.** Three places printed a fragment of the key: the status + snapshot (`self.api_key[:10]`), the runtime's error context (the same + ten characters) and the transport's debug log (`api_key[:8] + "***"`). + A prefix of a credential is the part an attacker brute-forces the + rest from, and because the three places disagreed about how much they + printed, "the full key never leaves the SDK" was not a statement + anyone could check. One helper now answers it: + `api_key_fingerprint` returns the first eight hex characters of the + SHA-256 digest of the key, or `None` for a missing key, and all + three sites call it, so the disagreement is gone by construction. + (`66de8f5`, +233/-21; new test file + `tests/test_api_key_fingerprint.py`) +- **SDK-2 — the production detector is not a substring test.** + `_is_production_environment` decided whether the two fail-open guards + arm by asking one question: is any marker a SUBSTRING of the + hostname? `"test"` is a substring of `latest`, `contest` and + `attest`; `"localhost"` is a substring of `notlocalhost.example.com`. + So a host that merely CONTAINED a dev marker was classified non-prod. + Worse, the operator's `NULLRUN_ENV=production` was suppressed by the + `and not any(marker in host_lc ...)` clause, and + `NULLRUN_SKIP_BUDGET_CHECK=1` then bypassed the budget gate, the rate + limit and the tool-block check with a WARNING rather than a refusal. + Four ordered rules now: an exact loopback or a `.localhost` suffix is + dev; the host's FIRST LABEL being `staging` or `test` is dev; the + host being exactly `api.nullrun.io` is prod; and + `NULLRUN_ENV=production` or `prod` is prod for every other host, so + the operator's declaration is no longer vetoed by a spelling. Rule 2 + compares label by label, never as a substring: `staging.internal` is + dev and `latest.nullrun.io` is not. (`249df35`, +283/-28; new test + file `tests/test_production_environment_detection.py`) +- **SDK minor 1 — a failed `tools/list` refresh is not a policy + input.** The MCP adapter caches the tool annotations it stamps on + `/check`, and the gate's `mcp_destructive_policy` and + `mcp_readonly_policy` umbrella policies are evaluated against them, + so the cache is a policy input and not a convenience. The window was + 300 seconds. A FAILED refresh was worse than the window: the handler + kept the previous snapshot and the adapter kept presenting it as the + server's current answer, so an upstream that went down left the + client stamping a stale `destructive=False` with no bound on the + staleness at all. The window is 30 seconds now, the floor the + constructor already enforced, and a failed refresh no longer presents + the snapshot to the gate: the gate gets the unknown shape the adapter + already uses for a tool it has never seen, and falls back to its own + classifier. (`6ab5a2c`, +321/-18; new test file + `tests/test_mcp_annotation_staleness.py`) +- **SDK-3 — the CI installs from the lock and pins every action.** + `uv.lock` existed and pinned the dependencies, but no workflow + consumed it: all four install sites ran `pip install -e ".[dev]"`, + and the publish job ran `pip install --upgrade build`. The build + backend was unpinned, and it is not part of uv's resolution, so no + lock-based job could see it. The publish job holds the PyPI Trusted + Publishing token (`permissions: id-token: write`), and EVERY `uses:` + in it named a branch or a tag: a moved tag executes in the job that + can mint an upload token, and the wheel it produces carries the + payload to every `pip install nullrun`. What changed: every `uses:` + names a 40-hex commit SHA (26 references, six actions, the old tag + kept in a trailing comment); the build backend is pinned exactly + (`hatchling==1.32.4`); every install comes from the lock + (`uv sync --frozen --extra dev`, and `--frozen` fails the job when + the lock and `pyproject.toml` disagree); and the publish tooling + moves into a PEP 735 `[dependency-groups]` entry resolved from the + lock, which carries the hashes. `scripts/check_supply_chain.py` lints + the workflows — it runs inside the token-holding job, before the + build and the upload, plus a local pre-commit hook and a `make` + target — and it has no third-party dependency: PyYAML when it is + importable, a text scanner otherwise, and the two scanners are proven + to agree. The pre-commit chain could not start at all: it pointed at + a mirror of `pre-commit/mypy` that no longer exists, and pre-commit + installs every repository in the file before it runs any hook, so the + whole run aborted and no hook in the file executed. The dead + reference is gone; the mypy check is not lost, because CI runs + `mypy src/` and `make type-check` runs `mypy src/nullrun --strict`. + (`06e8fce`, +1988/-83; new test file `tests/test_supply_chain_lint.py`) + +### Cleanup + +- **Redundant quotes dropped** — `ruff check src tests` was RED at the + SDK-7 commit: UP037 twice in `toolbox/mcp.py`. The quotes are + unnecessary — the module has `from __future__ import annotations` and + imports `BusinessImpact` at runtime — so removing them is a no-op at + runtime and turns the SDK's own lint gate green again. A red gate in + the tree is a red gate for the next release too. (`fc1666e`, +2/-2) +- **Docstrings that named a module that no longer exists** — the + error-envelope notes pointed at `contracts/errors.ts`, which was + deleted in the backend repo (WIRE-8 / ADR-113): a fourth error + dialect with zero importers, 16 lowercase-slug codes with 0 overlap + against the 116 machine codes, a misspelled `seats_limit_reached`, + and a status map wrong for 400 and 422. The envelope this module + actually parses is the control-plane `ApiErrorResponse`. Docstrings + only, no behaviour change. (`db7e776`, +4/-3) +- **NR-P004 documented on both seams** — the error-code catalogue + described NR-P004 as a `/gate`-only refusal. After SDK-13 the same + class is raised on the `/execute` seam, and the page would have been + stale: it named only `gate_indeterminate_total`, and it did not say + that the `/execute` seam used to wear NR-B001 and a policy-decision + type. Doc-only. (`f49aa7c`, +25/-15) + +### Tooling + +- **The retry clock is a seam, not the global `time.sleep`** — `time.sleep` + is ONE attribute on ONE module object, and the suite patched that + attribute in three places (the conftest cap, plus the same attribute + again through `nullrun.transport.time` and + `nullrun.breaker.circuit_breaker.time`, which ARE the stdlib module + — measured: `nullrun.transport.time is time` is True). A patch on + that attribute captures the sleep of EVERY thread in the process, so + the webhook-delivery singleton's 0.5 s idle poll landed in a + retry-delay list and broke the floor assertion for a reason that has + nothing to do with the retry path. The transport now has a `_sleep` + seam next to `_wait_on`, and the retry tests patch the seam instead + of the global. The conftest keeps ONE patch site for the global (the + 1 ms cap) and stops repeating it through the module aliases. + (`e572403`, +82/-23; mutation-proved — a foreign 0.5 s in the delays + was red before the fix, and 8 cells turn red when the seam is + bypassed) +- **A new supply-chain linter for the workflows** — see SDK-3 above. + `scripts/check_supply_chain.py` has no third-party dependency and + runs in CI, in the pre-commit chain and as a `make` target. + +### Verification + +| Check | Result | +|---|---| +| `ruff check src tests` | All checks passed | +| `mypy src/nullrun` | 5 errors, all in `src/nullrun/instrumentation/auto.py` — **identical on a pristine `origin/master` checkout in the same interpreter** (measured: 5 errors in 39 files there, 5 in 41 here). Pre-existing, not introduced by this diff. They are `attr-defined` / `call-overload` against `langgraph`'s `Pregel` overloads; `langgraph` is **not** in `uv.lock`, so CI's `mypy src/` (which installs from the lock) never imports it and sees none of them. | +| `pytest -q` | **2592 passed, 12 skipped** in 82.18s (local baseline on `origin/master` in the same interpreter: **1948 passed, 12 skipped** — **+644 new tests**) | +| Scratch diff | clean (`dist_local/`, `*.defect*` absent — both already in `.gitignore`) | +| `nullrun.__version__` | `0.24.0` | +| Wire-format compatibility | unchanged from 0.23.0 for the SDK→backend contract. The SDK's error-code catalog grew by 34 entries, each mapped to a code the backend already sends. One code is NEW in both repos and landed together: `APPROVAL_TOOL_MISMATCH` (HTTP 409). The SDK's own request payloads are unchanged — the MCP envelope puts `business_impact` + `action_digest` on the wire, which are fields the backend already resolves. | + +### Commits included + +``` +1ad95d0 fix(sdk): redaction follows the credential, not only the argument name (SDK minor 3) +c78ef32 fix(sdk): the SDK has no decision cache, and its surface says it has one (SDK minor 2) +6ab5a2c fix(sdk): a failed tools/list refresh is not a policy input (SDK minor 1) +a4fbed2 fix(sdk): the money path asks one question: did the backend answer? (SDK-1) +90bc05a fix(sdk): the write-ahead log is scoped per user and per key (SDK-4) +249df35 fix(sdk): the production detector is not a substring test (SDK-2) +66de8f5 fix(sdk): no character of the API key reaches a log line (SDK-5) +06e8fce build(sdk): the CI installs from the lock and pins every action (SDK-3) +2d19f45 fix(sdk): the chain-key refusal carries the chain class (ADR-180) +1bb76dd fix(sdk): the callback obeys the same fail-OPEN/CLOSED table as @protect (SDK-16) +e1ec87c fix(sdk): the error-code catalog names a class the dispatcher can build (SDK-15) +e572403 fix(sdk): the retry clock is a seam, not the global time.sleep (P3-10) +b0f5645 fix(sdk): name the refused frame's code in the log (P2-19) +2a79524 fix(sdk): refuse a control-plane frame the client could have verified (P2-19) +1b9ef6d fix(sdk): every MCP tool call runs its own /gate pre-flight (P2-18) +946a3a2 fix(sdk): the edge refusal reports its OWN last_source (P2-17) +4d386bb test(sdk): the suite must drive the PRODUCTION enforcement family (SDK-14) +f49aa7c docs(sdk): NR-P004 covers the /execute seam too (SDK-13) +ac62e10 fix(sdk): an SDK-internal fault in the @protect gate is a REFUSAL (SDK-13) +5d1e5e0 fix(sdk): an unknown control-plane action is REFUSED, not silently dropped (SDK-12) +9dcb22d fix(sdk): an indeterminate gate decision is a REFUSAL, not a permission (SDK-11) +dc197a9 fix(sdk): a reservation id the SDK cannot bind is a REFUSAL (SDK-10) +0d1186e test(sdk9): pin the money direction — a refusal is never a failed-open +0964705 fix(sdk): the box's answer cache is read and written under a lock (SDK-9) +b519481 fix(sdk): the fail-open switch is bounded per tool (SDK-8) +fc1666e style(sdk): drop the redundant quotes ruff flags (SDK-7 leftover) +776eb01 fix(sdk): the MCP adapter derives one envelope from the tool call (SDK-7) +22f8d77 fix(sdk): a refused /track must reach the application (SDK-6) +c4e8c4a fix(sdk): the approval push reads the VERIFIED payload, not the outer body (SDK-5) +28f1b00 fix(sdk): a gate refusal leaves the callback — only a refusal may escape (SDK-4) +db7e776 docs(sdk): point the error-envelope notes at the real catalogue +5adc309 fix(sdk): APPROVAL_TOOL_MISMATCH — a money approval must bind a tool (DEC-5/DEC-7) +87a4c2b fix(sdk): no in-process gate cache, edge /execute never invents an allow (SDK-1/2/3) +``` + ## [0.23.0] - 2026-10-29 Minor release. Two enforcement paths that could not complete their own diff --git a/Makefile b/Makefile index e567cd6..02939b7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install test lint type-check coverage clean build publish-test publish smoke-test +.PHONY: install test lint type-check coverage clean build publish-test publish smoke-test lint-supply-chain # ── Setup ───────────────────────────────────────────────────── install: @@ -45,23 +45,34 @@ lint-fix: type-check: mypy src/nullrun --strict -check: lint type-check test +check: lint lint-supply-chain type-check test + +# ── Supply chain ────────────────────────────────────────────── +# SDK-3 (security audit 2026-11-06). Fails on a workflow that pins an +# action to a mutable ref, and on a dependency install that ignores +# uv.lock. CI runs the same script (.github/workflows/ci.yml, job +# ``supply-chain``) and the publish workflows run it inside the job that +# holds the PyPI Trusted Publishing token. +lint-supply-chain: + python scripts/check_supply_chain.py # ── Build & Publish ─────────────────────────────────────────── clean: rm -rf dist/ build/ *.egg-info htmlcov/ .coverage coverage.xml +# ``build`` and ``twine`` come from the ``publish`` dependency group in +# uv.lock, so the local publish path resolves the same versions as CI. +# ``--inexact`` keeps the dev packages that are already in the venv. build: clean - pip install build - python -m build - pip install twine - twine check dist/* + uv sync --frozen --only-group publish --inexact + uv run --frozen --no-sync python -m build + uv run --frozen --no-sync twine check dist/* publish-test: build - twine upload --repository testpypi dist/* + uv run --frozen --no-sync twine upload --repository testpypi dist/* publish: build - twine upload dist/* + uv run --frozen --no-sync twine upload dist/* # ── Dev helpers ─────────────────────────────────────────────── smoke-test: build diff --git a/README.md b/README.md index fe91381..952fd6d 100644 --- a/README.md +++ b/README.md @@ -469,7 +469,9 @@ An execution is bound server-side for **24 hours**. Reservations expire on a TTL The part that costs money: the spend is real and the period-bound counter did not account for it. **The control center under-reports what you spent by exactly the amount buffered during the outage, and the DLQ is the record of the gap.** If your agents can run unattended for more than a day against an unreachable backend, budget on the assumption that it is spent and unaccounted, not pending. Full details, including how to inspect and replay the DLQ: [docs/wal-and-dlq.md](docs/wal-and-dlq.md). ### What does fail open -Fail-open here is narrow and deliberate, and the authoritative table lives in `runtime.py` (ADR-008). In short: a **transport** failure on the check path is open, so an unreachable backend cannot freeze your agent; a **wire response that names an enforcement failure** is closed, because the backend made a decision and the SDK will not overrule it; a **body that is not a verdict at all** — no `decision`, no `decision_source`, or an unrecognised value in either — is closed, because something answered and what it said was not a decision, and reading that as permission would let a non-NullRun responder authorise a call no policy engine evaluated; a **401** is closed, because no retry fixes a revoked key; and the `/execute` path is closed by default (`FallbackMode.STRICT`). +Fail-open here is narrow and deliberate, and the authoritative table lives in `runtime.py` (ADR-063 §2.4). In short: a **transport** failure on the check path is open, so an unreachable backend cannot freeze your agent; a **wire response that names an enforcement failure** is closed, because the backend made a decision and the SDK will not overrule it; a **body that is not a verdict at all** — no `decision`, no `decision_source`, or an unrecognised value in either — is closed, because something answered and what it said was not a decision, and reading that as permission would let a non-NullRun responder authorise a call no policy engine evaluated; a **401** is closed, because no retry fixes a revoked key; on the money path (`/track`) the same rule runs in the other direction — an **answer that refused the consume** is closed even when its body names no enforcement failure, because the spend is real and the backend did not book it, and a request that never reached the backend is written to a durable WAL and replayed, so it is neither booked nor dropped; and the `/execute` path is closed by default (`FallbackMode.STRICT`). + +**The one escape hatch is bounded per tool.** `NULLRUN_SENSITIVE_FAIL_OPEN=1` lets a protected tool's body run while the policy engine is unreachable — a dev/test switch, refused in production unless you also set `NULLRUN_ALLOW_SENSITIVE_FAIL_OPEN=1`. It is a whole-system switch, so what it is allowed to open is decided per tool: `runtime.register_fail_open_tools(["charge_card"])`. The classification is empty by default, so **the switch alone opens nothing** — a tool that is not classified stays fail-closed even with the variable set, and the refusal says so. This is deliberately NOT the sensitivity registry (`is_sensitive_tool`): that one answers "is this tool destructive?", and a destructive tool is the last thing that should be allowed to run unchecked. --- diff --git a/docs/errors/NR-P004.md b/docs/errors/NR-P004.md new file mode 100644 index 0000000..3559b4a --- /dev/null +++ b/docs/errors/NR-P004.md @@ -0,0 +1,86 @@ +# NR-P004 — Indeterminate decision + +| Field | Value | +|---|---| +| **Code** | `NR-P004` | +| **Category** | **P**rotocol | +| **Exception class** | `NullRunGateIndeterminateError` | +| **Retryable** | Yes | +| **Default `user_action`** | "The SDK could not obtain a decision from the NullRun gate: the call failed for a reason that is not a network failure, so the SDK stopped rather than assume the call was allowed. This is usually an SDK-side fault — retry, and if it persists, report the exception's cause with the SDK version." | + +## When + +Raised when the SDK's `/gate` or `/execute` call fails with something +that is **not** a transport failure — the SDK's OWN code faulted while +asking the gateway a question. The gateway may well have answered; the +SDK could not read what it got, or never got as far as asking. There is +therefore no decision to honour, and the SDK refuses the call instead of +assuming it was allowed. + +This is NOT the same as a dead backend. A transport failure is +[NR-B001](NR-B001.md) / the ADR-008 fail-OPEN posture: the gateway could +not be REACHED, there is no answer to honour, and the agent must not +freeze during an outage. Here the enforcement path is un-decidable +rather than undecided — an unanswered question is not a yes. + +Before this code existed, the runtime's gate handler ended in a bare +`except Exception` that failed OPEN for every fault, so an SDK-internal +error was logged and the call proceeded with no reservation: the tool +ran, the backend booked nothing, and the cost event was dropped (the +audit finding SDK-11). + +The `/execute` seam is the same fact on the other side of the call +(audit finding SDK-13, ADR-131). There, the `@protect` decorator's +catch-all used to report an SDK-internal fault as **`NR-B001`** and to +type it as a **policy decision**, so a FastAPI host rendered it **HTTP +429 / category "decision"** — the operator was told the policy refused +when the truth was that the SDK had a bug. It now raises this same +code, and the two-step fail-open opt-in cannot open it. + +## Common causes + +1. **A request body that cannot be serialised** — `json.dumps` on the + signed gate body raises, and the SDK never reaches the network. +2. **A decode fault** — the response body is not JSON, or a field has + an unexpected shape. +3. **A bug in the SDK's request builder or the edge path** — a + `KeyError` / `TypeError` / `AttributeError` raised while assembling + or splitting the request. + +## How to fix + +1. **Retry** — the fault may be transient (a malformed body from one + particular input, a race in the edge path). +2. **If it persists, report it** — the exception's `cause` carries the + original fault and `__cause__` is set, so the traceback reaches the + actual bug. Report it with the SDK version. +3. **Check the counter** — `gate_indeterminate_total` counts the + `/gate` refusals and `policy_gate_indeterminate_total` counts the + `/execute` ones. They are deliberately NOT `gate_fail_open_total`: + that one counts calls that were allowed, these count calls that were + stopped. A non-zero value means the SDK itself is faulting, not that + the backend is down — and the two counters separate the two seams, + so an operator chasing an SDK bug knows which one moved. + +## Catch pattern + +```python +from nullrun.breaker.exceptions import NullRunGateIndeterminateError + +try: + nullrun.init(api_key="nr_live_...") +except NullRunGateIndeterminateError as exc: + if exc.error_code == "NR-P004": + log.error("gate decision indeterminate: %s", exc.__cause__) + # The SDK's own fault — surface it, do not treat it as + # "the backend is down and the call was allowed". + return render_sdk_fault_page() + raise +``` + +## Related codes + +- [NR-B001](NR-B001.md) — network error. The gateway could not be + reached; that one fails OPEN by policy, this one does not. +- [NR-B002](NR-B002.md) — 5xx from the NullRun backend. +- [NR-P001](NR-P001.md) — wire-protocol version mismatch. diff --git a/docs/errors/NR-P005.md b/docs/errors/NR-P005.md new file mode 100644 index 0000000..aec3c70 --- /dev/null +++ b/docs/errors/NR-P005.md @@ -0,0 +1,85 @@ +# NR-P005 — Unknown control-plane action type + +| Field | Value | +|---|---| +| **Code** | `NR-P005` | +| **Category** | **P**rotocol | +| **Exception class** | `NullRunUnknownActionError` | +| **Retryable** | No | +| **Default `user_action`** | "The NullRun control plane sent an action type this SDK version does not know, so the action was NOT executed. This is a server/SDK version mismatch — upgrade the SDK to a version that understands the control plane's action vocabulary. The workflow itself is not blocked." | + +## When + +Raised when a control-plane action arrives whose `type` is not in the +SDK's `ActionType` enum — a wire value the SDK cannot interpret, and +therefore cannot execute. + +The action dispatcher used to log the unknown type at ERROR, record a +forensic history event, and **return**. The action was then neither +executed nor refused, while the caller's own bookkeeping said it had +been handled: a control-plane instruction that is silently dropped is +a control-plane instruction that does not exist (the audit finding +SDK-12). + +## Common causes + +1. **A server/SDK version mismatch** — the backend grew its action + vocabulary (`Alert | Pause | Kill | Snapshot` today) and the SDK in + the field has not caught up. The backend emits the action type as a + Rust `Debug` string (`"Kill"`, `"Pause"`, …), so a rename on the + backend side becomes an unknown type on the SDK side. +2. **A control-plane schema regression** — the server sends a value + that was never in the vocabulary. +3. **A hostile or corrupt response** — anything the SDK cannot parse. + +## What it is NOT + +It is deliberately **not** a `NullRunBlockedException` and **not** a +`NullRunDecision`. An unknown action type must not degrade into "the +workflow is blocked": that was the DoS amplifier the B14 fix removed — +one unrecognised value from the control plane would stop every +subsequent tool call in the workflow. The workflow keeps running; the +*action* did not run. + +It is also deliberately **not** a `NullRunTransportError`. ADR-126's +two-step opt-in opens transport errors, and "the body runs" is not an +acceptable answer to "the control plane sent an instruction this SDK +cannot execute". + +## How to fix + +1. **Upgrade the SDK** — this is a version-alignment problem, not a + transient fault. Retrying the same action against the same SDK + version fails identically (`retryable` is `False` for that reason). +2. **Check the counter** — `unknown_action_total` counts these. The + expected rate is zero; a non-zero rate means the control plane is + sending instructions this SDK cannot run. +3. **Check the logs** — the dispatcher logs at ERROR with the action + type and the workflow id, and the `/track/batch` call site logs the + same event as a control-plane integrity event (not as a "skipped + malformed action"). + +## Catch pattern + +```python +from nullrun.breaker.exceptions import NullRunUnknownActionError + +try: + nullrun.handle_action(action_type, workflow_id, reason) +except NullRunUnknownActionError as exc: + if exc.error_code == "NR-P005": + log.error("control plane sent an unknown action: %s", exc.action) + # The ACTION did not run. The workflow is NOT blocked — do not + # turn this into a policy refusal. + return render_version_mismatch_notice() + raise +``` + +## Related codes + +- [NR-P001](NR-P001.md) — wire-protocol version mismatch. The same + root cause seen one layer earlier: the SDK and the server disagree + about the contract. +- [NR-W002](NR-W002.md) — the workflow was killed by the control + plane. That one is a decision the SDK can act on; this one is an + instruction the SDK cannot parse. diff --git a/docs/errors/README.md b/docs/errors/README.md index 591a458..fb3180c 100644 --- a/docs/errors/README.md +++ b/docs/errors/README.md @@ -9,7 +9,7 @@ The codes follow a `NR-` pattern: | `NR-C` | **C**onfiguration | Missing or invalid SDK config (no api_key, no workflow, etc.) | | `NR-A` | **A**uthentication | API key rejected, auth response malformed, approval flow anomaly | | `NR-B` | **B**ackend | 5xx, network error, budget exhausted, sensitive-tool extraction failure | -| `NR-P` | **P**rotocol | Wire-protocol version mismatch (X-NULLRUN-PROTOCOL too old / too new) | +| `NR-P` | **P**rotocol | Wire-protocol version mismatch (X-NULLRUN-PROTOCOL too old / too new), a gate decision the SDK could not determine, or a control-plane action the SDK cannot execute | | `NR-W` | **W**orkflow state | Workflow killed, paused, soft-deleted | | `NR-CH` | **Ch**ain | Chain context invalid (not found, org/key mismatch, parent execution mismatch) | | `NR-T` | **T**ool | Tool in block list | @@ -52,6 +52,8 @@ The codes follow a `NR-` pattern: | Code | When | See | |---|---|---| | `NR-P001` | Wire-protocol version mismatch (`X-NULLRUN-PROTOCOL` too old / too new) | [NR-P001](NR-P001.md) | +| `NR-P004` | The decision could not be determined — a `/gate` or `/execute` call failed for a reason that is NOT a transport failure, so the SDK refused the call | [NR-P004](NR-P004.md) | +| `NR-P005` | The control plane sent an action type this SDK does not know — the action was NOT executed (the workflow is not blocked) | [NR-P005](NR-P005.md) | ### Workflow state (NR-W) diff --git a/docs/wal-and-dlq.md b/docs/wal-and-dlq.md index 6f08cad..ee3463e 100644 --- a/docs/wal-and-dlq.md +++ b/docs/wal-and-dlq.md @@ -8,8 +8,23 @@ someone money. ## The files -One base path (`NULLRUN_WAL_PATH`, else `/nullrun.wal`) and five -files: +One base path, and the files listed below it. The base path is: + +1. `NULLRUN_WAL_PATH`, when the operator set it. It wins outright. +2. Otherwise `/nullrun--.wal` — scoped per OS user and + per **API key**. + +The key scope is the point. Before SDK-4 (security audit 2026-11-06) every +runtime on the host resolved to the same fixed `/nullrun.wal`, and +the atomic whole-file replace in `Transport._persist_to_wal` replaced another +runtime's undelivered events with its own buffer — spend that had already +happened was never booked. One API key belongs to exactly one organisation, +so a key-scoped WAL is an org-scoped WAL in money terms. + +The path is resolved **once**, when the transport is constructed, and never +changes. The organisation id is only known after authentication; a path that +changed at authentication would orphan every file written before it, and the +money would be lost with nothing in the logs to show for it. | File | Holds | Read by | |---|---|---| @@ -19,12 +34,43 @@ files: | `sdk.wal.holdover` | refusals the full DLQ had no room for | same — as a hold, never as a send | | `sdk.wal.dlq` | events the backend refused | **a human**, via `nullrun-wal` | | `sdk.wal.lock` | the advisory lock, 0 bytes | `flock` | +| `sdk.wal.v3` | v3 consume payloads that were not delivered | the next `Transport.start()` | The first three self-heal. The DLQ does not, and cannot: a refusal is by definition not something a retry fixes, so re-sending a dead-lettered event would be refused again. Automatic DLQ replay is the poison pill the DLQ exists to break out of. +### The v3 file + +`sdk.wal.v3` holds the **v3 consume payload** — the `/api/v1/track` body +that settles a reservation — for calls whose tool ran but whose consume was +not delivered. It is a separate file from `sdk.wal` because the two replay +through different endpoints: the base WAL goes to `/api/v1/track/batch`, and +the batch path does not run the CONSUME script. A v3 payload replayed +through the batch path is never settled, so the money is never booked — the +same loss as dropping it, with more machinery. + +The payload is stored **verbatim**, `idempotency_key` included. The backend +deletes the reservation key after the first successful consume, so a replay +without the key is answered with 503 `RESERVATION_NOT_FOUND` and the row +could never leave the file. With the key the backend replays its stored +answer (200, `idempotent_replay: true`) and the row clears. + +A row leaves the file only when the backend accepted it. A payload the +backend refuses again stays on disk: an undelivered spend nobody retries is +a lost spend. + +### The legacy file + +`/nullrun.wal` — the unscoped path the SDK used before SDK-4 — is +never written to any more. If it exists and is not empty, the SDK reports it +at startup and then leaves it alone: it was shared by every user and every +org on the host, so its rows may belong to another tenant, and replaying +them into this org's WAL would book a stranger's spend against this account. +Inspect it with `nullrun-wal --wal /nullrun.wal status` and decide +under the org that owns it. + ## What is guaranteed * **At-least-once.** No recovery file is unlinked before its events are @@ -128,15 +174,27 @@ volume, each worker has its own in-memory buffer and its own events; the correct configuration is a per-worker `NULLRUN_WAL_PATH`, and with a shared one correctness rests on `event_id` dedup rather than on the lock. +**`NULLRUN_WAL_PATH` must be unique per runtime.** Pointing two runtimes at +one path collides exactly like the old shared default did — the whole-file +replace in `_persist_to_wal` destroys the other runtime's undelivered events +— and the SDK cannot detect it. The default is safe because it is scoped per +API key; an explicit path overrides that and takes the responsibility with +it. + ## Inspecting and replaying the DLQ ```bash -nullrun-wal status # files, row counts, reasons, window +nullrun-wal status # every WAL this user owns +nullrun-wal --wal status # one WAL nullrun-wal replay --dry-run # what would be sent nullrun-wal replay --reason reservation_not_found --limit 100 nullrun-wal replay --execute # actually send ``` +`status` without `--wal` lists every scoped WAL this user owns — there is one +per API key — and `replay` refuses to guess between them. A guess could +point the operator at another tenant's file. + `replay` is a dry run unless `--execute` is passed, and it removes a row **only** after the backend confirms that exact `event_id`. Anything unconfirmed stays on disk, including a line the tool could not parse: the diff --git a/pyproject.toml b/pyproject.toml index 186da6d..f832276 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,24 @@ [build-system] -requires = ["hatchling"] +# The build backend is pinned to an EXACT version on purpose. +# +# SDK-3 (security audit 2026-11-06) found ``requires = ["hatchling"]`` +# with no constraint. The publish job resolves this list fresh, inside +# the job that holds the PyPI Trusted Publishing token +# (``permissions: id-token: write``). An unpinned backend means the +# version that builds and uploads the wheel is chosen at publish time, +# not at commit time, so a new upstream release lands inside the +# trusted-publishing job without a review. +# +# An exact pin makes that resolution deterministic. Bump it explicitly +# (with a test run) rather than letting the range float. +requires = ["hatchling==1.32.4"] build-backend = "hatchling.build" [project] name = "nullrun" # Full release history lives in CHANGELOG.md; only the current version # is pinned here. -version = "0.23.0" +version = "0.24.0" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. The headline is the canonical §1 statement # from positioning.md — "runtime decision layer for tool-using AI agents" @@ -170,6 +182,34 @@ dev = [ "tomli>=2.0; python_version < '3.11'", ] +# PEP 735 dependency groups. A group is a developer-side install set: it +# never lands in the wheel metadata and it never reaches an end user. +# +# SDK-3 (security audit 2026-11-06) found the publish workflow running +# ``pip install --upgrade build`` and ``pip install twine`` inside the job +# that holds the PyPI Trusted Publishing token. Both resolved at run time, +# inside the trusted-publishing job. The group below puts them in +# ``uv.lock`` with exact pins, so the publish job installs the same bytes +# every run. +# +# The versions are exact on purpose: the lock is the contract, and an +# exact pin says which bytes the lock carries. +# +# These are NOT project extras. ``tests/test_websocket_extra_is_declared.py`` +# pins the set of ``[project.optional-dependencies]`` extras to exactly +# {opentelemetry, websocket, dev}, and a build tool has no business in an +# end user's extra. +[dependency-groups] +publish = [ + "build==1.6.1", + # twine 7.0.0 is the version that understands ``Metadata-Version: 2.5``, + # which is what hatchling >= 1.30 writes and what nullrun 0.23.0 on + # PyPI already ships. twine 6.2.0 rejects it ("2.5 is not a valid + # metadata version"), so ``twine check`` would fail on the project's + # own artifact. + "twine==7.0.0", +] + [project.scripts] # `nullrun-wal` inspects and replays the local write-ahead log. It exists # because the DLQ holds events the SDK could not deliver and nothing else diff --git a/scripts/check_supply_chain.py b/scripts/check_supply_chain.py new file mode 100644 index 0000000..2188eaa --- /dev/null +++ b/scripts/check_supply_chain.py @@ -0,0 +1,710 @@ +#!/usr/bin/env python3 +"""Fail when a workflow pins an action to a mutable ref, or installs a +dependency outside the lock file. + +SDK-3 (security audit 2026-11-06) found three holes in this repository's CI: + +1. Every ``uses:`` pointed at a tag or a branch. A tag or a branch is a + mutable name: whoever controls the action repository can move it, and the + workflow then runs code that was never reviewed here. The fix is to pin the + 40-hex commit SHA and keep the tag in a trailing comment. +2. No workflow consumed ``uv.lock``. Each job ran ``pip install -e ".[dev]"``, + so the resolved versions were chosen at run time, not at commit time. +3. The publish job ran ``pip install --upgrade build`` and ``pip install + twine`` inside the job that holds the PyPI Trusted Publishing token. + +This script is the check that keeps those holes closed. It exits non-zero and +prints one line per violation. + +Rules +----- + +``action-ref-not-sha-pinned`` + A ``uses:`` value must be ``owner/repo@<40 lowercase hex>``. A local + action (``./path``) is allowed: it lives in this repository, so the + commit already pins it. A ``docker://`` image must be pinned by digest. + +``action-version-not-pinned`` + ``astral-sh/setup-uv`` must name an exact ``version``. Without it the + action downloads whatever ``latest`` resolves to. + +``dependency-not-exactly-pinned`` + A ``pip install`` requirement must be an exact ``==`` pin. A range + (``>=``), a bare name, or an editable install of this project all resolve + at run time. + +``requirements-file-without-hashes`` + ``pip install -r FILE`` must carry ``--require-hashes``. + +``uv-sync-not-frozen`` + A ``uv sync`` must carry ``--frozen`` or ``--locked``, and must not carry + ``--upgrade``. Without one of those the job can rewrite the lock file. + +``uv-run-not-frozen`` + A ``uv run`` must carry ``--frozen`` or ``--locked``. Without one of those + the command can re-resolve and rewrite the lock file before it runs. + +``uv-install-bypasses-lock`` + ``uv pip install``, ``uv add``, ``uv remove``, ``uvx`` and ``uv tool run`` + install a package outside the lock file. + +``pip-install-in-trusted-job`` + A workflow that holds an OIDC token (``id-token: write``) must not install + packages with ``pip`` at all, not even with an exact ``==`` pin. Inside + that job a pip install resolves outside ``uv.lock``, which is the hole + the audit found. The job installs from the lock with ``uv sync + --frozen`` instead. + +Usage +----- + +``python scripts/check_supply_chain.py`` checks ``.github/workflows`` in this +repository. Pass file or directory arguments to check something else. The +paths resolve relative to this script, never to the current directory. + +``python scripts/check_supply_chain.py --self-test`` plants a broken workflow +in a temporary directory and asserts the lint goes red on it. A lint that has +only ever been green on a clean tree may be a lint that never fails. + +The script needs no third-party package. PyYAML is used when it is +importable; otherwise the same rules run over a text scan of the file. +""" + +from __future__ import annotations + +import argparse +import re +import shlex +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows" + +WORKFLOW_SUFFIXES = (".yml", ".yaml") + +# ── Patterns ──────────────────────────────────────────────────────────────── + +SHA40 = re.compile(r"^[0-9a-f]{40}$") +PINNED_ACTION = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@([0-9a-f]{40})$") +DOCKER_DIGEST = re.compile(r"^docker://[^\s@]+@sha256:[0-9a-f]{64}$") +LOCAL_ACTION = re.compile(r"^\./") +EXACT_VERSION = re.compile(r"^\d+\.\d+\.\d+$") + +# ``name==1.2.3`` or ``name[extra]==1.2.3``. A range, a bare name, a URL and +# an editable path all fail this. +EXACT_REQUIREMENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*(\[[^\]]+\])?==[^\s,;]+$") + +USES_LINE = re.compile(r"^(?P\s*)(?:-\s+)?uses:\s*(?P\S+)") +RUN_LINE = re.compile(r"^(?P\s*)(?:-\s+)?run:\s*(?P.*)$") +BLOCK_SCALAR = re.compile(r"^[|>][-+]?\d*$") + +# ``pip install`` and its variants. ``uv pip install`` is NOT here: it +# bypasses uv.lock even when the version is exact, so it has its own rule. +PIP_INSTALL = re.compile( + r"^(?:(?:python[0-9.]*|py)\s+-m\s+pip|pip[0-9.]*)\s+install\b(?P.*)$" +) +UV_SYNC = re.compile(r"^uv\s+sync\b(?P.*)$") +UV_RUN = re.compile(r"^uv\s+run\b(?P.*)$") +UV_PIP_INSTALL = re.compile(r"^uv\s+pip\s+install\b(?P.*)$") +UV_MUTATE = re.compile(r"^uv\s+(?:add|remove)\b") +UV_TOOL = re.compile(r"^(?:uvx|uv\s+tool\s+run)\b(?P.*)$") + +SHELL_SEPARATORS = re.compile(r"&&|\|\||[;|]") + +# A workflow that asks for an OIDC token. Inside that workflow a pip +# install is the hole the audit found, even with an exact pin. +OAUTH_TRUST = re.compile(r"id-token:\s*write") + +# A workflow with the two holes the audit found: a mutable action ref and a +# dependency that resolves at run time. The self-test plants this and +# asserts the lint goes red on it. +BROKEN_WORKFLOW = """\ +name: broken +on: + push: + branches: [master] +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: pip install -e ".[dev]" "pytest-xdist>=3.6" +""" + +EXPECTED_SELF_TEST_RULES = { + "action-ref-not-sha-pinned", + "dependency-not-exactly-pinned", +} + + +@dataclass(frozen=True) +class Violation: + """One rule failure at one place in one file.""" + + path: str + line: int + rule: str + subject: str + message: str + + def render(self) -> str: + where = self.path if self.line is None else f"{self.path}:{self.line}" + return f"{where}: {self.rule}: {self.message} (offending: {self.subject})" + + +# ── Reading the workflow ──────────────────────────────────────────────────── + + +def workflow_files(paths: list[Path]) -> list[Path]: + """Expand the arguments into the list of workflow files to check.""" + found: list[Path] = [] + for path in paths: + if path.is_dir(): + found.extend( + sorted( + child + for child in path.rglob("*") + if child.is_file() and child.suffix in WORKFLOW_SUFFIXES + ) + ) + elif path.is_file(): + found.append(path) + else: + raise FileNotFoundError(f"no such path: {path}") + return found + + +def yaml_available() -> bool: + try: + import yaml # noqa: F401 + except Exception: + return False + return True + + +def structural_content(text: str) -> tuple[list[tuple[str, int]], list[tuple[str, int]]]: + """Return the ``uses`` values and the ``run`` blocks of a workflow. + + The parse goes through ``yaml.compose`` rather than ``yaml.safe_load`` + because the composed node carries the line of every key and value. The + rules report a file position, so the position must be the real one and + not "the first line that mentions this string" — two steps in one file + can use the same action, and each is a separate finding. + """ + try: + import yaml + except Exception: + return [], [] + + try: + document = yaml.compose(text) + except Exception: + return [], [] + + uses: list[tuple[str, int]] = [] + runs: list[tuple[str, int]] = [] + + def walk(node: object) -> None: + if isinstance(node, yaml.MappingNode): + for key_node, value_node in node.value: + if not isinstance(key_node, yaml.ScalarNode): + continue + if key_node.value == "uses" and isinstance(value_node, yaml.ScalarNode): + uses.append((value_node.value, value_node.start_mark.line + 1)) + elif key_node.value == "run" and isinstance(value_node, yaml.ScalarNode): + # A block scalar (``run: |``) starts on the line AFTER + # the key. A plain scalar (``run: pytest``) starts on the + # key's own line. + block = value_node.style in ("|", ">") + runs.append( + (value_node.value, value_node.start_mark.line + 1 + (1 if block else 0)) + ) + else: + walk(value_node) + elif isinstance(node, yaml.SequenceNode): + for item in node.value: + walk(item) + + walk(document) + return uses, runs + + +def text_content(text: str) -> tuple[list[tuple[str, int]], list[tuple[str, int]]]: + """Return the ``uses`` values and the ``run`` blocks of a workflow. + + This is the fallback for a runner without PyYAML. It reads the same + surfaces a workflow file actually has: a ``uses:`` key and a ``run:`` + key with either an inline value or a block scalar. + """ + lines = text.splitlines() + uses: list[tuple[str, int]] = [] + runs: list[tuple[str, int]] = [] + + index = 0 + while index < len(lines): + line = lines[index] + + match = USES_LINE.match(line) + if match: + value = match.group("value").split("#", 1)[0].strip() + if value: + uses.append((value, index + 1)) + + run_match = RUN_LINE.match(line) + if run_match: + rest = run_match.group("rest").strip() + if BLOCK_SCALAR.match(rest): + base_indent = len(run_match.group("indent")) + block: list[str] = [] + cursor = index + 1 + while cursor < len(lines): + candidate = lines[cursor] + if not candidate.strip(): + block.append("") + cursor += 1 + continue + if len(candidate) - len(candidate.lstrip()) <= base_indent: + break + block.append(candidate) + cursor += 1 + runs.append(("\n".join(block), index + 2)) + index = cursor + continue + if rest: + runs.append((rest, index + 1)) + + index += 1 + + return uses, runs + + +# ── The rules ─────────────────────────────────────────────────────────────── + + +def check_uses(value: str, path: str, line: int) -> list[Violation]: + subject = value.strip() + if LOCAL_ACTION.match(subject): + return [] + if subject.startswith("docker://"): + if DOCKER_DIGEST.match(subject): + return [] + return [ + Violation( + path=path, + line=line, + rule="action-ref-not-sha-pinned", + subject=subject, + message=( + "a container image must be pinned by digest (docker://image@sha256:<64 hex>)" + ), + ) + ] + if PINNED_ACTION.match(subject): + return [] + return [ + Violation( + path=path, + line=line, + rule="action-ref-not-sha-pinned", + subject=subject, + message=("an action must be pinned to a 40-hex commit SHA (owner/repo@ # )"), + ) + ] + + +def check_setup_uv_version(text: str, path: str) -> list[Violation]: + """``astral-sh/setup-uv`` must name an exact version.""" + violations: list[Violation] = [] + lines = text.splitlines() + + for number, line in enumerate(lines, start=1): + match = USES_LINE.match(line) + if not match: + continue + value = match.group("value").split("#", 1)[0].strip() + if not value.startswith("astral-sh/setup-uv@"): + continue + + version = "" + version_line = 0 + cursor = number + while cursor < len(lines): + candidate = lines[cursor] + if candidate.strip() and re.match(r"^\s*(?:-\s+)?uses:", candidate): + break + version_match = re.match(r"^\s*version:\s*['\"]?([^'\"\s]+)", candidate) + if version_match: + version = version_match.group(1) + version_line = cursor + 1 + break + cursor += 1 + + if not version: + violations.append( + Violation( + path=path, + line=number, + rule="action-version-not-pinned", + subject=value, + message=( + 'astral-sh/setup-uv must set an exact version (with: version: "X.Y.Z")' + ), + ) + ) + elif not EXACT_VERSION.match(version): + violations.append( + Violation( + path=path, + line=version_line, + rule="action-version-not-pinned", + subject=version, + message=( + 'astral-sh/setup-uv must set an exact version (with: version: "X.Y.Z")' + ), + ) + ) + + return violations + + +def split_shell(line: str) -> list[str]: + """Split one shell line into the commands it runs.""" + parts = SHELL_SEPARATORS.split(line) + return [part.strip() for part in parts if part.strip()] + + +def check_pip_install(command: str, path: str, line: int) -> list[Violation]: + match = PIP_INSTALL.match(command) + if not match: + return [] + + rest = match.group("rest").strip() + try: + tokens = shlex.split(rest) + except ValueError: + tokens = rest.split() + + flags = [token for token in tokens if token.startswith("-")] + requirements: list[str] = [] + editable = False + requirement_file = False + + index = 0 + while index < len(tokens): + token = tokens[index] + if token in ("-e", "--editable"): + editable = True + elif token in ("-r", "--requirement"): + requirement_file = True + index += 1 + if index < len(tokens): + requirements.append(tokens[index]) + index += 1 + continue + elif token.startswith("-"): + pass + else: + requirements.append(token) + index += 1 + + violations: list[Violation] = [] + + if requirement_file and "--require-hashes" not in flags: + violations.append( + Violation( + path=path, + line=line, + rule="requirements-file-without-hashes", + subject=command, + message="pip install -r must carry --require-hashes", + ) + ) + + if editable: + violations.append( + Violation( + path=path, + line=line, + rule="dependency-not-exactly-pinned", + subject=command, + message=( + "an editable install of this project resolves its dependencies at run time" + ), + ) + ) + return violations + + for requirement in requirements: + if requirement_file and requirement.endswith((".txt", ".in")): + continue + if EXACT_REQUIREMENT.match(requirement): + continue + violations.append( + Violation( + path=path, + line=line, + rule="dependency-not-exactly-pinned", + subject=requirement, + message=( + "a dependency must be an exact == pin; use " + "uv sync --frozen to install from uv.lock" + ), + ) + ) + + return violations + + +def check_uv_sync(command: str, path: str, line: int) -> list[Violation]: + match = UV_SYNC.match(command) + if not match: + return [] + + try: + tokens = shlex.split(match.group("rest")) + except ValueError: + tokens = match.group("rest").split() + + violations: list[Violation] = [] + if "--frozen" not in tokens and "--locked" not in tokens: + violations.append( + Violation( + path=path, + line=line, + rule="uv-sync-not-frozen", + subject=command, + message="uv sync must carry --frozen or --locked", + ) + ) + if "--upgrade" in tokens or "--upgrade-package" in tokens: + violations.append( + Violation( + path=path, + line=line, + rule="uv-sync-not-frozen", + subject=command, + message="uv sync must not carry --upgrade in CI", + ) + ) + return violations + + +def check_uv_run(command: str, path: str, line: int) -> list[Violation]: + """``uv run`` must not be allowed to re-resolve the lock. + + ``uv run`` syncs the project environment before it runs the command. + Without ``--frozen`` or ``--locked`` that sync may re-resolve and + rewrite ``uv.lock``, which turns a reviewed lock into a suggestion. + """ + match = UV_RUN.match(command) + if not match: + return [] + + try: + tokens = shlex.split(match.group("rest")) + except ValueError: + tokens = match.group("rest").split() + + if "--frozen" in tokens or "--locked" in tokens: + return [] + + return [ + Violation( + path=path, + line=line, + rule="uv-run-not-frozen", + subject=command, + message="uv run must carry --frozen or --locked", + ) + ] + + +def check_no_pip_in_trusted_workflow( + text: str, runs: list[tuple[str, int]], path: str +) -> list[Violation]: + """No pip install in a workflow that holds an OIDC token. + + The rule is per workflow, not per job: a file that asks for + ``id-token: write`` anywhere must not install packages with pip + anywhere. The scan runs over the raw text, so the rule holds in the + text scanner too and not only when PyYAML is importable. + """ + if not OAUTH_TRUST.search(text): + return [] + + violations: list[Violation] = [] + for block, first_line in runs: + for offset, raw in enumerate(block.splitlines()): + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + for command in split_shell(stripped): + if PIP_INSTALL.match(command): + violations.append( + Violation( + path=path, + line=first_line + offset, + rule="pip-install-in-trusted-job", + subject=command, + message=( + "this workflow holds an OIDC token " + "(id-token: write); install from uv.lock " + "with uv sync --frozen, never with pip" + ), + ) + ) + return violations + + +def check_run_block(block: str, path: str, first_line: int) -> list[Violation]: + """Apply the install rules to every command in one ``run:`` block.""" + violations: list[Violation] = [] + for offset, raw in enumerate(block.splitlines()): + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + for command in split_shell(stripped): + if PIP_INSTALL.match(command): + violations.extend(check_pip_install(command, path, first_line + offset)) + elif UV_SYNC.match(command): + violations.extend(check_uv_sync(command, path, first_line + offset)) + elif UV_RUN.match(command): + violations.extend(check_uv_run(command, path, first_line + offset)) + elif ( + UV_PIP_INSTALL.match(command) or UV_MUTATE.match(command) or UV_TOOL.match(command) + ): + violations.append( + Violation( + path=path, + line=first_line + offset, + rule="uv-install-bypasses-lock", + subject=command, + message=( + "this command installs a package outside uv.lock; use uv sync --frozen" + ), + ) + ) + return violations + + +# ── The entry points ──────────────────────────────────────────────────────── + + +def find_violations(paths: list[Path], use_yaml: bool | None = None) -> list[Violation]: + """Check every workflow under ``paths`` and return the violations.""" + if use_yaml is None: + use_yaml = yaml_available() + + violations: list[Violation] = [] + for workflow in workflow_files(paths): + text = workflow.read_text(encoding="utf-8") + display = workflow.as_posix() + + if use_yaml: + uses, runs = structural_content(text) + else: + uses, runs = text_content(text) + + for value, line in uses: + violations.extend(check_uses(value, display, line)) + + violations.extend(check_setup_uv_version(text, display)) + + violations.extend(check_no_pip_in_trusted_workflow(text, runs, display)) + + for block, first_line in runs: + violations.extend(check_run_block(block, display, first_line)) + + return violations + + +def self_test() -> int: + """Prove the lint goes red. + + The lint is written to fail, but a rule that never fires is a rule + that is not there. This plants a workflow with a mutable action ref + and a run-time dependency install, then checks that BOTH scanners + (the YAML one and the text one) report it. + """ + with tempfile.TemporaryDirectory() as directory: + workflow = Path(directory) / "broken.yml" + workflow.write_text(BROKEN_WORKFLOW, encoding="utf-8") + + for use_yaml in (True, False): + violations = find_violations([workflow], use_yaml=use_yaml) + found = {violation.rule for violation in violations} + missing = EXPECTED_SELF_TEST_RULES - found + if missing: + scanner = "yaml" if use_yaml else "text" + print( + f"supply-chain: self-test FAILED ({scanner} scanner did not " + f"report {sorted(missing)})", + file=sys.stderr, + ) + return 1 + + print( + "supply-chain: self-test OK (both scanners catch a mutable action ref " + "and a run-time dependency install)" + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Fail on a mutable action ref or an unpinned dependency install." + ) + parser.add_argument( + "paths", + nargs="*", + type=Path, + help=( + "workflow files or directories to check. " + f"Defaults to {DEFAULT_WORKFLOW_DIR.relative_to(REPO_ROOT).as_posix()}." + ), + ) + parser.add_argument( + "--no-yaml", + action="store_true", + help="use the text scanner even when PyYAML is importable", + ) + parser.add_argument( + "--self-test", + action="store_true", + help="plant a broken workflow and assert the lint goes red on it", + ) + arguments = parser.parse_args(argv) + + if arguments.self_test: + return self_test() + + paths = arguments.paths or [DEFAULT_WORKFLOW_DIR] + + try: + violations = find_violations(paths, use_yaml=not arguments.no_yaml) + except FileNotFoundError as error: + print(f"supply-chain: {error}", file=sys.stderr) + return 2 + + if violations: + for violation in violations: + print(violation.render()) + print( + f"supply-chain: {len(violations)} violation(s) in " + f"{len({violation.path for violation in violations})} file(s)", + file=sys.stderr, + ) + return 1 + + print("supply-chain: OK") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/nullrun/__init__.py b/src/nullrun/__init__.py index 6151eae..e9470d0 100644 --- a/src/nullrun/__init__.py +++ b/src/nullrun/__init__.py @@ -519,6 +519,17 @@ def my_agent: "nullrun.breaker.exceptions", "NullRunMalformedGateResponseError", ), + "NullRunGateIndeterminateError": ( + "nullrun.breaker.exceptions", + "NullRunGateIndeterminateError", + ), + # The control plane sent an action type this SDK cannot execute + # (SDK-12). Infrastructure, not a decision: the workflow is NOT + # blocked, but the action did NOT run and the caller is told. + "NullRunUnknownActionError": ( + "nullrun.breaker.exceptions", + "NullRunUnknownActionError", + ), "NullRunRateLimitRedisError": ("nullrun.breaker.exceptions", "NullRunRateLimitRedisError"), "NullRunBudgetRecheckFailedError": ("nullrun.breaker.exceptions", "NullRunBudgetRecheckFailedError"), "NullRunExecutionNotFoundError": ("nullrun.breaker.exceptions", "NullRunExecutionNotFoundError"), @@ -565,6 +576,10 @@ def my_agent: "nullrun.breaker.exceptions", "NullRunApprovalToolDigestMismatchError", ), + "NullRunApprovalToolMismatchError": ( + "nullrun.breaker.exceptions", + "NullRunApprovalToolMismatchError", + ), # Layer 2: on_error context type "ErrorContext": ("nullrun.observability.error_hooks", "ErrorContext"), # Layer 3: status dataclasses @@ -743,6 +758,8 @@ def __dir__() -> list[str]: "InsecureTransportError", "NullRunProtocolError", "NullRunMalformedGateResponseError", + "NullRunGateIndeterminateError", + "NullRunUnknownActionError", "NullRunRateLimitRedisError", "NullRunBudgetRecheckFailedError", "NullRunExecutionNotFoundError", @@ -759,6 +776,7 @@ def __dir__() -> list[str]: "NullRunApprovalReplayRejectedError", "NullRunApprovalDigestMismatchError", "NullRunApprovalToolDigestMismatchError", + "NullRunApprovalToolMismatchError", # User-facing message catalog — the single entry point for # turning an SDK exception into a string safe to display to # end users. ``set_user_message`` lets a deployment brand its diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index a19dd96..30a7c38 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -5,5 +5,5 @@ string and the SDK_MIN_VERSION constant. """ -__version__ = "0.23.0" +__version__ = "0.24.0" __platform_version__ = "1.0.0" diff --git a/src/nullrun/actions.py b/src/nullrun/actions.py index 60f7ddc..757fc2c 100644 --- a/src/nullrun/actions.py +++ b/src/nullrun/actions.py @@ -22,12 +22,31 @@ from nullrun.breaker.exceptions import ( NullRunBlockedException, + NullRunUnknownActionError, NullRunWorkflowKilledError, + WorkflowKilledInterrupt, WorkflowPausedException, ) +from nullrun.observability import metrics logger = logging.getLogger(__name__) +# The refusals the DEFAULT handlers raise ON PURPOSE: kill, pause, +# block. A handler that refuses this way HAS EXECUTED — the refusal is +# the action's outcome, not a fault in the handler, so it belongs on +# the INFO channel. Anything else a handler raises is a fault in the +# handler and belongs on the ERROR channel. +# +# Deliberately NOT the whole ``ENFORCEMENT_EXCEPTIONS`` family: that +# family also holds classes that mean "the SDK could not determine the +# answer" (``NullRunGateIndeterminateError``), and downgrading those +# to INFO here would hide a refusal behind a log level. +_ACTION_REFUSALS: tuple[type[BaseException], ...] = ( + WorkflowKilledInterrupt, + WorkflowPausedException, + NullRunBlockedException, +) + @dataclass class ActionEvent: @@ -69,8 +88,13 @@ class ActionHandler: Handler for NullRun circuit breaker actions. This executes protective actions when triggered: - - KILL: Immediately stops the workflow (raises NullRunWorkflowKilledError, NR-W002) - - PAUSE: Temporarily halts the workflow (raises WorkflowPausedException) + - KILL: signals the kill (the default handler raises + NullRunWorkflowKilledError, NR-W002). The raise is SWALLOWED by + ``handle`` — see its docstring — so what actually stops the + workflow is the GATE refusing the next /check (ADR-063 §3), not + this handler. + - PAUSE: signals the pause (raises WorkflowPausedException) — + swallowed the same way, and the gate refuses the next /check. - ALERT: Sends notification (can be customized) - SNAPSHOT: Captures workflow state for debugging - WEBHOOK: Sends HTTP webhook notification @@ -179,15 +203,23 @@ def handle( **details: Additional details about the action Raises: - NullRunWorkflowKilledError: If action is "kill" - (subclass of WorkflowKilledInterrupt, which is - the canonical catch name.) - WorkflowPausedException: If action is "pause" - NullRunBlockedException: If action is "block" + NullRunUnknownActionError: If ``action`` is not a known + action type. The action was NOT executed and the + caller is told so. It is deliberately NOT a block, so + the workflow keeps running — an unrecognised value + must not become a DoS amplifier. + + Note: + The handler's own refusal (kill / pause / block) does NOT + escape this method. It is recorded in the action history + and logged at INFO. The authoritative enforcement of a kill + or a pause is the GATE (ADR-063 §3): the gate refuses the + next /check. A handler that raises anything else is a + fault and is logged at ERROR. """ try: action_type = ActionType(action.lower()) - except ValueError: + except ValueError as exc: # Pre-fix this degraded silently to ``ActionType.BLOCK`` # (B14) and triggered ``_default_block`` which raises # ``NullRunBlockedException``. That made the SDK into a @@ -196,19 +228,22 @@ def handle( # would block every subsequent tool call in the workflow # with no actionable error. # + # B14's fix (log at ERROR, record the event, RETURN) kept + # the DoS property but left the action neither executed + # nor refused: the caller could not tell "it ran" from + # "it could not run" (SDK-12). + # # Post-fix: log at ERROR, record the event for forensic - # visibility, and DO NOT invoke any handler. The - # workflow keeps running under fail-open. The operator - # gets a clear signal that the control plane sent an - # action type the SDK doesn't understand — likely a - # version mismatch (server upgraded, SDK not yet) or a - # schema regression worth investigating. + # visibility, DO NOT invoke any handler, and RAISE a typed + # error so the caller learns the action did not run. The + # error is an infrastructure error, NOT a block — the + # workflow keeps running. logger.error( f"Unknown action type received from control plane: {action!r} " f"for workflow {workflow_id!r} (reason={reason!r}). " "This is a server/SDK version mismatch or a control plane " - "schema regression. Failing open — the workflow will continue " - "running. Investigate ASAP." + "schema regression. The action was NOT executed — the " + "workflow keeps running. Investigate ASAP." ) self._record_action( ActionType.BLOCK, @@ -216,7 +251,15 @@ def handle( f"unknown_action_type:{action}", details, ) - return + metrics.inc_runtime("unknown_action_total") + raise NullRunUnknownActionError( + f"the control plane sent an unknown action type {action!r} " + f"for workflow {workflow_id!r}; the action was not executed", + action=action, + workflow_id=workflow_id, + reason=reason, + cause=exc, + ) from exc handler = self._handlers.get(action_type, self._default_block) @@ -229,13 +272,28 @@ def handle( try: handler(workflow_id, reason or "Unknown", **details) # type: ignore[no-untyped-call] - except BaseException as e: - # Don't let handler exceptions propagate. We catch - # `BaseException` (not just `Exception`) because - # kill signals (NullRunWorkflowKilledError, the - # third-party kill-shaped signals must be recorded - # in history (already done above) and swallowed, - # NOT re-raised into the caller's frame. + except _ACTION_REFUSALS as e: + # The handler REFUSED ON PURPOSE — kill, pause, block. That + # is the action's outcome, not a fault in the handler, so it + # goes to INFO and the ERROR channel keeps meaning + # "something is broken". Pre-SDK-12 this arm logged at + # ERROR, so every server-side kill raised a false alarm. + # + # The refusal is deliberately NOT re-raised: the history + # entry above is already the audit trail, and the + # authoritative enforcement of a kill/pause is the GATE + # (ADR-063 §3 — the gate refuses the next /check). + # Re-raising here would abort the caller's action loop over + # an action that was delivered correctly. + logger.info( + f"Action {action_type.value} refused for workflow " + f"{workflow_id}: {e}" + ) + except Exception as e: + # A real fault in the handler (a buggy custom handler, a + # broken call). ``KeyboardInterrupt`` / ``SystemExit`` / + # ``GeneratorExit`` are NOT caught: they are BaseException, + # not Exception, and a Ctrl-C must reach the caller. logger.error(f"Action handler error: {e}") def _default_kill( diff --git a/src/nullrun/breaker/exceptions.py b/src/nullrun/breaker/exceptions.py index d0f758e..eff0fde 100644 --- a/src/nullrun/breaker/exceptions.py +++ b/src/nullrun/breaker/exceptions.py @@ -423,11 +423,146 @@ class NullRunMalformedGateResponseError(NullRunProtocolError): retryable = True +class NullRunGateIndeterminateError(NullRunInfrastructureError): + """The gate's decision could not be determined — and the SDK will not guess. + + Raised when the ``/gate`` call fails with something that is NOT a + transport failure: the SDK's OWN code faulted while asking the gate + a question (a request body that cannot be serialised, a decoding + error, a programming error in the request builder, a fault in the + edge path's token split). + + Why this is not a transport error + --------------------------------- + ADR-008's fail-OPEN covers ONE thing: the gate could not be + REACHED, so there is no answer to honour and the agent must not + freeze. That is a fact about the world — the backend is down. + + This class is the opposite. The backend may well have answered; the + SDK simply cannot read what it got, or never got as far as asking. + The question "may this call proceed?" therefore has no answer, and + an unanswered question is not a yes. ADR-063 §2.4's governing rule + says it exactly: *"a fail-OPEN where the answer is known and merely + inconvenient to obtain is a bug"* — and the sharper form: an + enforcement failure must never become implicit permission. + + Before this class existed, the runtime's gate handler ended in a + bare ``except Exception`` that failed OPEN for every fault, so an + SDK-internal ``KeyError`` was logged and the call proceeded with no + reservation (SDK-11). That is the same shape as the SDK-10 hole one + class away: the call ran, and its cost event was dropped. + + It is deliberately NOT a :class:`NullRunTransportError` subclass. + The ``@protect`` boundary treats a transport error as the one class + ADR-126's two-step opt-in can open; an indeterminate decision must + not be convertible into "the body runs". Deriving from + :class:`NullRunInfrastructureError` puts it in the boundary's + ``except NullRunInfrastructureError: raise`` arm instead. + """ + + error_code = "NR-P004" + user_action = ( + "The SDK could not obtain a decision from the NullRun gate: the " + "call failed for a reason that is not a network failure, so the " + "SDK stopped rather than assume the call was allowed. This is " + "usually an SDK-side fault — retry, and if it persists, report " + "the exception's cause with the SDK version." + ) + retryable = True + + def __init__( + self, + message: str, + *, + error_code: str | None = None, + user_action: str | None = None, + retryable: bool | None = None, + docs_url: str | None = None, + cause: BaseException | None = None, + ) -> None: + # ``cause`` is the original fault. It is what makes this class + # actionable: the message says the decision was indeterminate, + # and the cause says WHY (a bad request body, a decode error). + # + # The other keywords mirror the base class: a caller may + # override the class-level values per instance, and the base + # never mutates the class attribute. + super().__init__( + message, + error_code=error_code, + user_action=user_action, + retryable=retryable, + docs_url=docs_url, + cause=cause, + ) + + +class NullRunUnknownActionError(NullRunInfrastructureError): + """The control plane sent an action type this SDK does not know. + + Raised when a control-plane action arrives whose ``type`` is not + in the SDK's ``ActionType`` enum — a wire value the SDK cannot + interpret, and therefore cannot execute. + + Why this raises instead of returning quietly (SDK-12) + ----------------------------------------------------- + The action dispatcher used to log the unknown type at ERROR, record + a forensic history event, and RETURN. The caller then had no way + to tell "the action ran" from "the action could not be run": the + action was neither executed nor refused, while the caller's own + bookkeeping said it had been handled. A control-plane instruction + that is silently dropped is a control-plane instruction that does + not exist. + + The refusal is deliberately NOT a :class:`NullRunBlockedException`. + An unknown action type must not degrade into "the workflow is + blocked": that is what made the SDK a DoS amplifier in the first + place (B14) — one unrecognised value from the control plane would + stop every subsequent tool call in the workflow. This class is an + INFRASTRUCTURE error: the control plane sent something the SDK + cannot execute, and the fix is a version alignment, not a policy + verdict. + + It is deliberately NOT a :class:`NullRunTransportError` subclass, + for the same reason as :class:`NullRunGateIndeterminateError`: + ADR-126's two-step opt-in opens transport errors, and "the body + runs" is not an acceptable answer to "the control plane sent an + instruction this SDK cannot execute". + """ + + error_code = "NR-P005" + user_action = ( + "The NullRun control plane sent an action type this SDK version " + "does not know, so the action was NOT executed. This is a " + "server/SDK version mismatch — upgrade the SDK to a version that " + "understands the control plane's action vocabulary. The " + "workflow itself is not blocked." + ) + retryable = False + + def __init__( + self, + message: str, + *, + action: str | None = None, + workflow_id: str | None = None, + reason: str | None = None, + cause: BaseException | None = None, + **kwargs: Any, + ) -> None: + self.action = action + self.workflow_id = workflow_id + self.reason = reason + super().__init__(message, cause=cause, **kwargs) + + class NullRunChainError(NullRunDecision): """Chain-related failure. Covers backend codes: ``CHAIN_MAX_DURATION_EXCEEDED`` (402), ``CHAIN_CROSS_ORG`` (403), ``CHAIN_ORG_MISMATCH`` (403), + ``CHAIN_KEY_MISMATCH`` (403 — ADR-180: the chain belongs to the + API key that created it), ``CHAIN_NOT_FOUND`` / ``CHAIN_EXPIRED`` (404), and the Execution Graph v0 (2026-08-06) trio: ``PARENT_EXECUTION_NOT_FOUND`` / ``PARENT_EXECUTION_ORG_MISMATCH`` @@ -1333,6 +1468,37 @@ class NullRunApprovalToolDigestMismatchError(NullRunBlockedException): retryable = False +class NullRunApprovalToolMismatchError(NullRunBlockedException): + """The executing tool is not covered by the operator's approval (DEC-5 / DEC-7). + + Wire code ``APPROVAL_TOOL_MISMATCH`` (HTTP 409). The operator + approved an action for one tool; the agent is executing a DIFFERENT + tool under that approval. Before this code existed a money approval + bound only the AMOUNT (``BusinessImpact::Money`` carries no tool + name), so an approval for tool A authorised tool B. + + The backend binds the approval to the tool set the matched approval + RULE covers (intersected with the tools the ``/gate`` request + declared), so the refusal is structural, not advisory. + Cookbook pattern: request a FRESH approval for the tool you + actually intend to run — the existing grant can never authorise a + tool outside its set. + + Distinct from NR-A013 / NR-A014: those mean the PAYLOAD drifted + (business impact / capability digest). This means the TOOL is + different, and a fresh approval — not a re-derived digest — is the + remediation. + """ + + error_code = "NR-A017" + user_action = ( + "Approval tool mismatch — the operator's approval covers a " + "different tool than the one currently executing. Request a " + "fresh approval for the tool you intend to run and retry." + ) + retryable = False + + # ──────────────────────────────────────────────────────────────────────── # # Pre-B.1 these three wire codes (``MCP_DESTRUCTIVE_BLOCKED``, @@ -1604,3 +1770,64 @@ class NullRunWorkflowKilledError(WorkflowKilledInterrupt): "or inspect the reason before retrying." ) retryable = False + + +# --------------------------------------------------------------------------- +# The enforcement-refusal family +# --------------------------------------------------------------------------- +# ONE tuple that means "the enforcement path said NO" — as opposed to +# "the enforcement path could not be reached". +# +# The distinction is the whole point. ADR-008's fail-OPEN policy +# covers the SECOND: a dead backend must not freeze the agent. It +# explicitly does NOT cover the FIRST: conflating "the gate refused" +# with "the gate was unreachable" is what let a real budget block +# through as a debug log line (SDK-4, and the same defect on the +# /track path — SDK-6). +# +# All of these derive from ``Exception``, so an ordering-sensitive +# ``except`` clause is required: this tuple must be caught BEFORE the +# broad transport arm, or the block is silently downgraded to a +# fail-OPEN transport error. +# +# The family is named by BASE class, so a refusal subclass added later +# cannot silently fall back into the fail-OPEN arm: +# * ``NullRunDecision`` — every "the gate said no" refusal: +# tool-block, budget, the approval family, chain, +# workflow-inactive, PAUSE. +# * ``RateLimitError`` — the 429 refusal. It is NOT a +# ``NullRunDecision``: the gate refused, but the refusal wears a +# transport-shaped class because the wire code is a 429. Naming +# ``NullRunDecision`` alone would still have swallowed it. +# * ``WorkflowKilledInterrupt`` — the dashboard KILL switch. It +# derives from ``NullRunError`` directly, NOT from +# ``NullRunDecision``. +# * ``NullRunMalformedGateResponseError`` — the gate ANSWERED, and +# what it said was not a verdict (SDK-10). It is a protocol error +# by class, but the SDK's only safe answer to it is to REFUSE: +# the runtime already decided that a body without a usable +# decision fails CLOSED (``_require_gate_decision``), and the +# reservation id is part of that decision — it is what binds the +# spend to the reservation the server minted. Before this member +# existed, the instrumentation's blanket arms downgraded it to a +# fail-OPEN transport error, so an unusable answer let the call +# run with no reservation and its cost event was DROPPED — the +# agent got the call for free. It is in the family because the +# operational rule the family encodes is the one that matters +# here: none of these may be downgraded to "the gate was +# unreachable, carry on". +# * ``NullRunGateIndeterminateError`` — the decision is +# INDETERMINATE (SDK-11): the gate call produced no decision the +# SDK can act on, and the failure was NOT a transport failure, so +# the SDK cannot tell "the backend said no" from "the SDK never +# found out". Same operational rule: it must not be downgraded to +# a fail-OPEN transport error, because an SDK-internal fault was +# being laundered into "allowed" — the call ran with no +# reservation and its cost event was dropped. +ENFORCEMENT_EXCEPTIONS: tuple[type[BaseException], ...] = ( + NullRunDecision, + RateLimitError, + WorkflowKilledInterrupt, + NullRunMalformedGateResponseError, + NullRunGateIndeterminateError, +) diff --git a/src/nullrun/decorators.py b/src/nullrun/decorators.py index d18ac1a..1958140 100644 --- a/src/nullrun/decorators.py +++ b/src/nullrun/decorators.py @@ -63,6 +63,8 @@ def researcher(q): set_span_id, set_trace_id, ) +from nullrun.observability import metrics +from nullrun.observability.redaction import scrub_secret_values from nullrun.runtime import NullRunRuntime, get_runtime from nullrun.transport import is_fallback_decision_source @@ -149,6 +151,12 @@ def _safe_repr(value: object, max_len: int = 50) -> str: # ``details=`` substring is replaced before we potentially # truncate it away. r = _strip_details_balanced(r) + # Redact credential-shaped VALUES. Key-name redaction only knows + # the argument names someone listed; a secret that arrives under + # a benign name (``{"note": ""}``) survives it. This pass + # runs BEFORE the truncation below, so a credential that straddles + # the cut is caught whole. + r = scrub_secret_values(r) # Truncate to ``max_len`` so a giant repr doesn't bloat span # events. We append ``...`` so consumers can see the # cut happened. @@ -890,15 +898,16 @@ def sync_wrapper(*args: Any, **kwargs: Any) -> Any: def _raise_deferred_enforcement(tool_name: str) -> None: """Raise a gate decision a framework callback could not enforce. - LangChain discards exceptions raised from a callback handler — it - logs ``Error in callback`` and continues. So when - ``NullRunCallback.on_llm_start`` calls ``check_workflow_budget`` - and gets a real block (budget exhausted, workflow KILL/PAUSE), it - cannot abort the LLM call from inside the callback. + A langchain-core handler's exception is swallowed BY DEFAULT — the + framework logs ``Error in callback`` and continues. The + NullRun handler opts out of that default by setting + ``raise_error = True`` (SDK-4), so a real refusal now reaches the + caller and aborts the LLM call. - The callback stashes the decision instead; this runs at the - ``@protect`` boundary, which can abort, and re-raises the oldest - unraised one. + This boundary is the SECOND channel: it drains a decision that was + recorded per-thread by a callback and re-raises it here, where the + SDK can abort on the boundary's own thread. It is the only channel + that survives a host framework which ignores ``raise_error``. The import is deliberately LAZY and failure-tolerant. The stashing side lives in ``instrumentation.langgraph``, which is only imported @@ -953,9 +962,13 @@ def _run_tool_policy_gate( the outage. Opt-out: set ``NULLRUN_SENSITIVE_FAIL_OPEN=1`` to restore fail- - OPEN behavior on transport error (dev / test only). Real - ``decision=block`` from the gateway is still honored and still - raises ``NullRunBlockedException``. + OPEN behavior on transport error (dev / test only) — but only for + a tool the operator classified as fail-open-allowed + (``runtime.register_fail_open_tools([...])``). The switch alone + opens nothing: a tool that is not classified stays fail-CLOSED + even with the variable set. Real ``decision=block`` from the + gateway is still honored and still raises + ``NullRunBlockedException``. ## Wire contract @@ -998,13 +1011,16 @@ def _run_tool_policy_gate( NullRunBlockedException, NullRunDecision, NullRunExecutionNotFoundError, + NullRunGateIndeterminateError, NullRunInfrastructureError, NullRunTransportError, RateLimitError, TransportErrorSource, ) - fail_open = runtime.sensitive_fail_open_enabled() + fail_open = runtime.sensitive_fail_open_enabled() and runtime.is_fail_open_allowed( + fn.__name__ + ) # *display* workflow_id via the runtime's precedence chain # (contextvar → self.workflow_id → None). Sentinel stays as the # last resort for never-bound keys (no workflow context). @@ -1037,7 +1053,8 @@ def _run_tool_policy_gate( logger.warning( f"tool policy gate unavailable for {fn.__name__!r}: " f"{exc.source} on /{exc.endpoint}. " - f"NULLRUN_SENSITIVE_FAIL_OPEN=1 — body will run." + f"NULLRUN_SENSITIVE_FAIL_OPEN=1 and {fn.__name__!r} is " + f"classified as fail-open-allowed — body will run." ) return _code = { @@ -1055,8 +1072,12 @@ def _run_tool_policy_gate( f"The NullRun policy engine is unreachable " f"({exc.source.value}). The body of " f"'{fn.__name__}' did NOT run (fail-CLOSED). " - f"Set NULLRUN_SENSITIVE_FAIL_OPEN=1 to opt out for " - f"tests / staging — production should leave it off." + f"To opt out for tests / staging: set " + f"NULLRUN_SENSITIVE_FAIL_OPEN=1 AND register the tool " + f"as fail-open-allowed " + f"(runtime.register_fail_open_tools(['{fn.__name__}'])) " + f"— the switch alone opens nothing. Production should " + f"leave both off." ), ) runtime._emit_sdk_error( @@ -1080,24 +1101,51 @@ def _run_tool_policy_gate( # first-class attributes. raise except Exception as exc: # noqa: BLE001 - if fail_open: - logger.warning( - f"tool policy gate unavailable for {fn.__name__!r}: " - f"{exc}. NULLRUN_SENSITIVE_FAIL_OPEN=1 — body will run." - ) - return - err = NullRunBlockedException( - workflow_id=workflow_id, - reason=f"policy engine unavailable: {exc}", - tool_name=fn.__name__, - error_code="NR-B001", - user_action=( - f"The NullRun policy engine raised an unexpected " - f"exception during the @protect pre-check of " - f"'{fn.__name__}'. The body did NOT run. Check the " - f"chained exception (raise ... from exc) for the " - f"root cause." - ), + # NOT a transport failure, and NOT a decision the gateway + # named. Every class the arms above recognise is already + # classified, so what lands here is a fault in the SDK's OWN + # code — a request body that cannot be serialised, a bug in + # the signer, a TypeError in the transport's request builder. + # + # The decision is INDETERMINATE, and an indeterminate decision + # is not permission. ADR-063 §2.4's governing rule: a + # fail-OPEN is acceptable only when the check is genuinely + # un-decidable (no network) AND something downstream can + # correct it. Neither holds here: the policy engine WAS + # reachable, and the only thing downstream is `/track` — which + # does not gate on the policy answer, so a lost decision stays + # lost. + # + # Pre-fix this arm did three wrong things at once (SDK-13): + # (a) it reported the fault as NR-B001 — a NETWORK code — + # even though every network failure is classified by + # the arm above and never reaches here; + # (b) it typed the refusal as a policy BLOCK + # (``NullRunBlockedException`` ⊂ ``NullRunDecision``), + # so the FastAPI integration rendered it HTTP 429 / + # category "decision" — the operator was told the policy + # refused (or rate-limited) when the truth is that the + # SDK has a bug; + # (c) under the two-step opt-out it RETURNED, so the body + # ran. That is what the audit's "proceeds" line + # describes, and it is reachable ONLY for SDK-internal + # faults: the /execute path converts every transport + # failure into ``NullRunTransportError``, which the arm + # above owns. The opt-out is documented as covering + # "the policy engine is UNREACHABLE"; an SDK bug is a + # different fact, and nothing downstream corrects it. + # + # This is the /execute twin of ADR-129's row 29: same fact, + # same class, same counter discipline. + metrics.inc_runtime("policy_gate_indeterminate_total") + logger.error( + f"tool policy gate for {fn.__name__!r} failed with a " + f"non-transport error, refusing the call: {exc!r}" + ) + err = NullRunGateIndeterminateError( + "the /execute call failed with a non-transport error, so the " + f"SDK could not determine a decision for {fn.__name__!r}: {exc!r}", + cause=exc, ) runtime._emit_sdk_error( err, @@ -1128,7 +1176,9 @@ def _run_tool_policy_gate( if fail_open: logger.warning( f"tool policy gate for {fn.__name__!r} returned " - f"{decision_source}; NULLRUN_SENSITIVE_FAIL_OPEN=1 — body will run." + f"{decision_source}; NULLRUN_SENSITIVE_FAIL_OPEN=1 " + f"and {fn.__name__!r} is classified as " + f"fail-open-allowed — body will run." ) return _code = { @@ -1146,8 +1196,11 @@ def _run_tool_policy_gate( f"The NullRun policy engine returned a fallback " f"({decision_source}) for '{fn.__name__}'. The " f"body did NOT run. Retry once the policy engine " - f"is back — or set NULLRUN_SENSITIVE_FAIL_OPEN=1 " - f"for tests / staging." + f"is back — or, for tests / staging, set " + f"NULLRUN_SENSITIVE_FAIL_OPEN=1 AND register the " + f"tool as fail-open-allowed " + f"(runtime.register_fail_open_tools(['{fn.__name__}'])) " + f"— the switch alone opens nothing." ), ) runtime._emit_sdk_error( diff --git a/src/nullrun/edge.py b/src/nullrun/edge.py index 5dcd0ab..f792489 100644 --- a/src/nullrun/edge.py +++ b/src/nullrun/edge.py @@ -18,8 +18,14 @@ a disguise: the operator reads "the box said yes" while what happened is that the cloud said yes and the box was never asked. So in edge mode the cloud is not a fallback for the gate — it is not a fallback at all. -A box that cannot be reached produces a *refusal*, and under STRICT (the -default, as everywhere else) that refusal is final. +A box that cannot be reached produces a *refusal*, and that refusal is +final: no fallback mode, no environment variable and no operator +setting turns an unanswered box question into permission. ADR-067 §3 +("an unavailable lease store must degrade to deny, never to allow"), +and the reason it differs from the cloud's fail-OPEN on a transport +error is that the cloud's is recoverable — the spend is still booked +by `/track` — while the edge's is not: there is no cloud to book it +through. **2. Let the caller write the money.** The wire carries a model and token counts, not an amount. The box prices the call from the rates the @@ -65,6 +71,7 @@ import logging import os +import threading import time import uuid from collections.abc import Mapping @@ -72,8 +79,6 @@ import httpx -from nullrun.transport import DecisionSource - logger = logging.getLogger(__name__) # Decision source for a real answer from the box. @@ -86,6 +91,64 @@ # point of E2. EDGE_LEASE = "edge_lease" +# `last_source` marker for "the box answered, and the answer was NO". +# +# NOT a wire `decision_source`. The refusal path still puts +# `EDGE_LEASE` in the returned body's `decision_source`, and that is +# deliberate: `decision_source` says WHO decided, and on a refusal the +# lease was the authority that spoke — the box answered, with "no". +# The direction is carried by `decision`, which is `block`. Changing +# the wire value would need a new name in the runtime's known-source +# set, and until the runtime knows it, a body carrying it is a +# MALFORMED response: a clean refusal would become a crash. +# +# What this constant is for is `last_source`, which answers a +# different question — "which way did the box go on the last call?". +# Before it existed, the allow path and the refusal path both wrote +# `EDGE_LEASE` there, so the two collapsed into one string and a +# caller reading the attribute could not tell a box ALLOW from a box +# REFUSAL. +# +# Deliberately not a fallback source (`is_fallback_decision_source` is +# False for it): a box that said no is a VERDICT, not a degraded +# transport. Reading it as synthetic would discard the refusal before +# the caller saw it — the same hole SDK-3 closed for the other edge +# sources. +EDGE_LEASE_REFUSED = "edge_lease_refused" + +# Decision source for "the box did not answer". +# +# Deliberately NOT `DecisionSource.FALLBACK`, and that is the whole +# fix for the SDK-3 hole. +# +# `is_fallback_decision_source` is the runtime's fail-OPEN predicate: +# a body whose source is a fallback source means "the engine never +# answered", and ADR-008 lets the budget pre-flight proceed on it (the +# cloud's `/track` can still bill the call later, so a dead backend +# must not freeze the agent's loop). +# +# The edge is the opposite case and the difference is not stylistic: +# in edge mode the BOX is the enforcement surface, the cloud is not +# consulted at all, and there is no `/track` recovery — the cloud the +# spend would have been recovered through is the same one that is down. +# An unanswered box question therefore has to stay a refusal. +# +# Reusing FALLBACK for it made the two surfaces indistinguishable, and +# the runtime — which can only see the source — discarded the edge's +# refusal and reported "no decision". A source the runtime cannot +# mistake for a transport error is what makes the refusal final. +# (ADR-067 §3: "an unavailable lease store must degrade to deny, never +# to allow".) +EDGE_UNREACHABLE = "edge_unreachable" + +# Decision source for "the box was never asked about this call". +# +# The difference from EDGE_UNREACHABLE is who is at fault: there the +# box was asked and could not answer; here nobody asked. Both refuse, +# and neither is a fallback source, because in edge mode there is no +# other surface that could have said yes. +EDGE_NOT_ENFORCED = "edge_not_enforced" + # Default timeout for one enforcement call. # # Short on purpose. The box is on the customer's LAN and answers in @@ -134,7 +197,48 @@ def __init__( # Named source instead of a bool, because a bare flag here # would be indistinguishable from "the box said no" at every # call site, and those two must not collapse. + # + # It answers "which way did the box go on the last call?": + # `EDGE_LEASE` for an allow, `EDGE_LEASE_REFUSED` for a + # refusal, `EDGE_UNREACHABLE` for no answer, and + # `EDGE_NOT_ENFORCED` for a box that was never asked. self.last_source: str | None = None + # The box's answers, keyed by the event id they were asked for. + # + # This is a CACHE of the box's answer, not a decision source. + # The enforcement point in edge mode is the box's `/enforce` + # call; `/execute` has no box equivalent and must not invent + # one, so it reads back what the box said for this event. + # + # Read with `take_answer`, which POPS. A pop is what makes a + # stale answer impossible: an answer that was never consumed + # (a pre-flight with no matching /execute) cannot be handed to + # a later action, and a later action that never asked the box + # gets "no answer" and is refused rather than allowed on the + # strength of somebody else's decision. + self._answers: dict[str, dict[str, Any]] = {} + # The lock that guards `_answers`. + # + # There is one EdgeTransport per runtime and one runtime per + # agent, but an agent's tool calls share the runtime — so this + # dict is genuinely multi-threaded. Without the lock a thread + # that evicts from it can be resized by another thread between + # the `iter()` and the `next()`, and CPython raises + # `RuntimeError: dictionary changed size during iteration` + # (or the eviction's `pop` raises `KeyError` when the other + # thread took the key first). + # + # Why that is more than a crash: the exception escapes + # `enforce`, and the gate's fail-OPEN arm catches it and + # returns "no decision" — so the tool body runs with no policy + # decision at all, while the box has already charged the + # lease. SDK-9. + self._answers_lock = threading.Lock() + # Bound on the cache. Entries are popped on use; the cap only + # matters for a pre-flight that is never followed by an + # /execute, and it exists so a long-lived transport cannot grow + # this dict without limit. + self._answer_cap = 64 # -- the call ------------------------------------------------------- @@ -155,12 +259,31 @@ def enforce( Never raises for a business answer, and does not raise for a missing one either: an unreachable box comes back as a `block` - carrying `decision_source = FALLBACK`, which the runtime's - existing STRICT/PERMISSIVE handling turns into the right thing - without a second policy living here. PERMISSIVE therefore still - means what it means everywhere else — the caller has explicitly - accepted running unenforced — and it is the only way this - returns a non-block for an unanswered question. + carrying `decision_source = EDGE_UNREACHABLE`. + + That source is the load-bearing part. It is NOT a fallback + source, so the runtime honours the block instead of reading it + as "the engine never answered" and proceeding (SDK-3). The + cloud's own fail-OPEN on a transport error (ADR-008) is + *recoverable* — the spend is still booked by `/track` — while + the edge's is not: in edge mode there is no cloud to book it + through, and ADR-067 §3 requires an unavailable lease store to + degrade to deny. A refusal that a caller can switch off with + PERMISSIVE would not be a refusal. + + `EDGE_LEASE` is the source for every answer the box actually + gave; `EDGE_UNREACHABLE` is the source for the absence of one. + The `decision` field says which way the box went. + + The transport's own `last_source` says the same thing at a + finer grain: `EDGE_LEASE` for an allow, `EDGE_LEASE_REFUSED` + for a refusal. The two paths used to write the same string + there, so a caller reading `last_source` could not tell a box + ALLOW from a box REFUSAL. + + The answer is recorded under the event id, so the `/execute` + half of the same logical action can read back exactly what the + box said about it (see `take_answer`). """ body: dict[str, Any] = { "lease_id": self.lease_id, @@ -174,6 +297,64 @@ def enforce( # is the expensive bug the whole mechanism exists to prevent. body["event_id"] = event_id or str(uuid.uuid4()) + answer = self._ask(body, on_transport_error) + self._record(body["event_id"], answer) + return answer + + def take_answer(self, event_id: str | None) -> dict[str, Any] | None: + """The box's answer for this event, consumed. + + ``None`` means the box was never asked about this event, which + is a refusal case and not a "no opinion" case: the caller must + not proceed on an event nobody priced. + """ + if not event_id: + return None + with self._answers_lock: + return self._answers.pop(event_id, None) + + def not_enforced(self, execution_id: str | None = None) -> dict[str, Any]: + """No answer from the box for this call. Fail CLOSED. + + The box was never asked — not "it could not answer", but + "nobody asked". The cloud has the same rule and the same + reason: a direct `/execute` with no prior `/gate` is answered + with 404 EXECUTION_NOT_FOUND, because the backend's + `execution:{id}` binding does not exist. Answering "allow" + here would be the same hole with a different spelling. + """ + self.last_source = EDGE_NOT_ENFORCED + return { + "decision": "block", + "decision_source": EDGE_NOT_ENFORCED, + "error_code": "EDGE_NOT_ENFORCED", + "explanation": ( + "The edge box was never asked about this call. NullRun " + "refuses rather than allowing a call no lease was " + "consulted about — the same rule the cloud applies when " + "/execute arrives with no prior /gate." + ), + "suggestions": [ + "Run the pre-flight check for this call before executing it.", + ], + "edge_error_code": None, + "execution_id": execution_id, + } + + def _record(self, event_id: str, answer: dict[str, Any]) -> None: + with self._answers_lock: + self._answers[event_id] = answer + while len(self._answers) > self._answer_cap: + # Copy the key list before evicting. The oldest entry is + # chosen from the snapshot, never from a live iteration + # of a dict another thread may be resizing. + keys = list(self._answers) + if not keys: + break + self._answers.pop(keys[0], None) + + def _ask(self, body: dict[str, Any], on_transport_error: Any) -> dict[str, Any]: + """One round trip to the box. Returns a gate-shaped dict.""" try: response = self._post(body) except Exception as exc: # noqa: BLE001 — any failure is one fact @@ -225,7 +406,13 @@ def enforce( } code = payload.get("error_code") or "EDGE_DENIED" - self.last_source = EDGE_LEASE + # The refusal path reports its OWN source. It used to write + # `EDGE_LEASE` here — the same string the allow path writes — + # so `last_source` could not tell a box ALLOW from a box + # REFUSAL. The wire `decision_source` below stays `EDGE_LEASE` + # on purpose: the lease was the authority that spoke either + # way, and `decision` says which way it went. + self.last_source = EDGE_LEASE_REFUSED logger.info("Edge refused the call: %s", code) return { "decision": "block", @@ -281,12 +468,18 @@ def _headers(self, body: dict[str, Any]) -> dict[str, str]: # -- failure -------------------------------------------------------- def _unreachable(self, exc: Exception, on_transport_error: Any) -> dict[str, Any]: - """No answer from the box. Fail CLOSED unless explicitly told not to. + """No answer from the box. Fail CLOSED, and stay failed. There is no cloud fallback here, and that is the point. See the module docstring: falling back would report the cloud's answer under the box's name, and in the outage this exists for there is no cloud to ask. + + The source is `EDGE_UNREACHABLE`, not `FALLBACK`, and the + runtime treats the two differently on purpose — see the + constant's comment. A `FALLBACK` here would be read as "the + engine never answered, carry on" (ADR-008's cloud rule) and the + refusal would be discarded before the caller ever saw it. """ message = f"{type(exc).__name__}: {exc}" logger.warning( @@ -302,9 +495,10 @@ def _unreachable(self, exc: Exception, on_transport_error: Any) -> dict[str, Any # change the decision. It is a reporter, not a gate. logger.debug("on_transport_error raised; ignoring", exc_info=True) + self.last_source = EDGE_UNREACHABLE return { "decision": "block", - "decision_source": DecisionSource.FALLBACK, + "decision_source": EDGE_UNREACHABLE, "error_code": "EDGE_UNREACHABLE", "explanation": ( f"The edge box at {self.base_url} did not answer ({message}). " @@ -438,6 +632,9 @@ def _int_env(source: Mapping[str, str], name: str, default: int) -> int: __all__ = [ "EDGE_LEASE", + "EDGE_LEASE_REFUSED", + "EDGE_UNREACHABLE", + "EDGE_NOT_ENFORCED", "DEFAULT_EDGE_TIMEOUT_SECONDS", "EdgeConfigurationError", "EdgeTransport", diff --git a/src/nullrun/instrumentation/auto.py b/src/nullrun/instrumentation/auto.py index b4c3026..f1f5b64 100644 --- a/src/nullrun/instrumentation/auto.py +++ b/src/nullrun/instrumentation/auto.py @@ -48,7 +48,9 @@ import httpx +from nullrun.breaker.exceptions import ENFORCEMENT_EXCEPTIONS from nullrun.instrumentation.langgraph import NullRunCallback +from nullrun.transport import is_backend_refusal logger = logging.getLogger(__name__) @@ -953,11 +955,9 @@ def _preflight_llm_call( # one-token reserve is the smallest honest number then. estimated_tokens=profile.estimated_input_tokens or None, max_output_tokens=profile.declared_max_output_tokens, - # Each provider call needs its OWN reservation. The chain-mode - # cache replays a previous response — including its reservation - # id — so a cached "allow" would hand this call the previous - # call's reservation. - use_gate_cache=False, + # Each provider call gets its OWN reservation: there is no + # in-process gate cache, so every call here is a separate + # /gate call that mints its own reservation id (SDK-1). ) if not isinstance(reservation_id, str) or not reservation_id: return (None, profile.model) @@ -975,8 +975,10 @@ def _track_with_reservation( The call shape is unchanged for the no-reservation case, so a duck-typed runtime (a test double, an older SDK) that implements ``track(event)`` keeps working. Passing a keyword it does not - accept would raise, and the transport swallows track failures — - which would silently drop the cost event. + accept would raise, and the transport swallows a track failure + that is NOT an enforcement refusal — which would silently drop + the cost event. An enforcement refusal is re-raised by the caller + (SDK-6): the backend said no, and the application must see it. """ if reservation_id: runtime.track(event, reservation_id=reservation_id) @@ -1269,7 +1271,26 @@ def _emit( _build_llm_call_event(host, usage, model_for_event), reservation_id, ) + except ENFORCEMENT_EXCEPTIONS: + # SDK-6 (reliability audit 2026-11-02): the backend REFUSED + # to book this call (CONSUME_OVERBUDGET, a budget block, a + # 429). ADR-008's /track row is CLOSED for enforcement + # rejections — the refusal propagates to the caller. The + # broad arm below used to swallow it, so a refused track + # looked like a success to the application. + raise except Exception as e: + # SDK-1 (security audit 2026-11-06). The family above is a + # list of CLASSES, and a backend refusal that is not a + # verdict — a 503 `RESERVATION_NOT_FOUND`, a 500 + # `INTERNAL_ERROR` — is not in it. The backend ANSWERED and + # refused to book the money, so the refusal has to reach + # the caller exactly like a verdict does. + # + # The test is the WIRE, not the class: the exception + # carries the HTTP status of the answer. + if is_backend_refusal(e): + raise logger.debug("NullRun transport: track failed: %s", e) def close(self) -> None: @@ -1400,7 +1421,17 @@ def _emit( _build_llm_call_event(host, usage, model_for_event), reservation_id, ) + except ENFORCEMENT_EXCEPTIONS: + # SDK-6 — see the sync transport above. An enforcement + # refusal is not a transport failure: it must reach the + # caller. + raise except Exception as e: + # SDK-1 (security audit 2026-11-06) — see the sync + # transport above. A backend answer that refused the + # consume is not a transport failure either. + if is_backend_refusal(e): + raise logger.debug("NullRun transport: async track failed: %s", e) async def aclose(self) -> None: @@ -2058,7 +2089,17 @@ def _emit_from_agents_result(runtime: Any, result: Any) -> None: if span_model: agents_event["model"] = span_model runtime.track(agents_event) + except ENFORCEMENT_EXCEPTIONS: + # SDK-6 — the backend refused to book the money. It must + # reach the caller; a refused track must not look like a + # successful one. + raise except Exception as e: # pragma: no cover — defensive + # SDK-1 (security audit 2026-11-06) — see the sync + # transport above. A backend answer that refused the + # consume is not a transport failure either. + if is_backend_refusal(e): + raise logger.debug("NullRun: agents track failed: %s", e) diff --git a/src/nullrun/instrumentation/auto_requests.py b/src/nullrun/instrumentation/auto_requests.py index dd759b0..be06fd0 100644 --- a/src/nullrun/instrumentation/auto_requests.py +++ b/src/nullrun/instrumentation/auto_requests.py @@ -45,6 +45,7 @@ import threading from typing import Any +from nullrun.breaker.exceptions import ENFORCEMENT_EXCEPTIONS from nullrun.instrumentation.auto import ( _fingerprint_for, _match_extractor, @@ -122,6 +123,13 @@ def _emit_to_runtime( }, reservation_id, ) + except ENFORCEMENT_EXCEPTIONS: + # SDK-6 (reliability audit 2026-11-02): the backend REFUSED to + # book this call. ADR-008's /track row is CLOSED for + # enforcement rejections — the refusal propagates to the + # caller. The broad arm below used to swallow it, so a refused + # track looked like a success to the application. + raise except Exception as e: logger.debug("NullRun requests transport: track failed: %s", e) diff --git a/src/nullrun/instrumentation/autogen.py b/src/nullrun/instrumentation/autogen.py index b161575..501e462 100644 --- a/src/nullrun/instrumentation/autogen.py +++ b/src/nullrun/instrumentation/autogen.py @@ -20,6 +20,8 @@ from collections.abc import Callable from typing import Any +from nullrun.breaker.exceptions import ENFORCEMENT_EXCEPTIONS + logger = logging.getLogger(__name__) _autogen_patched = False @@ -136,6 +138,13 @@ def _wrap_create(self: Any, *args: Any, **kwargs: Any) -> Any: if model: event["model"] = model runtime.track(event) + except ENFORCEMENT_EXCEPTIONS: + # SDK-6 (reliability audit 2026-11-02): the + # backend REFUSED to book this call. The + # refusal propagates to the caller — a + # refused track must not look like a + # success. + raise except Exception as e: # pragma: no cover logger.debug("autogen create emit failed: %s", e) return result diff --git a/src/nullrun/instrumentation/crewai.py b/src/nullrun/instrumentation/crewai.py index 5c6239b..74c410c 100644 --- a/src/nullrun/instrumentation/crewai.py +++ b/src/nullrun/instrumentation/crewai.py @@ -32,6 +32,8 @@ from collections.abc import Callable from typing import Any +from nullrun.breaker.exceptions import ENFORCEMENT_EXCEPTIONS + logger = logging.getLogger(__name__) _crewai_patched = False @@ -72,6 +74,12 @@ def _emit_usage_metrics(runtime: Any, crew: Any) -> None: "raw_usage": dict(m), } ) + except ENFORCEMENT_EXCEPTIONS: + # SDK-6 (reliability audit 2026-11-02): the backend REFUSED + # to book this call. ADR-008's /track row is CLOSED for + # enforcement rejections — the refusal propagates to the + # caller, so a refused track cannot look like a success. + raise except Exception as e: # pragma: no cover - defensive logger.debug("crewai usage_metrics emit failed: %s", e) diff --git a/src/nullrun/instrumentation/langgraph.py b/src/nullrun/instrumentation/langgraph.py index d24d3cd..418a01d 100644 --- a/src/nullrun/instrumentation/langgraph.py +++ b/src/nullrun/instrumentation/langgraph.py @@ -29,6 +29,8 @@ import threading from typing import Any +import httpx + from nullrun.runtime import get_runtime from nullrun.tracing import ( SpanContext, @@ -38,42 +40,112 @@ ) # Exceptions that represent a REAL enforcement decision from the gate, -# as opposed to a failure to reach the gate. `NullRunBudgetError` is -# the budget-exhausted block, `WorkflowKilledInterrupt` / -# `WorkflowPausedException` are the dashboard KILL/PAUSE switches. All -# three derive from `Exception`, so an ordering-sensitive `except` -# clause is required: this tuple must be caught BEFORE the broad -# transport arm, or the block is silently downgraded to a fail-OPEN -# transport error. +# as opposed to a failure to reach the gate. All of them derive from +# `Exception`, so an ordering-sensitive `except` clause is required: +# this tuple must be caught BEFORE the broad transport arm, or the +# block is silently downgraded to a fail-OPEN transport error. # # ADR-008's fail-OPEN policy covers "the policy engine could not be # reached". It explicitly does NOT cover "the policy engine returned # `block`" — conflating the two is what let a real budget block # through as a debug log line. +# +# SDK-4 (reliability audit 2026-11-02) widened this family. The old +# tuple named THREE concrete classes, and every other refusal the gate +# can issue fell into the broad transport arm and was dropped at DEBUG: +# +# * `TOOL_BLOCKED` -> `NullRunToolBlockedError` -> `NullRunBlockedException` +# -> `NullRunDecision`. NOT in the old tuple, so a +# policy refusal was logged at debug and the agent +# continued. +# * `RATE_LIMIT_EXCEEDED` -> `RateLimitError` -> `NullRunTransportError` +# -> `NullRunInfrastructureError`. It is NOT a +# `NullRunDecision` — the gate refused, but the +# refusal wears a transport-shaped class because +# the wire code is a 429. Naming `NullRunDecision` +# alone would still have swallowed it. +# +# The family is therefore named by BASE class, so a refusal subclass +# added later cannot silently fall back into the fail-OPEN arm: +# * `NullRunDecision` — every "the gate said no" refusal: tool-block, +# budget, the approval family, chain, workflow-inactive, PAUSE. +# * `RateLimitError` — the 429 refusal, which is transport-shaped. +# * `WorkflowKilledInterrupt` — the dashboard KILL switch. It derives +# from `NullRunError` directly, NOT from `NullRunDecision`. +# +# The family itself now lives in ONE place — +# `nullrun.breaker.exceptions.ENFORCEMENT_EXCEPTIONS`. The /track path +# needs the SAME rule (SDK-6), and two hand-written copies of the +# family are two places to forget a refusal subclass. +# +# SDK-16 (reliability audit 2026-11-02, SDK-16 row of the handoff +# ledger) adds the SECOND half of the classification the callback +# needs. The family alone was not enough: `check_workflow_budget` +# RAISES four classes the family does not name — a 401 +# (`NullRunAuthError`), an unreadable answer +# (`NullRunMalformedGateResponseError`, already in the family), an +# unclassifiable refusal (`NullRunUnclassifiedRefusalError`) and a +# rate-limit backend that is down (`NullRunRateLimitRedisError`) — +# and the callback swallowed all of them in a broad arm while the +# `@protect` boundary re-raised them. The same refusal then killed +# the call or not depending on which path it arrived through. +# +# The callback's classification is now the SAME four arms the +# `@protect` boundary uses (`decorators.py::_run_tool_policy_gate`), +# in the same order: +# +# 1. the family -> a REFUSAL: it reaches the caller +# 2. a transport error -> the gate could not be REACHED: +# ADR-008's fail-OPEN (a dead backend must +# not freeze the agent) +# 3. any other infra -> the gate ANSWERED, or the SDK could not +# read the answer: fail-CLOSED +# 4. anything else -> a cosmetic fault in the SDK's own code: +# swallowed, because a callback must never +# break the host framework +# +# The order is load-bearing in BOTH directions: +# `RateLimitError` is a `NullRunTransportError` (so arm 2 would +# swallow the 429), and `NullRunTransportError` is a +# `NullRunInfrastructureError` (so arm 3 would turn a dead backend +# into a refusal and freeze the agent). try: # pragma: no cover - import shape, not behaviour from nullrun.breaker.exceptions import ( - NullRunBudgetError, - WorkflowKilledInterrupt, - WorkflowPausedException, + ENFORCEMENT_EXCEPTIONS, + NullRunInfrastructureError, + NullRunTransportError, ) - _ENFORCEMENT_EXCEPTIONS: tuple[type[BaseException], ...] = ( - NullRunBudgetError, - WorkflowKilledInterrupt, - WorkflowPausedException, + _ENFORCEMENT_EXCEPTIONS: tuple[type[BaseException], ...] = ENFORCEMENT_EXCEPTIONS + _TRANSPORT_ERRORS: tuple[type[BaseException], ...] = ( + httpx.HTTPError, + NullRunTransportError, ) + _INFRASTRUCTURE_ERRORS: tuple[type[BaseException], ...] = (NullRunInfrastructureError,) except ImportError: # pragma: no cover # The SDK is unusable without its own exception types, but never # let their absence break `import nullrun` for a consumer who # does not use LangChain. Empty tuple => every raise is treated # as a transport error, i.e. the pre-a626bbd behaviour. _ENFORCEMENT_EXCEPTIONS = () # type: ignore[assignment] + _TRANSPORT_ERRORS = () # type: ignore[assignment] + _INFRASTRUCTURE_ERRORS = () # type: ignore[assignment] -# Deferred-enforcement handoff. LangChain swallows exceptions raised -# from a callback, so a block discovered in `on_llm_start` cannot abort -# the LLM call from here. The decision is stashed per-thread; the -# `@protect` boundary (which CAN abort) drains it and raises. +# Deferred-enforcement handoff — the SECONDARY channel. +# +# The PRIMARY channel is the raise: `on_llm_start` re-raises the gate +# refusal, and `NullRunCallback.raise_error = True` makes +# langchain-core propagate it to the caller instead of swallowing it +# (SDK-4). That channel works across the thread hop the async +# executor path forces, which a thread-local CANNOT: the stash is +# written on the callback's thread and read on the `@protect` +# boundary's thread, and those are different threads whenever the +# graph is async. +# +# The stash is kept as the fallback for a host framework that ignores +# `raise_error`: the refusal is recorded per-thread and the `@protect` +# boundary (which CAN abort on its own thread) drains and raises it. # # Thread-local, not global: LangGraph runs concurrent chains in a # thread pool, and one chain's budget block must not abort another's. @@ -116,6 +188,90 @@ def drain_deferred_enforcement() -> BaseException | None: except Exception: # noqa: BLE001 — best-effort by construction return None + +def _swallow_callback_failure(fn: Any, exc: BaseException) -> None: + """Log a callback failure that must not break the host framework.""" + logger.debug( + "NullRunCallback.%s failed: %s — swallowed so the host " + "framework keeps running", + getattr(fn, "__name__", "callback"), + exc, + ) + + +def _only_a_gate_refusal_escapes(fn: Any) -> Any: + """A callback must never break the host framework — except for a refusal. + + SDK-4 (reliability audit 2026-11-02) turns ``NullRunCallback`` into + a handler that opts INTO propagation: langchain-core re-raises a + handler's exception when ``handler.raise_error`` is True + (``callbacks/manager.py::handle_event`` and + ``_ahandle_event_for_handler`` — both paths, sync and async). + + That opt-in is only safe if exactly ONE thing can escape a + NullRun callback: a real gate refusal. Without this guard a + cosmetic failure — a span bookkeeping bug, a malformed response + object — would abort the user's agent, which is a far worse + failure than the one being fixed. + + So every callback method is wrapped: a gate refusal is re-raised + (it must reach the caller), anything else is logged at debug and + swallowed (the pre-existing "best-effort" contract of these + callbacks, unchanged). + + SDK-16 (the handoff ledger's SDK-16 row) makes "a real gate + refusal" mean what the SDK's own fail-OPEN/CLOSED table says it + means. The guard used to re-raise the enforcement FAMILY and + swallow everything else, and three classes the runtime raises on + purpose — a 401, an unclassifiable refusal, a rate-limit backend + that is down — are not in that family. They are not transport + failures either: the gate answered, or the SDK could not read the + answer. The ``@protect`` boundary already re-raised them, so the + same refusal killed the call through one path and vanished + through the other. + + The arms are therefore the four of the ``@protect`` boundary, in + the same order: the family re-raised, a transport failure + swallowed (ADR-008's fail-OPEN), any other infrastructure error + re-raised (fail-CLOSED), and everything else swallowed (a + cosmetic failure must never break the host framework). + + ``functools.wraps`` keeps ``__wrapped__``, so + ``inspect.getsource`` still returns the METHOD's own source — the + AST pin in ``tests/test_langchain_enforcement.py`` keeps reading + the real ``except`` arms of ``on_llm_start`` and not this + wrapper's. + """ + import functools + + @functools.wraps(fn) + def _wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + try: + return fn(self, *args, **kwargs) + except _ENFORCEMENT_EXCEPTIONS: + # A real refusal. It MUST reach the caller. + raise + except _TRANSPORT_ERRORS as exc: # noqa: BLE001 + # The gate could not be REACHED. ADR-008's fail-OPEN: a + # dead backend must not freeze the agent. Swallowed, as it + # always was. + _swallow_callback_failure(fn, exc) + return None + except _INFRASTRUCTURE_ERRORS: + # The gate ANSWERED, or the SDK could not read the answer: + # a 401, an unreadable body, a refusal the SDK cannot + # classify, a rate-limit backend that is down. The SDK's + # own table says fail-CLOSED for everything that is not a + # transport failure, and the ``@protect`` boundary obeys + # it. The callback must not be the one path where the same + # refusal is allowed through. + raise + except Exception as exc: # noqa: BLE001 — a callback never breaks the chain + _swallow_callback_failure(fn, exc) + return None + + return _wrapper + # DEF-MP-TS12-SDK-05 (2026-09-29): `langchain-core` is a `dev` extra, # NOT a core dependency (`pyproject.toml` core deps are httpx only). # This module used to import it unconditionally at module scope, and @@ -511,6 +667,32 @@ class NullRunCallback(BaseCallbackHandler): inside an `@protect`-wrapped function is properly nested. """ + # SDK-4 (reliability audit 2026-11-02): opt INTO propagation. + # + # langchain-core SWALLOWS an exception raised by a callback handler + # unless the handler sets `raise_error = True`: + # + # * sync — `callbacks/manager.py::handle_event` catches the + # handler's exception, logs "Error in callback", and + # re-raises iff `handler.raise_error`. + # * async — `callbacks/manager.py::_ahandle_event_for_handler` + # does the same for a SYNC handler run through the executor. + # + # (Verified against langchain-core 1.5.6. The async ASYNC-handler + # path, `_run_coros`, swallows unconditionally — but this handler + # is synchronous, so it never goes through that path.) + # + # Without this flag a real gate refusal raised in `on_llm_start` + # was logged and dropped, and the agent carried on — the run + # COMPLETED after the gate had refused it. + # + # The flag is only safe because `_only_a_gate_refusal_escapes` + # guarantees that a gate refusal is the ONLY thing that can leave + # a NullRun callback. Every other failure is still swallowed, so + # the "never break the host chain" contract of these callbacks is + # unchanged. + raise_error = True + def __init__(self, runtime: Any | None = None) -> None: self.runtime = runtime or get_runtime() # run_id -> SpanContext for in-flight chain / tool / agent @@ -581,6 +763,7 @@ def _register_active_run(self, run_id: str, ctx: SpanContext) -> None: # LLM hooks (existing — token extraction only, no span bookkeeping) # ------------------------------------------------------------------ + @_only_a_gate_refusal_escapes def on_llm_start(self, serialized: Any, prompts: Any, **kwargs: Any) -> None: """ Called when LLM call starts. @@ -663,36 +846,49 @@ def on_llm_start(self, serialized: Any, prompts: Any, **kwargs: Any) -> None: try: self.runtime.check_workflow_budget() except _ENFORCEMENT_EXCEPTIONS as exc: - # A REAL gate decision — budget exhausted, workflow killed, - # workflow paused. NOT a transport failure, so the ADR-008 - # fail-OPEN policy does not apply: that policy covers - # "the gate could not be reached", never "the gate said no". + # A REAL gate decision — tool-blocked, rate-limited, budget + # exhausted, workflow killed, workflow paused. NOT a + # transport failure, so the ADR-008 fail-OPEN policy does + # not apply: that policy covers "the gate could not be + # reached", never "the gate said no". # # The pre-fix `except BaseException` lumped these together # with transport errors and dropped them to `logger.debug`, # so a genuine budget block became an invisible debug line # and the LLM call proceeded anyway. # - # Re-raising here would NOT work: LangChain swallows - # exceptions raised from a callback handler (verified - # against langchain-core 1.5.6 — `on_llm_start` logs - # "Error in callback" and continues). So the - # honest options are (a) record the decision so the - # enforcement surface can act on it, and (b) make it - # loud. We do both: the decision is stashed on the - # runtime for `@protect` to raise at a point that CAN - # abort, and it is logged at ERROR rather than debug. + # SDK-4 (reliability audit 2026-11-02): the refusal is + # RE-RAISED. The old comment here said re-raising "would + # NOT work" because langchain swallows callback + # exceptions — true for a default handler, but this class + # now sets `raise_error = True`, which is exactly the + # opt-in langchain-core honours in BOTH the sync and the + # async dispatch path. So the refusal leaves the callback + # and reaches the caller, and the LLM call is aborted + # instead of running with no reservation. + # + # Before SDK-4 the tuple named only the budget / KILL / + # PAUSE classes, so a `TOOL_BLOCKED` or a `RATE_LIMIT` + # refusal fell into the broad arm below and was DROPPED — + # the run completed after the gate had refused it. + # + # The decision is ALSO recorded for the `@protect` + # boundary: that is the fallback for a host framework + # that ignores `raise_error`, and it is the only channel + # that can abort on the boundary's own thread. logger.error( "NullRunCallback.on_llm_start: gate returned a real " "decision (%s: %s). This LLM call has no reservation; " "its cost event will be dropped by runtime._route_track. " - "Enforcement of this decision happens at the @protect " - "boundary, which CAN abort.", + "The refusal is re-raised so it reaches the caller and " + "aborts the LLM call; it is also recorded for the " + "@protect boundary.", type(exc).__name__, exc, ) record_deferred_enforcement(exc) - except Exception as exc: # noqa: BLE001 — never raise out of callback + raise + except _TRANSPORT_ERRORS as exc: # noqa: BLE001 — never raise out of callback # Genuine transport failure (backend unreachable, timeout, # 5xx). Fail-OPEN is the DOCUMENTED ADR-008 policy for # this class: a dead backend must not freeze the agent, @@ -703,6 +899,47 @@ def on_llm_start(self, serialized: Any, prompts: Any, **kwargs: Any) -> None: "cost event will be dropped by runtime._route_track)", type(exc).__name__, ) + except _INFRASTRUCTURE_ERRORS as exc: + # SDK-16: the gate ANSWERED, or the SDK could not read the + # answer, and the refusal is not a transport failure — a + # 401, an unclassifiable refusal, a rate-limit backend that + # is down. `check_workflow_budget` raises these on + # purpose, and the SDK's own fail-OPEN/CLOSED table says + # CLOSED for everything that is not a transport failure. + # The `@protect` boundary already obeyed that; this arm + # used to be the broad one below, so the same refusal + # killed the call through one path and vanished through + # the other. + # + # The LLM call must not run: it has no reservation, so its + # cost event would be dropped and the agent would get the + # call for free. + logger.error( + "NullRunCallback.on_llm_start: the gate could not be " + "used (%s: %s). This LLM call has no reservation; its " + "cost event will be dropped by runtime._route_track. " + "The refusal is re-raised so it reaches the caller and " + "aborts the LLM call; it is also recorded for the " + "@protect boundary.", + type(exc).__name__, + exc, + ) + record_deferred_enforcement(exc) + raise + except Exception as exc: # noqa: BLE001 — never raise out of callback + # A fault in the SDK's OWN code that is not a NullRun + # error — a broken runtime double, a callback bug. The + # runtime converts its own faults into a typed refusal + # (`NullRunGateIndeterminateError`, ADR-129) before they + # can reach here, so this arm is not a decision path: it + # is the callback's own "never break the host framework" + # contract. + logger.debug( + "NullRunCallback.on_llm_start: check_workflow_budget " + "raised %s — proceeding without reservation (llm_call " + "cost event will be dropped by runtime._route_track)", + type(exc).__name__, + ) try: self.runtime.track_event( event_type="span_start", @@ -716,6 +953,7 @@ def on_llm_start(self, serialized: Any, prompts: Any, **kwargs: Any) -> None: except Exception as exc: # noqa: BLE001 logger.debug(f"llm span_start emission failed: {exc}") + @_only_a_gate_refusal_escapes def on_llm_end(self, response: Any, **kwargs: Any) -> None: """ Called when LLM call ends. @@ -907,6 +1145,29 @@ def on_llm_end(self, response: Any, **kwargs: Any) -> None: else: logger.debug(f"LLM tracked: model={model}, NO usage data available") + except _ENFORCEMENT_EXCEPTIONS as e: + # SDK-6 (reliability audit 2026-11-02): the backend REFUSED + # to book this call's money (CONSUME_OVERBUDGET, a budget + # block, a 429). ADR-008's /track row is CLOSED for + # enforcement rejections — the refusal propagates to the + # caller. The broad arm below used to swallow it here, so + # the refusal never even reached + # ``@_only_a_gate_refusal_escapes`` and the run finished + # as if the money had been booked. + # + # It is ALSO recorded for the ``@protect`` boundary: that + # is the channel that can abort on the boundary's own + # thread when the host framework ignores ``raise_error``. + logger.error( + "NullRunCallback.on_llm_end: the backend refused to book " + "this LLM call (%s: %s). The refusal is re-raised so it " + "reaches the caller; it is also recorded for the " + "@protect boundary.", + type(e).__name__, + e, + ) + record_deferred_enforcement(e) + raise except Exception as e: logger.warning(f"Failed to track LLM event: {e}") finally: @@ -929,6 +1190,7 @@ def on_llm_end(self, response: Any, **kwargs: Any) -> None: # Chain / tool / agent hooks — emit span events # ------------------------------------------------------------------ + @_only_a_gate_refusal_escapes def on_chain_start( self, serialized: Any, @@ -949,12 +1211,15 @@ def on_chain_start( str(run_id), str(parent_run_id) if parent_run_id else None, name, kind="chain" ) + @_only_a_gate_refusal_escapes def on_chain_end(self, outputs: Any, *, run_id: Any = None, **kwargs: Any) -> None: self._end_run(run_id) + @_only_a_gate_refusal_escapes def on_chain_error(self, error: Any, *, run_id: Any = None, **kwargs: Any) -> None: self._end_run(run_id, error=str(error)) + @_only_a_gate_refusal_escapes def on_tool_start( self, serialized: Any, @@ -973,12 +1238,15 @@ def on_tool_start( str(run_id), str(parent_run_id) if parent_run_id else None, name, kind="tool" ) + @_only_a_gate_refusal_escapes def on_tool_end(self, output: Any, *, run_id: Any = None, **kwargs: Any) -> None: self._end_run(run_id) + @_only_a_gate_refusal_escapes def on_tool_error(self, error: Any, *, run_id: Any = None, **kwargs: Any) -> None: self._end_run(run_id, error=str(error)) + @_only_a_gate_refusal_escapes def on_agent_action( self, action: Any, @@ -998,6 +1266,7 @@ def on_agent_action( kind="agent", ) + @_only_a_gate_refusal_escapes def on_agent_finish(self, finish: Any, *, run_id: Any = None, **kwargs: Any) -> None: self._end_run(run_id) diff --git a/src/nullrun/instrumentation/llama_index.py b/src/nullrun/instrumentation/llama_index.py index 215dd70..929c807 100644 --- a/src/nullrun/instrumentation/llama_index.py +++ b/src/nullrun/instrumentation/llama_index.py @@ -16,6 +16,8 @@ from collections.abc import Callable from typing import Any +from nullrun.breaker.exceptions import ENFORCEMENT_EXCEPTIONS + logger = logging.getLogger(__name__) _llama_index_patched = False @@ -79,6 +81,12 @@ def on_chat_end(event: Any) -> None: if model: event_dict["model"] = model runtime.track(event_dict) + except ENFORCEMENT_EXCEPTIONS: + # SDK-6 (reliability audit 2026-11-02): the backend REFUSED + # to book this call. ADR-008's /track row is CLOSED for + # enforcement rejections — the refusal propagates to the + # caller, so a refused track cannot look like a success. + raise except Exception as e: # pragma: no cover - defensive logger.debug("llama_index on_chat_end: %s", e) diff --git a/src/nullrun/messages.py b/src/nullrun/messages.py index 73ca1f4..56e295a 100644 --- a/src/nullrun/messages.py +++ b/src/nullrun/messages.py @@ -130,6 +130,13 @@ # call — replay / retry-loop signal. NOT a transient failure; the # same approval_id will never succeed twice. Inspect retry logic. "NR-A015": "Your request couldn't be completed because the approval has already been used. Please start a new request.", + # NR-A017: approval tool mismatch (DEC-5 / DEC-7). The operator + # approved an action for one tool; the agent is executing a + # DIFFERENT tool under that approval. The approval binds a tool + # SET, so the existing grant can never authorise a tool outside it + # — the user's next move is a fresh approval for the tool they + # actually want to run, not a retry of the same one. + "NR-A017": "Your request couldn't be completed because the approval was for a different tool. Please request a new approval and try again.", # ---- Workflow lifecycle (server-side state) ------------------------------ # NR-W004: workflow soft-deleted or killed on the server. Distinct # from NR-W002 (BaseException path that bypasses ``nullrun.guard``) @@ -178,6 +185,14 @@ # support" — the host code needs an SDK upgrade, which only the # operator / developer can perform. "NR-P001": "This service needs an update. Please contact support.", + # NR-P004: the SDK could not OBTAIN a decision — the gate call + # failed for a reason that is NOT a network failure, so the SDK + # stopped rather than assume the call was allowed (ADR-129 for + # the /gate seam, ADR-131 for the /execute seam). The fault is the + # SDK's own and may be transient, so the end-user copy is the + # "try again" cluster — the same shape as NR-B001, because the + # user's action is identical: retry. + "NR-P004": "I'm unable to check this request right now. Please try again in a moment.", # ---- Chain (multi-leg conversation state) ------------------------------- # NR-CH001: chain context invalid — chain_id is unknown, belongs to # a different org, or exceeded max_duration. End-user outcome is diff --git a/src/nullrun/observability/__init__.py b/src/nullrun/observability/__init__.py index e3d88f2..fccae0b 100644 --- a/src/nullrun/observability/__init__.py +++ b/src/nullrun/observability/__init__.py @@ -76,6 +76,19 @@ class TransportMetrics: # We split it from hmac_verify_failures_total so the two # alert paths can have different runbooks. hmac_verify_expired_total: int = 0 + # Approval frames on the control-plane WebSocket that the SDK + # could NOT authenticate. Such a frame is REFUSED — it never + # releases a parked gate (SDK-5). A non-zero value means + # somebody pushed an approval the SDK cannot verify: either a + # client that never got its HMAC secret, or a forged frame. + ws_unverified_approval_total: int = 0 + # Control-plane frames the SDK REFUSED because it could verify + # signatures (it holds the API key and the HMAC secret) and the + # frame was not verifiably signed. The server signs every frame it + # sends for such a key, so an unsigned frame did not come from the + # server. A non-zero value means somebody injected frames into the + # socket, or the client and the server disagree about the secret. + ws_unverified_frame_total: int = 0 # --- WAL durability state (set by Transport, not incremented) ---------- # These answer "what can this deployment actually promise?", which the # code cannot answer on your behalf across OSes and filesystems. A field @@ -166,6 +179,32 @@ class RuntimeMetrics: # counter, the failure mode was invisible at INFO log level on # the FALLBACK path. gate_fail_open_total: int = 0 + # The gate's decision was INDETERMINATE and the SDK REFUSED + # (SDK-11). A separate counter from ``gate_fail_open_total`` + # because this one does NOT fail open: the call is stopped. It is + # the signal that the SDK itself is broken (a request body that + # cannot be serialised, a decode error, a fault in the edge + # path) — an SDK-side incident, not a backend outage, so it must + # not be alerted on as one. + gate_indeterminate_total: int = 0 + # A control-plane action arrived whose type this SDK does not know + # (SDK-12). The action was NOT executed; the SDK raised + # ``NullRunUnknownActionError`` and the caller was told. This + # counter is the operator's signal that the server and the SDK are + # out of step (the server grew its action vocabulary, the SDK did + # not follow). Expected rate: zero — a non-zero rate means the + # control plane is sending instructions this SDK cannot run. + unknown_action_total: int = 0 + # The /execute policy decision was INDETERMINATE and the SDK + # REFUSED (SDK-13). The twin of ``gate_indeterminate_total`` on + # the /execute side: the SDK's own code faulted while asking the + # policy engine, so the SDK could not tell "the engine refused + # this" from "the SDK never found out" — and the decorated body + # did NOT run. A SEPARATE counter from ``gate_indeterminate_total`` + # because the two are different seams: this one is the /execute + # request path, that one is the /gate path, and an operator + # chasing an SDK bug needs to know which one moved. + policy_gate_indeterminate_total: int = 0 # `/track/batch` with `llm_call` events missing `reservation_id` # is now fail-CLOSED at the backend (whole-batch 503 BUDGET_RECHECK_FAILED). # Pre-0.16.0 the SDK fell back to that path for calls without a @@ -256,8 +295,11 @@ def to_dict(self) -> dict[str, Any]: "circuit_half_open_count": self.transport.circuit_half_open_count, "circuit_closed_count": self.transport.circuit_closed_count, "fallback_mode_activations": self.transport.fallback_mode_activations, + "events_recorded_overage": self.transport.events_recorded_overage, "hmac_verify_failures_total": self.transport.hmac_verify_failures_total, "hmac_verify_expired_total": self.transport.hmac_verify_expired_total, + "ws_unverified_approval_total": self.transport.ws_unverified_approval_total, + "ws_unverified_frame_total": self.transport.ws_unverified_frame_total, "wal_dir_fsync": self.transport.wal_dir_fsync, "wal_lock": self.transport.wal_lock, "wal_lock_timeouts_total": self.transport.wal_lock_timeouts_total, @@ -282,6 +324,14 @@ def to_dict(self) -> dict[str, Any]: "timeouts": self.runtime.timeouts, "loop_detections": self.runtime.loop_detections, "gate_fail_open_total": self.runtime.gate_fail_open_total, + "gate_indeterminate_total": self.runtime.gate_indeterminate_total, + "unknown_action_total": self.runtime.unknown_action_total, + "policy_gate_indeterminate_total": ( + self.runtime.policy_gate_indeterminate_total + ), + "dropped_llm_call_no_reservation": ( + self.runtime.dropped_llm_call_no_reservation + ), }, } diff --git a/src/nullrun/observability/error_hooks.py b/src/nullrun/observability/error_hooks.py index 17d9bf6..2317cbe 100644 --- a/src/nullrun/observability/error_hooks.py +++ b/src/nullrun/observability/error_hooks.py @@ -96,9 +96,10 @@ class ErrorContext: # tool-call failures. tool_name: str | None = None - # First 10 characters of the api key in use, or ``None`` if - # no key was set yet. Used for log triage — the full key - # never leaves the SDK. + # An 8-character SHA-256 fingerprint of the api key in use, or + # ``None`` if no key was set yet. Used for log triage — the full + # key never leaves the SDK, and neither does any character of it + # (SDK-5; the value used to be the first ten characters). api_key_prefix: str | None = None # Backend correlation id (``X-Correlation-Id`` response diff --git a/src/nullrun/observability/redaction.py b/src/nullrun/observability/redaction.py new file mode 100644 index 0000000..3825bfb --- /dev/null +++ b/src/nullrun/observability/redaction.py @@ -0,0 +1,95 @@ +"""Credential redaction for everything the SDK prints. + +SDK-5 (2026-11-06): the status string and the error context used to +carry the FIRST TEN CHARACTERS of the API key, and the transport's +debug log carried the first eight with a ``***`` tail. Two problems: + +1. A prefix of a credential is a disclosure. The remaining characters + are what an attacker needs to brute-force the rest. +2. The two places disagreed about how much they printed, so "the full + key never leaves the SDK" was not a statement anyone could check. + +A fingerprint replaces the prefix. It is derived from the key, it is +stable (so it still correlates log lines and status snapshots), and it +carries NO character of the key. The backend does the same thing -- +``backend/src/auth/mod.rs::compute_fingerprint`` is SHA-256 over the +raw key -- so a log line can be correlated on both sides without +either side holding a fragment of the secret. +""" + +from __future__ import annotations + +import hashlib +import re + +#: Hex characters kept from the SHA-256 digest. +API_KEY_FINGERPRINT_LEN: int = 8 + + +def api_key_fingerprint(api_key: str | None) -> str | None: + """Return an 8-character fingerprint of ``api_key``. + + Returns ``None`` for a missing key -- a fingerprint of nothing is + a lie. The digest is truncated to 8 hex characters: enough to + correlate a log line with a status snapshot, not enough to be a + credential fragment. + """ + if not api_key: + return None + digest = hashlib.sha256(api_key.encode("utf-8")).hexdigest() + return digest[:API_KEY_FINGERPRINT_LEN] + + +# Credential shapes, matched on the VALUE rather than on the key it +# travels under. Key-NAME redaction (``SENSITIVE_ARG_KEYS``) cannot +# catch a secret that arrives in a benign key -- ``{"note": ""}`` +# prints the key in full. +# +# Every pattern here is deliberately narrow: it must not match a +# value an ordinary tool call carries. A shape that cannot be told +# apart from ordinary data is not a shape -- it is a guess, and a +# guess that scrubs real data is a bug of its own. +# +# Most of the shapes carry NO leading word boundary on purpose. A +# boundary would make the pattern miss a credential that sits right +# after a letter -- ``"prefix" + key`` -- and that is exactly the +# leak this pass exists to close. The one exception is ``sk-``, +# whose boundary keeps ``risk-assessment-report`` out of the scrub. +_SECRET_VALUE_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + # This product's own keys: ``nr_live_`` / ``nr_test_`` + the random part. + (re.compile(r"nr_(?:live|test)_[A-Za-z0-9_-]{8,}"), "***"), + # A JSON Web Token: a base64url header starting with ``eyJ``, two dots. + (re.compile(r"eyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}"), "***"), + # An AWS access key id: a four-character prefix plus sixteen + # uppercase alphanumerics. + (re.compile(r"(?:AKIA|ASIA)[0-9A-Z]{16}"), "***"), + # The OpenAI-style secret key: ``sk-`` plus a long opaque tail. + (re.compile(r"\bsk-[A-Za-z0-9_-]{16,}"), "***"), + # A PEM private key, whole block first, bare header second. + ( + re.compile( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]{0,8192}?" + r"-----END [A-Z ]*PRIVATE KEY-----" + ), + "***", + ), + (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), "***"), + # An HTTP bearer credential. + (re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{16,}"), "Bearer ***"), +) + + +def scrub_secret_values(text: str) -> str: + """Replace credential-shaped substrings in ``text`` with ``***``. + + This is the value-shaped half of redaction. The key-name half + (``SENSITIVE_ARG_KEYS``) knows WHICH arguments are sensitive; this + pass catches a credential that arrives under a name nobody listed. + + The text is scanned as-is, so the caller must run this BEFORE + truncating: a credential that straddles the cut is caught whole. + """ + scrubbed = text + for pattern, replacement in _SECRET_VALUE_PATTERNS: + scrubbed = pattern.sub(replacement, scrubbed) + return scrubbed diff --git a/src/nullrun/observability/status.py b/src/nullrun/observability/status.py index b7c5b03..0065a64 100644 --- a/src/nullrun/observability/status.py +++ b/src/nullrun/observability/status.py @@ -139,7 +139,11 @@ class NullRunStatus: # Auth api_key_valid: bool | None # None = never tested - api_key_prefix: str | None # first 10 chars, never the full key + # An 8-character SHA-256 fingerprint of the API key (SDK-5). + # The field name is the legacy one: the value used to be the + # first TEN CHARACTERS of the key. It now carries NO character of + # the key. + api_key_prefix: str | None organization_id: str | None workflow_id: str | None api_url: str @@ -169,13 +173,13 @@ def summary(self) -> str: console. Example outputs: - "NullRunStatus(ok, api_key=nr_live_S, org=…, wf=…)" + "NullRunStatus(ok, api_key_fp=3f2a1b9c, org=…, wf=…)" "NullRunStatus(degraded, wf_state=Killed, backend=unreachable)" "NullRunStatus(offline, ws=False, errors=2)" """ bits = [f"NullRunStatus({self.state}"] if self.api_key_prefix: - bits.append(f"api_key={self.api_key_prefix}") + bits.append(f"api_key_fp={self.api_key_prefix}") if self.organization_id: bits.append(f"org={self.organization_id[:8]}") if self.workflow_id: diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index 88c51ea..c43b4c9 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -8,7 +8,7 @@ - Local policy enforcement (instant, no network latency) - Pre-execution enforcement via /execute endpoint -## Pre-execution gate fail-OPEN/CLOSED contract (ADR-008) +## Pre-execution gate fail-OPEN/CLOSED contract (ADR-063 §2.4) The SDK enforces workflow safety through a set of *pre-execution gates* that run before a protected function body executes and may raise to halt @@ -17,13 +17,24 @@ | Gate | Transport-error behavior | Recovery behavior | Opt-out | |---|---|---|---| -| `check_workflow_budget` | OPEN (skip check, log warning) for TRANSPORT errors; **CLOSED for authentication errors** (401 raises `NullRunAuthError` — see below) | silent post-hoc correction in `/track` events via `cost_correction_applied=true` | `NULLRUN_SKIP_BUDGET_CHECK=1` -- **full billing bypass**, not just check bypass (see docstring WARNING) | +| `check_workflow_budget` | OPEN (skip check, log warning) for TRANSPORT errors ONLY; **CLOSED for everything else** — a 401 raises `NullRunAuthError`, a malformed answer raises `NullRunMalformedGateResponseError`, and a non-transport fault raises `NullRunGateIndeterminateError` (ADR-129) | silent post-hoc correction in `/track` events via `cost_correction_applied=true` | `NULLRUN_SKIP_BUDGET_CHECK=1` -- **full billing bypass**, not just check bypass (see docstring WARNING) | | `check_control_plane` | OPEN (treat state as `Normal`) | deferred enforcement -- next WS-push or `/status` poll sees the true state | none | -| `_enforce_sensitive_tool` (default `_fallback_mode=strict` since v3.53) | CLOSED -- transport returns `decision=block, decision_source=FALLBACK_*` | n/a | none for the strict path; `NULLRUN_SENSITIVE_FAIL_OPEN=1` opts into the legacy permissive override | -| `_enforce_sensitive_tool` (`_fallback_mode=permissive`, opt-in) | CLOSED -- body MUST NOT run when `decision_source` is any `FALLBACK_*` | n/a (body did not run) | `NULLRUN_SENSITIVE_FAIL_OPEN=1` -- explicitly documented as "OPEN-when-engine-unavailable" | +| `_run_tool_policy_gate` (the `@protect` path) | CLOSED -- the body MUST NOT run when the policy engine is unreachable | n/a | `NULLRUN_SENSITIVE_FAIL_OPEN=1` **plus** a per-tool classification (`runtime.register_fail_open_tools([...])`) -- the switch alone opens NOTHING (ADR-126) | +| `_run_tool_policy_gate` with the switch ON and the tool classified fail-open-allowed | OPEN for THAT tool only -- the body runs while the engine is unreachable | n/a | the two-step opt-out above; every tool that is not classified stays CLOSED (ADR-126) | +| `_run_tool_policy_gate` -- a NON-transport fault inside the /execute call (the SDK's own code) | CLOSED -- the refusal is `NullRunGateIndeterminateError` (NR-P004) and the body MUST NOT run (ADR-131) | n/a | none -- the two-step opt-out opens TRANSPORT failures only. An SDK-internal fault is not "the engine is unreachable", and nothing downstream corrects it | +| `runtime.execute` -> `Transport.execute` (the MCP path; `fallback_mode=strict` by default) | CLOSED -- transport returns `decision=block, decision_source=FALLBACK` and the MCP client is NOT called | n/a | none -- `NULLRUN_SENSITIVE_FAIL_OPEN` is NOT consulted on this path | +| `runtime.execute` -> `Transport.execute` (`fallback_mode=permissive`, opt-in) | OPEN -- transport synthesises `decision=allow, decision_source=FALLBACK` | n/a (the body runs) | the constructor knob `NullRunRuntime(fallback_mode="permissive")` -- a code-level opt-in, not an env var | | `_emit_span_start` / `_emit_span_end` | n/a -- never blocks | n/a | n/a | | `/track` batch path (legacy) | OPEN-on-network-error (event dropped, no retry) | n/a -- circuit breaker backoff applies | none | -| `/track` v3 single path (`track_single`) | OPEN-on-network-error and OPEN-on-5xx (event dropped, no retry); **CLOSED for enforcement rejections** — a typed `NullRunDecision` (`CONSUME_OVERBUDGET` 422, budget block 402) propagates to the caller | caller reconciles the delta from the exception's `reserved_millicents` / `max_allowed_millicents` / `actual_cost_cents` / `recorded` (the reservation is on the wire in millicents and the actual in cents; the backend does not send its configured epsilon, so `actual - reserved` is the only honest way to recover it); ADR-005 forbids implicit re-reserve, so there is nothing for the SDK to retry | none | +| `/track` v3 single path (`track_single`) | OPEN-on-network-error ONLY — the payload is written to the v3 WAL (`.v3`) BEFORE the caller's view drops it, and it is replayed on the next start; **CLOSED for every ANSWER that refused the consume** — any non-2xx, whether or not the body names an enforcement failure (SDK-1, ADR-183). A typed `NullRunDecision` (`CONSUME_OVERBUDGET` 422, budget block 402) propagates to the caller | the durable v3 WAL, replayed to `/api/v1/track` (SINGLE — the batch path does not run the CONSUME script, so a reservation replayed through it is never settled). The caller also reconciles the delta from the exception's `reserved_millicents` / `max_allowed_millicents` / `actual_cost_cents` / `recorded` (the reservation is on the wire in millicents and the actual in cents; the backend does not send its configured epsilon, so `actual - reserved` is the only honest way to recover it); ADR-005 forbids implicit re-reserve, so there is nothing for the SDK to retry | none | + +**The LangChain/LangGraph callback obeys the `check_workflow_budget` row.** +`NullRunCallback` calls the same gate. It fails OPEN for a transport error and +CLOSED for every other fault. It raises the same class the runtime raises, so +one refusal produces one outcome on every path (SDK-16). + +**The `@protect` boundary is a different gate.** It fails CLOSED for every +fault. The two-step opt-out in its own row is the only way it fails OPEN. **Fail-OPEN policy** — SDK-side transport failure (network timeout, 5xx, breaker open) is fail-OPEN on the *check* path so a dead @@ -43,6 +54,15 @@ * **SDK-side transport failure** (network timeout, 5xx, breaker open) → fail-OPEN on the *check* path so a dead backend doesn't freeze the user's agent loop (this is what the README describes). +* **A backend ANSWER that refused a consume** (`/track`, the money path) + → fail-CLOSED, even when the body names no enforcement failure. + SDK-1 (ADR-183): the question is not "what did the body say?" but + "did the backend ANSWER?". The consume did not land, the tool call + has already run and the money has already been spent, so reporting + the refusal as a successful track is a lost spend with a success + label on it. A request that never REACHED the backend keeps the + fail-OPEN row above — and its payload is durable in the v3 WAL, so + it is not dropped either. * **Authentication failure (401)** → fail-CLOSED. ``NullRunAuthError`` propagates. A 401 is a CREDENTIAL/CONFIG failure, not a transient transport condition: no retry can fix a @@ -51,6 +71,14 @@ OPEN here means the agent proceeds on a request the backend never approved. Added 2026-09-29 (DEF-MP-TS12-ENF-01) — this NARROWS the fail-OPEN set, it does not widen enforcement. +* **Indeterminate decision (SDK-11, ADR-129)** → fail-CLOSED. + ``NullRunGateIndeterminateError`` propagates. The gate call failed + for a reason that is NOT a transport failure, so the SDK cannot + tell "the backend said no" from "the SDK never found out". The + fail-OPEN above covers a gate that could not be REACHED; this is + the case where the SDK itself faulted while asking. Failing OPEN + here turned an SDK bug into "allowed" — the call ran with no + reservation and its cost event was dropped. * **Backend-side budget-enforcement failure** (the /gate or /track handler actually returned a wire response, just one indicating a Redis outage or aggregate rate-limit Redis unavailable) → the @@ -69,9 +97,18 @@ The "Opt-out" column makes it explicit that `NULLRUN_SKIP_BUDGET_CHECK=1` is a **different category** of action than `NULLRUN_SENSITIVE_FAIL_OPEN=1` (bypass vs. change semantics), despite -the similar naming. See `docs/adr/008-sdk-preflight-fail-policy.md` -for the full rules, including transport error classification -(`FALLBACK_NETWORK_ERROR` / `FALLBACK_GATEWAY_ERROR` / `FALLBACK_BREAKER_OPEN`). +the similar naming. See ADR-063 §2.4 (in the NULLRUN repository, +`docs/adr/ADR-063-pause-on-the-gate-and-the-fail-open-ledger.md`) for +the full ledger of every fail-OPEN site in both repos, including the +transport error classification (`FALLBACK_NETWORK_ERROR` / +`FALLBACK_GATEWAY_ERROR` / `FALLBACK_BREAKER_OPEN`). + +The `NULLRUN_SENSITIVE_FAIL_OPEN` switch is a **whole-system switch**, +and what it is allowed to open is decided **per tool** by the +operator's classification (`runtime.register_fail_open_tools([...])`). +The two are combined at the one call site (`decorators.py`), and the +set is empty by default -- so the switch alone opens nothing and every +tool stays fail-CLOSED. ADR-126. """ from __future__ import annotations @@ -103,6 +140,7 @@ NullRunUnclassifiedRefusalError, ) from nullrun.breaker.exceptions import ( + ENFORCEMENT_EXCEPTIONS, NullRunApprovalDeniedError, NullRunApprovalExpiredError, NullRunApprovalReplayRejectedError, @@ -113,7 +151,7 @@ NullRunBudgetError, NullRunDecision, NullRunDeniedError, - NullRunError, + NullRunGateIndeterminateError, NullRunInfrastructureError, NullRunTransportError, NullRunWorkflowKilledError, @@ -129,7 +167,10 @@ get_workflow_id, ) from nullrun.edge import EDGE_LEASE as EDGE_LEASE_DECISION_SOURCE +from nullrun.edge import EDGE_NOT_ENFORCED as EDGE_NOT_ENFORCED_DECISION_SOURCE +from nullrun.edge import EDGE_UNREACHABLE as EDGE_UNREACHABLE_DECISION_SOURCE from nullrun.observability import metrics +from nullrun.observability.redaction import api_key_fingerprint from nullrun.transport import ( HEADER_PROTOCOL, DecisionSource, @@ -141,6 +182,7 @@ _emit_for_transport_error, _protocol_header_value, _safe_json, + is_backend_refusal, is_fallback_decision_source, ) from nullrun.uuid7 import uuid7_str @@ -150,66 +192,38 @@ # Sentinel used when a gate fires outside a ``with workflow(...)`` UNKNOWN_WORKFLOW_ID: str = "__nullrun_unknown__" -# In-process gate cache for chain-mode. The cache key includes -# ``estimated_tokens`` (currently hardcoded to 1 in -# ``check_workflow_budget``) so two chain-mode calls with the same -# (workflow_id, chain_id, call_model) but different cost_estimate -# do not collide and silently serve a cheaper cached allow for -# a more expensive call. The cache is also invalidated when -# /track signals budget exhaustion (HTTP 422 from -# CONSUME_OVERBUDGET) and when ``chain_end`` is called — without -# this, a chain that exhausts its budget mid-loop could continue -# serving cached "allow" decisions for up to the TTL after the -# backend has actually blocked subsequent calls. Invalidations -# use the active chain_id from the contextvar so unrelated -# chains sharing the runtime instance are not affected. -_GATE_CACHE: dict[tuple[str, str | None, str | None, int], tuple[float, dict[str, Any]]] = {} -_GATE_CACHE_TTL_SECONDS: float = 5.0 - - -def _invalidate_gate_cache_for_chain(workflow_id: str | None, chain_id: str | None) -> int: - """Drop ``_GATE_CACHE`` entries whose ``(workflow_id, chain_id)`` - match the supplied pair, regardless of ``call_model`` or - ``estimated_tokens``. Returns the number of entries removed. - - Called from two sites: - 1. ``_route_track`` after ``track_single`` raises HTTP 422 - (CONSUME_OVERBUDGET) or HTTP 402 (REDIS_UNAVAILABLE for - consume-side authoritative check). The backend's - authoritative budget check has now said "no further - spend", so any cached "allow" for the same chain is stale - and must not be served until the chain ends or the - budget rolls over. - 2. ``chain_end`` after the /gate body returns. The chain is - closed on the server; the SDK's in-process cache for that - chain is no longer reachable from subsequent calls, so - dropping it frees memory and prevents a hypothetical - re-use of the same (workflow_id, chain_id) UUID from - hitting the cached entry (UUID v4 collision risk is - negligible but the cleanup costs nothing). - - Args: - workflow_id: The workflow whose entries should be dropped. - ``None`` matches every workflow with the same chain_id - (rare but defensive). - chain_id: The chain whose entries should be dropped. - ``None`` matches every chain with the same workflow_id. - - Returns: - The number of cache entries removed. Useful for tests / - metric counters; not currently surfaced. - """ - if not _GATE_CACHE: - return 0 - keys_to_drop = [ - k - for k in _GATE_CACHE - if (workflow_id is None or k[0] == workflow_id) - and (chain_id is None or k[1] == chain_id) - ] - for k in keys_to_drop: - _GATE_CACHE.pop(k, None) - return len(keys_to_drop) +# There is deliberately NO in-process gate cache. +# +# There was one: a chain-mode short-circuit keyed on +# ``(workflow_id, chain_id, call_model, estimated_tokens)`` that served +# a previous ``/gate`` answer for up to five seconds. It is gone +# because it was not a cache, it was a second authorisation path with +# no gate behind it (SDK-1): +# +# * The key carried no org id, no api key id and no tool set, so one +# chain's allow could be served to another chain — or to a +# different tool inside the same chain — whose call was never +# checked. +# * The cached body carried call #1's ``reservation_id``, and the +# runtime re-captured it for every later step of the chain. Every +# step of the chain therefore ran under ONE server-minted +# execution id. The backend settles an execution id at most once +# (``consume_v3.lua`` answers IDEMPOTENT_REPLAY for an execution +# that is already ``consumed``, and books nothing), so a three-step +# chain performed three tool executions and produced ONE +# accounting event: the money for steps 2 and 3 was never billed. +# * The tool-block check never ran for a cached step either — the key +# has no tool set in it. +# +# The chain feature it was written for does not need it. The backend +# owns the chain: ``chain_op`` (start / continue / end) and the chain's +# TTL are enforced there, and every ``/gate`` inside a chain reserves +# and settles its own envelope. The backend's own idempotency store +# (``idempotency_key`` = the SDK's ``operation_id``) is the safe +# version of what the cache was reaching for: it dedupes a genuine +# RETRY of one logical action and nothing else. A client-side cache +# that cannot tell a retry from a new step is the wrong place to put +# that decision. # Production-environment detection for security opt-out @@ -218,42 +232,78 @@ def _invalidate_gate_cache_for_chain(workflow_id: str | None, chain_id: str | No # NULLRUN_SKIP_BUDGET_CHECK в production env"); the SDK refuses # it outside dev / test environments. # -# Two signals combine to flag a production environment: +# Two signals flag a production environment: # 1. The configured ``api_url`` hostname matches the prod host # (``api.nullrun.io``) — the SDK's hard-coded default per the # constructor docstring. # 2. The operator set ``NULLRUN_ENV`` to ``production`` / ``prod`` -# (explicit override for non-standard deployments that point at +# (explicit declaration for non-standard deployments that point at # a custom URL but still serve prod traffic). # +# A host is dev ONLY when it says so in a way that cannot be an +# accident: an exact loopback, a ``.localhost`` suffix, or an explicit +# dev-environment LABEL (``staging.`` / ``test.``). NEVER by substring. +# SDK-2 (2026-11-06): the old check asked "is any marker a SUBSTRING of +# the hostname?". ``"test"`` is a substring of ``latest``, ``contest``, +# ``attest``; ``"localhost"`` is a substring of ``notlocalhost.example.com``. +# So a host that merely CONTAINED a dev marker was classified non-prod +# and the operator's ``NULLRUN_ENV=production`` was suppressed — the +# billing bypass ran against production with a WARNING instead of a +# refusal. +# # Test / dev harnesses that override ``api_url`` to a local / -# staging host are NOT flagged as production regardless of -# ``NULLRUN_ENV`` — operators who DO want the bypass in prod can -# set ``NULLRUN_ALLOW_SKIP_BUDGET_CHECK=1`` to acknowledge the -# risk explicitly (analogous to ``NULLRUN_SENSITIVE_FAIL_OPEN``). +# staging host are NOT flagged as production — operators who DO want +# the bypass in prod can set ``NULLRUN_ALLOW_SKIP_BUDGET_CHECK=1`` to +# acknowledge the risk explicitly (analogous to +# ``NULLRUN_SENSITIVE_FAIL_OPEN``). _PROD_API_HOST: str = "api.nullrun.io" +# Hosts that are local by construction. ``0.0.0.0`` is the +# "unspecified" address a dev harness binds to; it is a marker string +# here, never a bind address (S104). +_LOOPBACK_HOSTS: frozenset[str] = frozenset( + # "0.0.0.0" is a marker string compared against a hostname, never a + # bind address (S104). + {"localhost", "127.0.0.1", "0.0.0.0", "::1"} # noqa: S104 +) + +# A leading label that declares a dev environment. Compared as a +# LABEL, so ``api.test.nullrun.io`` keeps its own meaning and +# ``staging.internal`` is a dev host while ``latest.nullrun.io`` is +# not. +_DEV_HOST_LABELS: frozenset[str] = frozenset({"staging", "test"}) + +# Values of ``NULLRUN_ENV`` that declare production. +_PROD_ENV_VALUES: frozenset[str] = frozenset({"production", "prod"}) + def _is_production_environment(api_url: str | None = None) -> bool: """Return True when the SDK is running against the production NullRun backend. Used by ``check_workflow_budget`` to refuse - ``NULLRUN_SKIP_BUDGET_CHECK=1`` outside dev / test environments. - Detection rules (in order): - - 1. ``api_url`` is None or matches the prod host - (``api.nullrun.io``) — the default constructor value. - 2. ``NULLRUN_ENV`` is set to ``production`` / ``prod`` (case - insensitive) AND the api_url is not explicitly pointed at a - known non-prod host (``localhost``, ``127.0.0.1``, - ``staging``, ``test``). - - Local / staging / test deployments are NOT flagged so the dev - bypass keeps working. Operators who insist on the bypass in - prod can set ``NULLRUN_ALLOW_SKIP_BUDGET_CHECK=1`` — but the - env var is then surfaced in audit / telemetry so it's not a - silent fail-OPEN. + ``NULLRUN_SKIP_BUDGET_CHECK=1`` outside dev / test environments, + and by ``sensitive_fail_open_enabled`` to refuse the second + fail-open switch. + + Detection rules, in order: + + 1. The host EXACTLY a loopback (``localhost``, ``127.0.0.1``, + ``0.0.0.0``, ``::1``) or ending in ``.localhost`` → NOT + production. A local machine does not serve prod traffic, and + no env var may arm the guard for it. + 2. The host's FIRST label is a dev label (``staging.``, + ``test.``) → NOT production. This is an explicit environment + marker, not a name accident. + 3. The host is exactly ``api.nullrun.io`` → production. + 4. ``NULLRUN_ENV`` is ``production`` / ``prod`` (case insensitive) + → production for every other host. The operator's declaration + wins over the host heuristic; the substring test that used to + suppress it is gone. + + A host the SDK does not recognise, with no ``NULLRUN_ENV``, stays + non-production: that is the pre-existing behaviour for a custom + deployment and it keeps the documented dev escape hatch working. """ from urllib.parse import urlparse @@ -268,20 +318,30 @@ def _is_production_environment(api_url: str | None = None) -> bool: host = parsed.hostname if parsed is not None else None host_lc = (host or "").lower() - # Signal 1: explicit prod host. ``api.nullrun.io`` is the + # Rule 1: exact loopback / ``.localhost`` suffix. Never production, + # whatever NULLRUN_ENV says — a loopback is a local machine. + if host_lc in _LOOPBACK_HOSTS or host_lc.endswith(".localhost"): + return False + + # Rule 2: an explicit dev-environment LABEL. Compared label by + # label, never as a substring: ``staging.internal`` is dev, + # ``latest.nullrun.io`` and ``notlocalhost.example.com`` are not. + first_label = host_lc.split(".")[0] + if first_label in _DEV_HOST_LABELS: + return False + + # Rule 3: explicit prod host. ``api.nullrun.io`` is the # canonical production endpoint; the constructor's default # value is exactly this URL so any caller that did not # override ``api_url`` lands here. if host_lc == _PROD_API_HOST: return True - # Signal 2: explicit NULLRUN_ENV=production (or "prod") AND - # the api_url does not point at a known dev/staging host. + # Rule 4: the operator declared production. This wins over every + # remaining host heuristic — SDK-2: the old substring match used + # to suppress it. explicit_env = os.getenv("NULLRUN_ENV", "").strip().lower() - non_prod_hosts = ("localhost", "127.0.0.1", "0.0.0.0", "staging", "test") # noqa: S104 -- string marker, not a bind address - if explicit_env in {"production", "prod"} and not any( - marker in host_lc for marker in non_prod_hosts - ): + if explicit_env in _PROD_ENV_VALUES: return True return False @@ -934,6 +994,7 @@ def __init__( # every value is lowercased at insertion time. self._sensitive_tools_lower: frozenset[str] = frozenset() self._strict_mode_tools_lower: frozenset[str] = frozenset() + self._fail_open_tools_lower: frozenset[str] = frozenset() self._sensitive_tools: set[str] = { # Financial operations "stripe.charge", @@ -964,6 +1025,24 @@ def __init__( "admin.disable_user", } self._strict_mode_tools: set[str] = set() + # Tools the operator has classified as FAIL-OPEN-ALLOWED. + # + # This is the per-tool classification that bounds the + # NULLRUN_SENSITIVE_FAIL_OPEN switch: the switch may let a + # tool's body run while the policy engine is unreachable ONLY + # for a tool in this set. It is EMPTY by default, so the switch + # on its own opens nothing. + # + # It is deliberately NOT the sensitivity registry above. That + # registry answers "is this tool destructive?" (the SDK's + # built-in list of financial / messaging / admin operations); + # a destructive tool is the last thing that should be allowed + # to run unchecked. This set answers the opposite question: + # "which tools may this agent run unchecked during an + # outage?". The two questions must not be merged -- merging + # them would make the SDK's own list of destructive operations + # an allowlist for running without a policy decision. + self._fail_open_tools: set[str] = set() # Snapshot the lowercase view of the built-in list so the # hot path is a single frozenset membership check (no set # comprehension per call). Subsequent add/remove/ @@ -1060,7 +1139,7 @@ def status(self) -> Any: # --- Auth state --- api_key_valid: bool | None = None - api_key_prefix: str | None = self.api_key[:10] if self.api_key else None + api_key_prefix: str | None = api_key_fingerprint(self.api_key) if self.organization_id is not None: # If we have an org, auth at least started — it # may have failed (we'd be in misconfigured), but @@ -1337,7 +1416,7 @@ def _emit_sdk_error( stage=stage, workflow_id=resolved_workflow_id, tool_name=tool_name, - api_key_prefix=(self.api_key[:10] if self.api_key else None), + api_key_prefix=api_key_fingerprint(self.api_key), correlation_id=correlation_id, extra=extra or {}, ), @@ -1401,11 +1480,7 @@ def _authenticate(self) -> None: self.workflow_id = data.get("workflow_id") if self.workflow_id is None: - masked_key = ( - (self.api_key[:8] + "***") - if self.api_key and len(self.api_key) >= 8 - else "***" - ) + masked_key = api_key_fingerprint(self.api_key) or "unknown" logger.warning( f"API key {masked_key!s} is a legacy key with no " f"workflow binding; remote kill/pause will not be " @@ -2046,8 +2121,25 @@ def check_control_plane(self, workflow_id: str) -> None: # producer cannot drift into two different spellings of one value — # which is exactly how `is_fallback_decision_source` ended up with # two hand-maintained copies. + # + # `edge_unreachable` and `edge_not_enforced` are the two edge + # REFUSALS (the box did not answer / the box was never asked). They + # are here for the same reason `edge_lease` is, and with the same + # consequence: the runtime must recognise them as real verdicts and + # honour the block. They are deliberately NOT fallback sources — + # `fallback` means "the engine never answered, and the cloud's + # `/track` can still book the call later" (ADR-008), which is not + # true in edge mode. See `nullrun.edge.EDGE_UNREACHABLE`. _KNOWN_DECISION_SOURCES = frozenset( - {"gateway", "cached", "fallback", "local", EDGE_LEASE_DECISION_SOURCE} + { + "gateway", + "cached", + "fallback", + "local", + EDGE_LEASE_DECISION_SOURCE, + EDGE_UNREACHABLE_DECISION_SOURCE, + EDGE_NOT_ENFORCED_DECISION_SOURCE, + } ) def _require_gate_decision(self, response: Any) -> str: @@ -2185,6 +2277,14 @@ def sensitive_fail_open_enabled(self) -> bool: ``NULLRUN_ALLOW_SENSITIVE_FAIL_OPEN=1``, mirroring ``NULLRUN_ALLOW_SKIP_BUDGET_CHECK``. + **This method answers "is the switch on?", not "may this tool + fail open?"** The switch is a whole-system switch; what it is + allowed to open is decided per tool by the operator's + classification (``register_fail_open_tools`` / + ``is_fail_open_allowed``). The two are combined at the one + call site -- ``decorators.py`` -- and the tool's name is + needed for the second half, which is why it is not asked here. + Lives here rather than in ``decorators`` so the environment policy stays in one module with :func:`_is_production_environment`, and so the two opt-outs @@ -2216,11 +2316,13 @@ def sensitive_fail_open_enabled(self) -> bool: pass return False logger.warning( - "sensitive tool gate: failing OPEN via " - "NULLRUN_SENSITIVE_FAIL_OPEN=1 in production " - "(NULLRUN_ALLOW_SENSITIVE_FAIL_OPEN=1 also set). This is an " - "explicit operator ack — ensure the incident-response runbook " - "drove it, and unset both vars when the incident closes." + "the sensitive fail-open switch is ON in production " + "(NULLRUN_SENSITIVE_FAIL_OPEN=1 + " + "NULLRUN_ALLOW_SENSITIVE_FAIL_OPEN=1). Only tools the " + "operator classified as fail-open-allowed may run " + "unchecked. This is an explicit operator ack — ensure the " + "incident-response runbook drove it, and unset both vars " + "when the incident closes." ) try: metrics.inc_runtime("sensitive_fail_open_allowed_in_prod") @@ -2229,8 +2331,9 @@ def sensitive_fail_open_enabled(self) -> bool: return True logger.debug( - "sensitive tool gate: failing OPEN via NULLRUN_SENSITIVE_FAIL_OPEN=1 " - "(non-production api_url=%r).", + "the sensitive fail-open switch is ON (non-production " + "api_url=%r). Only tools the operator classified as " + "fail-open-allowed may run unchecked.", self.api_url, ) return True @@ -2241,7 +2344,6 @@ def check_workflow_budget( model: str | None = None, estimated_tokens: int | None = None, max_output_tokens: int | None = None, - use_gate_cache: bool = True, ) -> str | None: """ Pre-flight budget check via /api/v1/gate. Called from @protect @@ -2265,11 +2367,12 @@ def check_workflow_budget( zero-code path needs in order to reserve the call's own cost instead of one token's worth of it. max_output_tokens: overrides the call-context output cap. - use_gate_cache: set False by a path that needs its OWN - reservation per call. The chain-mode cache replays a - previous response — including its reservation id — so a - cached "allow" would hand a second LLM call the FIRST - call's reservation. + + Every call is a separate ``/gate`` call and mints its own + reservation. There is no in-process cache and no + ``use_gate_cache`` switch: a cached answer would hand a later + call an earlier call's reservation, and the backend settles a + reservation once (SDK-1). Decision → exception mapping: "allow" → return @@ -2307,17 +2410,34 @@ def check_workflow_budget( - Credential failure (``NullRunAuthenticationError``), malformed body, and ``NullRunUnclassifiedRefusalError`` all `raise` rather than fail open. Ordering matters: - each precedes the bare ``except Exception`` arm, which is - a superset of all three. + each precedes the transport arm below. + - A refusal the gate NAMED (``ENFORCEMENT_EXCEPTIONS``) + `raise`s, and the family arm comes FIRST — a + ``RateLimitError`` is a ``NullRunTransportError``, so a + later transport arm would swallow the 429 and fail OPEN + on it. - A non-fallback synthetic source is likewise treated as a transport error and fails open. - The rationale for the fail-open arms is that a transient backend + The transport arm is the ONLY fail-OPEN class (ADR-129). Anything + else that faults inside the gate call — the SDK's own code + raising while it asks the gate a question — leaves the decision + INDETERMINATE, and that raises ``NullRunGateIndeterminateError`` + instead of proceeding. The distinction is the whole point: + + - UNREACHABLE (transport failure): there is no answer to + honour, and nothing downstream can produce one. + - INDETERMINATE (any other fault): the backend may have + answered and the SDK cannot read it. An unanswered + question is not a yes. + + The rationale for the fail-open arm is that a transient backend outage must never freeze the user's agent, and `/track` does not gate on budget, so the worst case while `/gate` is unreachable is that enforcement is advisory until the gateway recovers. That rationale covers UNREACHABLE, not REFUSING — which is why the - ADR-062 arms were split out above. + ADR-062 arms were split out above — and not INDETERMINATE, which + is why the residual arm refuses (SDK-11). Uses `estimated_tokens=1` (the minimum the API accepts). Goal is the binary question "is there any budget left?", not cost @@ -2615,106 +2735,105 @@ def check_workflow_budget( # action by the backend's binding key. check_req["idempotency_key"] = check_req["operation_id"] - # In-process gate cache for chain-mode invocations. See - # module-top comment on _GATE_CACHE for full rationale. + # One /gate call per logical action. There is no in-process + # cache and no short-circuit for chain mode: a cached answer + # would carry an earlier call's reservation id, and the backend + # settles a reservation once — so the later steps of a chain + # would run tool calls that are never billed (SDK-1). See the + # module-top comment for the full mechanism. response: dict[str, Any] - cache_key: tuple[str, str | None, str | None, int] | None = None - cache_enabled = ( - chain_id is not None - and use_gate_cache - and not os.environ.get("NULLRUN_GATE_CACHE_DISABLE", "").strip() == "1" - ) - if cache_enabled: - cache_key = (str(workflow_id), chain_id, call_model, check_req.get("estimated_tokens", 1)) - cached = _GATE_CACHE.get(cache_key) - if cached is not None and (time.monotonic() - cached[0]) < _GATE_CACHE_TTL_SECONDS: - # Cache hit within TTL — reuse the response without a - response = cached[1] - else: - # Cache miss or expired — go to the server, then store. - try: - response = self._transport.check(check_req) - # The backend ANSWERED. A refusal (402/403/429) is - # still an answer and must not be read as - # unreachable. - self._note_backend_attempt(True) - except NullRunAuthenticationError: - # DEF-MP-TS12-ENF-01 (RUN_ID 20260929T1338, 2026-09-29): - # a 401 is a CREDENTIAL failure, not a transport - # failure, and must NOT be read as "allowed". - # - # `_retry_with_backoff` raises `NullRunAuthError` (a - # `NullRunAuthenticationError`) on any 401 and - # re-raises it WITHOUT retrying. Pre-fix the - # `except` clauses below swallowed it and returned - # None, which the caller reads as "no block" — the - # agent proceeded on a request the backend had - # refused. Classification is by TYPE here, never by - # inspecting the message. - raise - except NullRunUnclassifiedRefusalError: - # ADR-062 §2.2. The gate DID answer — it refused — - # and the SDK cannot tell a policy decision from a - # backend fault. That is not "gate unavailable": - # failing OPEN here would read an unclassifiable - # refusal as "allowed", which is the exact hole the - # category work closes. Must precede the arm below - # (a `NullRunError` superset). - raise - except (httpx.HTTPError, NullRunError) as exc: - # Narrow catch: fail-OPEN only on transport + - # classified SDK errors. Internal bugs - # (KeyError, AttributeError) surface rather - # than silently allowing an unbounded call. - self._note_backend_attempt(False) - logger.warning(f"check_workflow_budget: /gate unavailable, failing open: {exc}") - metrics.inc_runtime("gate_fail_open_total") - return None - except ValueError as exc: - # `Transport.check` ends in `response.json()`. A - # body that is not JSON — proxy error page, - # captive portal, TLS interception box — raises - # JSONDecodeError, a ValueError. Pre-fix this - # matched neither arm above and escaped as an - # untyped crash; and had it matched the - # fail-OPEN arm, a non-NULLRUN responder would - # have been read as "gate unavailable, carry on", - # which is precisely the authorisation-without- - # policy hole the missing `decision` default - # shares. It is a malformed ANSWER, not an - # unreachable gate, so it raises. - self._raise_malformed_gate_response(exc) - assert cache_key is not None # narrowed by cache_enabled above - _GATE_CACHE[cache_key] = (time.monotonic(), response) - else: - try: - response = self._transport.check(check_req) - # Same rationale as the cached branch: a refusal is an - # answer, not an outage. - self._note_backend_attempt(True) - except NullRunAuthenticationError: - # Same rationale as the cached branch above. Ordering - # matters: this arm precedes the broad `except - # Exception`, which is a superset. - raise - except ValueError as exc: - # Same rationale as the cached branch: a body that is - # not JSON is a malformed answer, and must precede the - # broad `except Exception` below, which is a superset - # of this arm. - self._raise_malformed_gate_response(exc) - except NullRunUnclassifiedRefusalError: - # ADR-062 §2.2 — same rationale as the cached branch. - # This is a refusal the SDK cannot classify, not an - # unreachable gate, so it must not fail open. It is - # listed explicitly because the arm below is a bare - # ``except Exception``. - raise - except Exception as exc: # noqa: BLE001 - self._note_backend_attempt(False) - logger.warning(f"check_workflow_budget: /gate unavailable, failing open: {exc}") - metrics.inc_runtime("gate_fail_open_total") - return None + try: + response = self._transport.check(check_req) + # The backend ANSWERED. A refusal (402/403/429) is still an + # answer and must not be read as unreachable. + self._note_backend_attempt(True) + except ENFORCEMENT_EXCEPTIONS: + # ADR-063 §2.4 row 2, as amended by ADR-129. A refusal the + # gate NAMED is never downgraded to "the gate was + # unreachable, carry on". + # + # The family arm comes FIRST because the ordering is + # load-bearing: `RateLimitError` is a + # `NullRunTransportError`, so the transport arm below would + # swallow the 429 and fail OPEN on it. + raise + except NullRunAuthenticationError: + # DEF-MP-TS12-ENF-01 (RUN_ID 20260929T1338, 2026-09-29): + # a 401 is a CREDENTIAL failure, not a transport failure, + # and must NOT be read as "allowed". + # + # `_retry_with_backoff` raises `NullRunAuthError` (a + # `NullRunAuthenticationError`) on any 401 and re-raises it + # WITHOUT retrying. Pre-fix the `except` clauses below + # swallowed it and returned None, which the caller reads as + # "no block" — the agent proceeded on a request the backend + # had refused. Classification is by TYPE here, never by + # inspecting the message. + raise + except ValueError as exc: + # `Transport.check` ends in `response.json()`. A body that + # is not JSON — proxy error page, captive portal, TLS + # interception box — raises JSONDecodeError, a ValueError. + # Pre-fix this matched neither arm above and escaped as an + # untyped crash; and had it matched the fail-OPEN arm, a + # non-NULLRUN responder would have been read as "gate + # unavailable, carry on", which is precisely the + # authorisation-without-policy hole the missing `decision` + # default shares. It is a malformed ANSWER, not an + # unreachable gate, so it raises. + self._raise_malformed_gate_response(exc) + except NullRunUnclassifiedRefusalError: + # ADR-062 §2.2. The gate DID answer — it refused — and the + # SDK cannot tell a policy decision from a backend fault. + # That is not "gate unavailable": failing OPEN here would + # read an unclassifiable refusal as "allowed", which is the + # exact hole the category work closes. Listed explicitly + # because the arm below catches transport errors by TYPE. + raise + except (httpx.HTTPError, NullRunTransportError) as exc: + # ADR-008's fail-OPEN — and the ONLY class it covers. The + # gate could not be REACHED, so there is no answer to + # honour: the enforcement path is un-decidable, not + # undecided. A dead backend must not freeze the agent. + # + # Everything that is NOT a transport failure falls through + # to the arm below, which refuses. That is the whole point + # of the split (ADR-129): pre-fix this arm was a bare + # ``except Exception``, so an SDK-internal fault was + # laundered into "allowed" and the call ran with no + # reservation. + self._note_backend_attempt(False) + logger.warning(f"check_workflow_budget: /gate unavailable, failing open: {exc}") + metrics.inc_runtime("gate_fail_open_total") + return None + except Exception as exc: # noqa: BLE001 + # NOT a transport failure. The backend may have answered; + # the SDK's own code faulted while asking it a question + # (a request body that cannot be serialised, a decode + # error, a fault in the edge path's token split). The + # decision is INDETERMINATE, and an indeterminate decision + # is not permission. + # + # ADR-063 §2.4's governing rule: a fail-OPEN is acceptable + # only when the check is genuinely un-decidable AND + # something downstream can correct it. This is neither: + # the check is decidable in principle, and the only thing + # downstream is `/track` — which does not gate on budget. + # + # The refusal is typed and counted, not logged and + # forgotten: a log line is not a control, and the operator + # needs to see that the fleet is refusing calls. + self._note_backend_attempt(False) + metrics.inc_runtime("gate_indeterminate_total") + logger.error( + "check_workflow_budget: /gate failed with a non-transport " + f"error, refusing the call: {exc!r}" + ) + raise NullRunGateIndeterminateError( + "the /gate call failed with a non-transport error, so the " + f"SDK could not determine a decision: {exc!r}", + cause=exc, + ) from exc # The reservation id the backend minted for THIS call. It is # captured into the server-minted contextvar (the legacy @@ -3317,11 +3436,10 @@ def chain_end(self, chain_id: str) -> dict[str, Any]: organization_id=self.organization_id, trace_id=get_trace_id(), ) - # Drop the in-process gate cache for this chain AFTER the wire - # call so a transient transport failure does not free the - # cache before the server confirms closure. - workflow_id_str = str(self.workflow_id) if self.workflow_id else None - _invalidate_gate_cache_for_chain(workflow_id_str, chain_id) + # There is no in-process gate cache to drop: every step of a + # chain is admitted by its own /gate call and holds its own + # reservation, so closing the chain frees nothing locally + # (SDK-1). return result def approximate_budget(self) -> dict[str, Any]: @@ -3650,6 +3768,99 @@ def get_sensitive_tools(self) -> set[str]: """ return self._sensitive_tools | self._strict_mode_tools + # ============================================================================= + # Fail-open classification — the per-tool bound on + # NULLRUN_SENSITIVE_FAIL_OPEN + # ============================================================================= + + def register_fail_open_tools(self, tool_names: list[str]) -> None: + """Classify tools as ones the fail-open switch may open. + + ``NULLRUN_SENSITIVE_FAIL_OPEN=1`` lets a protected tool's body + run while the policy engine is unreachable. Before this + classification existed the switch applied to EVERY protected + tool, so one operator decision (set the variable) removed + enforcement from the whole agent. + + The switch is now bounded by this classification: it opens a + tool ONLY if that tool was registered here. The set is empty + by default, so the switch alone opens nothing and every tool + stays fail-CLOSED. + + This is a code-level classification on purpose. It is not an + environment variable, so it cannot be inherited by a deploy + pipeline or a CI job that did not choose it — it is a + deliberate act by whoever writes the agent. + + This is NOT the sensitivity registry (``add_sensitive_tool`` + and friends). That registry answers "is this tool + destructive?"; this one answers "may this tool run unchecked + during an outage?". A tool may be in both, and the two + answers are independent. + + Args: + tool_names: Names of the tools to classify. Match is + case-insensitive, like the sensitivity registry. + + Example: + runtime = NullRunRuntime.get_instance() + runtime.register_fail_open_tools(["charge_card"]) + """ + with self._tools_lock: + for tool_name in tool_names: + self._fail_open_tools.add(tool_name) + # Rebuild the lowercase snapshot once after the batch + # insert (a single set comprehension beats N rebuilds + # in the loop). + self._fail_open_tools_lower = frozenset( + t.lower() for t in self._fail_open_tools + ) + + def remove_fail_open_tool(self, tool_name: str) -> None: + """Un-classify a tool: the fail-open switch stops applying to it. + + Args: + tool_name: Name of the tool to remove from the fail-open + classification. + + Example: + runtime = NullRunRuntime.get_instance() + runtime.remove_fail_open_tool("charge_card") + """ + with self._tools_lock: + self._fail_open_tools.discard(tool_name) + self._fail_open_tools_lower = frozenset( + t.lower() for t in self._fail_open_tools + ) + + def get_fail_open_tools(self) -> set[str]: + """The tools currently classified as fail-open-allowed. + + Returns: + A copy of the classified tool names, so a caller cannot + mutate the runtime's state through the returned set. + """ + with self._tools_lock: + return set(self._fail_open_tools) + + def is_fail_open_allowed(self, tool_name: str) -> bool: + """Whether the fail-open switch may open this tool. + + This is the per-tool bound on ``NULLRUN_SENSITIVE_FAIL_OPEN``. + It is a pure classification lookup: it does not read the + environment and does not decide whether the switch is on. + + Args: + tool_name: Name of the tool. + + Returns: + True when the operator classified this tool as one that + may run unchecked while the policy engine is unreachable. + """ + needle = tool_name.lower() + with self._tools_lock: + return needle in self._fail_open_tools_lower + def execute( self, tool_name: str, @@ -3970,9 +4181,13 @@ def execute( # then echoes the new id back via ``reservation_id`` # (mirrored by the backend's GateResponse::require_approval # constructor — see backend/src/enforcement/gate_wire_adapter.rs). - # Capture here is fail-OPEN (drops malformed values - # silently via the helper's UUID parse), matching - # ``check_workflow_budget``'s behaviour. + # + # The capture REFUSES a present-but-unusable id (SDK-10) — + # it is not fail-OPEN. The old comment here said "fail-OPEN + # (drops malformed values silently via the helper's UUID + # parse)", which described the defect: a dropped reservation id + # meant the tool ran with no reservation, so its spend was + # never booked. _capture_server_minted_execution_id(result) # Sync the kwargs dict to the captured id so the post-approval @@ -4235,6 +4450,11 @@ class (e.g. ``NullRunApprovalReplayRejectedError`` for "NullRunApprovalReplayRejectedError": frozenset({"approval_id"}), "NullRunApprovalDigestMismatchError": frozenset({"approval_id"}), "NullRunApprovalToolDigestMismatchError": frozenset({"approval_id"}), + # DEC-5 / DEC-7: the approval binds a TOOL SET; the + # executing tool must be a member. Same promotion as the + # two digest siblings — ``approval_id`` lands as a + # first-class attr, not only inside ``details``. + "NullRunApprovalToolMismatchError": frozenset({"approval_id"}), } def _build_payload( @@ -4551,10 +4771,25 @@ def _route_track( Failure mode ------------ ``track_single`` raises on 422 / 503 / 5xx (see - ``nullrun.breaker.exceptions``). We catch and log at - WARNING level; the event is dropped (NOT retried via - the batch path — that would risk double-billing the - idempotency contract). + ``nullrun.breaker.exceptions``). + + A backend ANSWER raises. The consume did not land, so + the caller must not be told the track succeeded. The + payload goes to the v3 WAL first, so the refusal is + raised AND recoverable (ADR-183). + + An enforcement verdict also raises, and it is NOT + written to the WAL. The backend will answer the same + way next time. + + A request that never reached the backend keeps the + ADR-008 fail-OPEN row. The payload still goes to the + v3 WAL, the failure is logged at WARNING, and the + caller is not blocked. + + The event is never retried via the batch path. That + endpoint does not run the CONSUME script, so a + reservation replayed through it is never settled. """ from nullrun.context import get_server_minted_execution_id @@ -4613,36 +4848,44 @@ def _route_track( # DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET: when the backend # refuses the consume with HTTP 422 (CONSUME_OVERBUDGET) # or HTTP 402 (REDIS_UNAVAILABLE on the consume path) - # the chain has hit its budget ceiling. Any cached - # "allow" for the same (workflow_id, chain_id) must NOT - # be served for the next 0–5 s — otherwise a chain firing - # faster than the cache TTL over-reserves against a - # budget the server has just rejected. Invalidate before - # logging so the order in logs matches the order in code. + # the chain has hit its budget ceiling. # - # chain_id lives in the contextvar (set by the - # ``with chain(...)`` contextmanager or - # ``set_chain_id(...)`` manual setter), NOT on the - # wire_event — wire_event is the per-call track dict - # and the chain_id is implicit (the backend re-derives - # it from the reservation binding on /track). Reading - # from ``get_chain_id()`` ensures we invalidate ONLY - # the failing chain's cache entries, not every chain - # for the same workflow. - from nullrun.context import get_chain_id - + # Pre-fix the SDK also dropped an in-process gate cache + # here, so the next step of the chain could not be served + # a stale "allow". There is no such cache any more: every + # step is admitted by its own /gate call, and a refused + # consume is refused by the server, which is the only + # place the budget lives (SDK-1). status_code = getattr(exc, "status_code", None) - if status_code in (402, 422): - _invalidate_gate_cache_for_chain( - wire_event.get("workflow_id"), - get_chain_id(), - ) _emit_for_transport_error( exc, stage="track_v3_single", correlation_id=smid, status_code=status_code, ) + # SDK-1 (security audit 2026-11-06). The tool call has + # ALREADY run and the money has ALREADY been spent. The + # only thing that keeps that spend from being lost is a + # durable copy of the consume payload, so it is written + # BEFORE anything decides whether to raise. + # + # The payload is written VERBATIM — `reservation_id` and + # `idempotency_key` included — and the WAL replay sends it + # back to `/api/v1/track` (single). The batch endpoint is + # NOT a recovery route: it does not run the CONSUME + # script, so a reservation replayed through it is never + # settled and the money is never booked. + # + # A determinate verdict is NOT persisted. A + # `NullRunDecision` (budget, approval, chain, + # consume-overbudget) is the backend's ANSWER: re-sending + # it produces the same answer and a row that could never + # leave the WAL. + if not isinstance(exc, NullRunDecision): + if self._transport.persist_v3_track(single_payload): + metrics.inc_runtime("v3_track_persisted_to_wal") + else: + metrics.inc_runtime("v3_track_wal_persist_failed") # DEF-TC6-006 (2026-10-02, QA RUN_ID 20261002T0826): # a bare `except Exception` here laundered an ADR-005 # enforcement rejection into a transport warning. The @@ -4671,10 +4914,42 @@ def _route_track( # plain 5xx stay in the transport class and keep # dropping, because they name no enforcement failure # and raising on them would be the over-correction. + # (PRE-SDK-1 RULE. Superseded by SDK-1 below — a + # protocol error and a plain 5xx on /track are refusals + # of the CONSUME, not of the check.) # The invalidation above runs first either way — the # cached-allow blast radius (DEF-CACHE-STALE-ALLOW- # AFTER-OVERBUDGET) is closed before the raise, not # traded away for it. + # + # SDK-1 (security audit 2026-11-06) changes this. The + # scope above was WRONG on the money path, and the + # exception hierarchy is why it was wrong: a backend + # refusal (`NullRunBackendError`) sits under + # `NullRunInfrastructureError`, a SIBLING of + # `NullRunDecision`, so `isinstance(exc, NullRunDecision)` + # never caught a 503 `RESERVATION_NOT_FOUND` or a 500 + # `INTERNAL_ERROR`. Those are refusals of the CONSUME: + # the tool call has already run, the money has already been + # spent, and the backend refused to book it. Reporting + # `{"allowed": True}` for that is a lost spend with a + # success label on it. + # + # The refusal is now decided BY THE WIRE (see + # `transport.is_backend_refusal`), not by the exception + # class: the question is "did the backend ANSWER?", and the + # answer is on the exception as the HTTP status of the + # response. A request that never reached the backend stays + # a transport failure and keeps dropping — that is the + # ADR-008 fail-open row, and it is unchanged. + # + # The class list this replaced named three classes and left + # nine backend codes outside them: the `PROTOCOL_*` family, + # `IDEMPOTENCY_CONFLICT`, `IDEMPOTENCY_KEY_MISMATCH` and + # the three `RATE_LIMIT_*` codes. The backend rate-limits + # `/track`, so a 429 there is LIVE — and it was a dropped + # spend. The taxonomy of an answer does not change what the + # SDK owes the caller: the consume did not land. if isinstance(exc, NullRunDecision): logger.warning( "_route_track: /track refused the consume for " @@ -4684,6 +4959,17 @@ def _route_track( exc, ) raise + if is_backend_refusal(exc): + logger.warning( + "_route_track: /track refused the consume for " + "execution_id=%s (%s) — propagating (SDK-1: the " + "backend answered and refused, so the consume did " + "not land and the refusal must not be reported as " + "a successful track)", + smid, + exc, + ) + raise logger.warning( "_route_track: track_single failed for execution_id=%s (%s) — event dropped", smid, @@ -4929,15 +5215,66 @@ def __getattr__(name): raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +def _refuse_malformed_reservation(reason: str) -> None: + """Refuse an answer whose ``reservation_id`` cannot be bound. + + SDK-10 (reliability audit 2026-11-02). The capture used to log a + warning and return ``None`` for a ``reservation_id`` that is not a + UUID — reading "the server handed back an id I cannot use" as + "the server reserved nothing". Those are two different facts, and + the difference is money: + + * the backend's ``/gate`` Allow arm threads the Step-9 Lua + reserve's server-minted ``uuidv7`` into ``GateResponse`` + ``reservation_id`` (the dispatcher's third tuple position), and + the ``/execute`` arm reuses the same id (ADR-073), so a REAL + answer always carries a UUID there; + * an id that is present and NOT a UUID therefore did not come from + the reserve path — a non-NULLRUN responder (captive portal, + TLS-interception box), a corrupted proxy hop, or a backend that + minted something else. + + Dropping it silently let the call run with NO reservation: the + ``/track`` event had no ``execution_id`` to carry, so the spend + was never booked, and the backend's reserve sat on the period + counter until its TTL ran out. The agent got the call for free. + + So the answer is REFUSED, not downgraded. The class is the one the + SDK already uses for "something answered, and what it said was not + a verdict" (``_require_gate_decision``): the reservation id is part + of the verdict, because it is what binds the money. + + Always raises; the ``-> None`` return type lets the call sites use + it as the tail of an arm. + """ + from nullrun.breaker.exceptions import NullRunMalformedGateResponseError + + logger.error( + "_capture_server_minted_execution_id: response.reservation_id " + "cannot be bound (%s) — refusing the call", + reason, + ) + metrics.inc_runtime("gate_malformed_reservation_total") + raise NullRunMalformedGateResponseError( + f"/gate returned a reservation_id that cannot be bound: {reason}. " + f"That id is what ties this call's spend to the reservation the " + f"server minted, so the SDK refuses the call instead of running it " + f"unbilled." + ) + + # The module-level slot is a proxy over the registry. The def _capture_server_minted_execution_id(response: dict[str, Any]) -> str | None: """Capture ``response["reservation_id"]`` into the server-minted execution_id contextvar. - Returns the captured id (or ``None`` on miss / malformed) so the - caller can log it on debug paths. The contextvar itself is the - authoritative side-effect — readers consult - ``get_server_minted_execution_id`` from ``nullrun.context``. + Returns the captured id, or ``None`` when the answer carries NO + reservation (a block reserves nothing). A reservation id that is + PRESENT but unusable is a different case — it is an answer the SDK + cannot act on, and it REFUSES (SDK-10, ``_refuse_malformed_reservation``). + + The contextvar itself is the authoritative side-effect — readers + consult ``get_server_minted_execution_id`` from ``nullrun.context``. Import is lazy (inside the function) to keep ``nullrun.runtime`` import order stable: ``context`` itself @@ -4954,37 +5291,33 @@ def _capture_server_minted_execution_id(response: dict[str, Any]) -> str | None: ) raw = response.get("reservation_id") if isinstance(response, dict) else None - if not raw: + if raw is None: # Block response with no reservation_id; clear so the - # previous scope's value can't leak. + # previous scope's value can't leak. An ABSENT id means the + # server reserved nothing — that is a fact, not a fault. clear_server_minted_execution_id() return None if not isinstance(raw, str): - clear_server_minted_execution_id() - logger.warning( - "_capture_server_minted_execution_id: response.reservation_id " - "is %s, expected str — dropping", - type(raw).__name__, + _refuse_malformed_reservation( + f"it is {type(raw).__name__}, expected a string" ) - return None - # Defence-in-depth UUID parse — backend's mint_execution_id - # emits RFC-4122 uuidv7 but a buggy proxy could echo garbage. - # Drop without raising (fail-OPEN on capture; the backend will - # still reject malformed ids with 400 on /track). + if not raw: + _refuse_malformed_reservation("it is an empty string") + + # The backend's ``/gate`` Allow arm threads the Step-9 reserve's + # server-minted ``uuidv7`` into ``reservation_id``, and the + # ``/execute`` arm reuses that same id (ADR-073) — so a real + # answer always carries a UUID here. A present-but-unparseable id + # did not come from the reserve path, and the spend it would have + # paid for cannot be bound to it. Refuse (SDK-10). import uuid as _uuid try: _uuid.UUID(raw) except (ValueError, AttributeError): - clear_server_minted_execution_id() - logger.warning( - "_capture_server_minted_execution_id: response.reservation_id=%r " - "is not a valid UUID — dropping", - raw, - ) - return None + _refuse_malformed_reservation(f"{raw!r} is not a valid UUID") set_server_minted_execution_id(raw) set_server_minted_reservation_at(_time.monotonic()) diff --git a/src/nullrun/toolbox/mcp.py b/src/nullrun/toolbox/mcp.py index 364db51..17073c9 100644 --- a/src/nullrun/toolbox/mcp.py +++ b/src/nullrun/toolbox/mcp.py @@ -18,7 +18,10 @@ Scope (kept deliberately small): * Cache `tools/list` for ``MCP_ADAPTER_CACHE_SECONDS`` - (default 300s, matches the gate's ``heartbeat`` cadence). + (default 30s). The cached annotations are a policy input for + the gate's MCP umbrella policies, so the window is short; a + failed refresh never presents the stale snapshot to the gate + as if it were current. * On every ``call_tool(name, args)``, set ``call_mcp_class='mcp'`` + ``call_mcp_annotations=...`` via the public ``context`` helpers so the runtime's @@ -54,22 +57,34 @@ import logging import time from collections.abc import Callable, Iterable +from contextvars import Token from dataclasses import dataclass from typing import Any +from nullrun.business_impact import BusinessImpact, compute_action_digest from nullrun.context import ( + get_call_impact, + reset_call_impact, + set_call_impact, set_mcp_tool_context, ) logger = logging.getLogger(__name__) -# Cache TTL for the ``tools/list`` discovery response. Matches -# the v3.31 gate's ``heartbeat`` cadence so the adapter and the -# gate see roughly the same version of the server's tool inventory -# over time. Operators who need a tighter or looser TTL can -# override it via the constructor. -DEFAULT_CACHE_SECONDS = 300 +# Cache TTL for the ``tools/list`` discovery response. +# +# The cached annotations are a POLICY INPUT: the gate's +# ``mcp_destructive_policy`` / ``mcp_readonly_policy`` umbrella +# policies are evaluated against the ``destructive`` / ``read_only`` +# hints this cache hands to ``/check``. A cached ``destructive=False`` +# that the server has since changed to ``True`` is a stale policy +# input, so the window is kept short. +# +# 30 seconds is the floor the constructor already enforces. Operators +# who cannot accept any staleness pass ``cache_seconds=0`` and pay +# one ``tools/list`` round trip per tool call. +DEFAULT_CACHE_SECONDS = 30 @dataclass(frozen=True) @@ -131,6 +146,47 @@ def read_ann(hint: str) -> bool | None: ) +def _derive_mcp_impact( + tool_name: str, + arguments: dict[str, Any], +) -> tuple[BusinessImpact, dict[str, Any], str] | None: + """Derive the ONE envelope that describes one MCP tool call. + + ADR-125. The envelope is a ``tool_call`` impact carrying the tool + name and the EXACT argument dict the adapter forwards as the + canonical bag (``tool_arguments``). The backend re-derives the + digest from the live payload, compares it with the digest stored + on the approval row, and then compares the envelope's ``params`` + with the live bag — so the params MUST be that same dict, not a + masked or reshaped copy of it. A reshaped copy would bind the + operator's approval to arguments nobody approved. + + Returns ``(impact, wire_dict, digest)``, or ``None`` when the + envelope cannot be built: the digest layer cannot round-trip the + arguments (an f64, a non-ASCII tool name, an over-long parameter + name). The caller then sends the ``no_impact`` digest with NO + envelope, which the backend refuses at consume time — the + fail-CLOSED arm, not the legacy "no digest at all" arm that binds + an approval to nothing. + """ + if not isinstance(arguments, dict): + # A non-mapping argument bag is not a tool payload this + # envelope can describe. + return None + try: + impact = BusinessImpact.tool_call(tool_name, arguments) + except (ValueError, TypeError, AttributeError) as exc: + logger.warning( + "MCPAdapter: could not derive a tool_call BusinessImpact for %r (%s); " + "the call goes on the wire with the no_impact digest and NO envelope, " + "so an operator approval for it cannot be consumed.", + tool_name, + exc, + ) + return None + return impact, impact.to_wire_dict(), compute_action_digest(impact) + + class MCPAdapter: """Forward MCP-aware metadata (``tool_class`` + ``mcp_annotations``) for every tool call from a connected @@ -191,12 +247,17 @@ def __init__( ) -> None: if not server_name: raise ValueError("MCPAdapter: server_name is required") - if cache_seconds < 30: - # Less than 30s would hammer the upstream and + if cache_seconds < 0 or 0 < cache_seconds < 30: + # A value between 0 and 30 would hammer the upstream and # skew the v3.31 ``mcp_observed_tools`` drift # detector's notion of "stable tool inventory". + # + # 0 is allowed and means "no cache": the annotations are + # fetched fresh before every tool call, at the cost of one + # ``tools/list`` round trip per call. It is the operator's + # only way to remove the staleness window entirely. raise ValueError( - f"MCPAdapter: cache_seconds must be >= 30 (got {cache_seconds})" + f"MCPAdapter: cache_seconds must be 0 or >= 30 (got {cache_seconds})" ) self.server_name = server_name self._mcp_client = mcp_client @@ -211,6 +272,13 @@ def __init__( # call_tool, refreshed every ``cache_seconds``. self._cache: dict[str, _CachedTool] = {} self._cached_at: float = 0.0 + # False when the last refresh attempt FAILED. The snapshot in + # ``self._cache`` is then older than the TTL and must not be + # presented to the gate as if it were the server's current + # answer — a stale ``destructive=False`` is a stale policy + # input. Diagnostics may still read the snapshot; the gate + # gets the unknown shape instead. + self._cache_fresh: bool = False # every ``call_tool``. When provided, ``call_tool`` blocks on # ``runtime.execute(...)`` returning decision="block" so a # permissive MCP server cannot bypass the operator's @@ -278,6 +346,7 @@ def _refresh_cache(self) -> None: fresh[cached.name] = cached self._cache = fresh self._cached_at = time.monotonic() + self._cache_fresh = True logger.debug( "MCPAdapter: refreshed tool cache for %s (%d tools)", self.server_name, @@ -285,9 +354,14 @@ def _refresh_cache(self) -> None: ) except Exception as exc: # noqa: BLE001 # Cache refresh is best-effort. If the upstream is - # down we'll keep using the previous cache rather - # than failing every call. Operators see stale - # data in the audit log; the next call will retry. + # down we keep the previous snapshot for DIAGNOSTICS + # rather than failing every call. + # + # The snapshot is NOT presented to the gate as current + # truth: ``_cache_fresh`` goes False and ``call_tool`` + # stamps the unknown shape instead, so the gate falls + # back to its own classifier. The next call retries. + self._cache_fresh = False logger.warning( "MCPAdapter: %s tools/list refresh failed: %s (keeping %d cached)", self.server_name, @@ -296,8 +370,14 @@ def _refresh_cache(self) -> None: ) def _maybe_refresh(self) -> None: + # ``cache_seconds == 0`` means "no cache": every access + # re-lists. The comparison is on the clock, so it is spelled + # out here rather than left to ``elapsed > 0`` -- a clock + # that does not tick between two calls would silently keep + # the cache. if ( - not self._cache + self._cache_seconds <= 0 + or not self._cache or (time.monotonic() - self._cached_at) > self._cache_seconds ): self._refresh_cache() @@ -354,6 +434,17 @@ def call_tool( consume time, so a tampered payload is refused (NR-015, SDK half). + **When the caller supplies NEITHER, the adapter derives + both from the tool call itself** (ADR-125): a ``tool_call`` + envelope carrying the tool name and the exact argument bag + it forwards, plus that envelope's digest. Without them the + approval row carries no digest at all, and the backend's + post-approval ``/execute`` re-entry cannot server-derive + one — it refuses the consume with ``BUSINESS_IMPACT_INVALID`` + (NR-010), so an operator approval for an MCP tool could + never be spent, and even where it could, the approval would + bind the tool NAME but not the ARGUMENTS. + Gate enforcement: ``call_tool`` ALWAYS routes the invocation through ``runtime.execute(...)`` (the /api/v1/execute gate endpoint) BEFORE the underlying MCP client is called. The @@ -371,11 +462,23 @@ def call_tool( EXECUTION_NOT_FOUND when the ``execution:{id}`` binding is missing. The ``@protect`` path gets that ``/gate`` from its own pre-flight; a DIRECT ``adapter.call_tool(...)`` has no such - wrapper, so the adapter runs the pre-flight ITSELF when no - server-minted execution id is in scope. When one IS in scope - the pre-flight is SKIPPED: that path reserves money at the - gate, and a second ``/gate`` would charge the org twice for - one logical tool call. + wrapper, so the adapter runs the pre-flight ITSELF. + + The pre-flight runs for EVERY tool call, because every tool + call is its own action with its own budget. Two cases skip it, + and only those two: + + * the approval re-entry — the caller carries an + ``approval_id``, so the ambient server-minted id IS the + approval row's execution id, and a second ``/gate`` would + mint a different one; + * the ``@protect`` scope — the scope already reserved for + itself, and a second ``/gate`` would reserve twice and + orphan the first (NR-059). + + A second tool call in the same scope is NOT one of those two. + It gets its own ``/gate``, its own reservation, its own + execution id. The backend settles a reservation once (SDK-1). B2 (2026-09-30): there is no ungated path. ``runtime=None`` used to mean "skip the round-trip entirely"; it now means @@ -391,13 +494,33 @@ def call_tool( """ self._maybe_refresh() cached = self._cache.get(tool_name) - if cached is None: + if not self._cache_fresh: + # The last refresh FAILED, so the snapshot in the cache is + # older than the TTL and may no longer describe the server. + # Hand the gate the unknown shape instead of a stale + # answer: the umbrella MCP policies then fall back to the + # gate's own classifier rather than to a cached + # ``destructive=False`` the server may have retired. + annotations: dict[str, Any] | None = { + "read_only": None, + "destructive": None, + "open_world": None, + } + tool_class = "invalid" + logger.debug( + "MCPAdapter: tools/list refresh failed for %s; stamping " + "class=invalid on /check for tool %r instead of the stale " + "cached annotations", + self.server_name, + tool_name, + ) + elif cached is None: # Unknown tool. Be honest with the gate: this is # not a recognised MCP server's tool. The gate # will fall through to its ``classify_tool`` # parser (which would have classified this as # ``invalid`` anyway given the cached inventory). - annotations: dict[str, Any] | None = { + annotations = { "read_only": None, "destructive": None, "open_world": None, @@ -441,6 +564,116 @@ def call_tool( runtime = self._resolve_runtime() execute_input = arguments if arguments is not None else {} + # Is a scope-level envelope ALREADY in scope? Read BEFORE the + # adapter resolves its own envelope below, because the adapter + # installs one of its own and that would make this answer + # "yes" for every call. + # + # The ``@protect`` path is the only thing in the SDK that + # installs one, and it installs it for the WHOLE scope — after + # its own ``/gate`` reserved for that scope. So "an envelope + # was already here" is the marker of a scope that already + # gated itself. + scope_already_gated = get_call_impact() is not None + + # ── The ONE envelope for this logical action (ADR-125) ───── + # + # The adapter derives the envelope from the tool call it is + # about to make: the tool name and the EXACT argument dict it + # forwards as the canonical bag. Without it the approval row + # carries NO digest, and the backend's post-approval + # /execute re-entry cannot server-derive the digest at all — + # it refuses the consume with BUSINESS_IMPACT_INVALID (NR-010), + # so an operator approval for an MCP tool could never be + # spent. See ADR-125 §1. + business_impact, action_digest, impact_token = self._resolve_impact( + tool_name=tool_name, + arguments=execute_input, + business_impact=business_impact, + action_digest=action_digest, + ) + try: + return self._call_tool_gated( + runtime=runtime, + tool_name=tool_name, + arguments=arguments, + execute_input=execute_input, + approval_id=approval_id, + business_impact=business_impact, + action_digest=action_digest, + passthrough=passthrough, + scope_already_gated=scope_already_gated, + ) + finally: + if impact_token is not None: + reset_call_impact(impact_token) + + def _resolve_impact( + self, + *, + tool_name: str, + arguments: dict[str, Any], + business_impact: dict[str, Any] | None, + action_digest: str | None, + ) -> tuple[dict[str, Any] | None, str | None, Token[BusinessImpact | None] | None]: + """Resolve the envelope this call goes on the wire with. + + Returns ``(business_impact, action_digest, impact_token)``. + + * A caller that brought its OWN envelope (or its own digest) + keeps it: the SDK cannot reconstruct the caller's envelope, + so inventing a digest for it would be a claim the backend + cannot verify. Forward it verbatim. + * Otherwise the adapter derives the envelope from the call + itself and installs it on the call context, so the + pre-flight ``/gate`` hashes the SAME bytes this ``/execute`` + carries (ADR-065: one envelope per logical action). + * An envelope already in scope (the ``@protect`` path) is NOT + overwritten — that path's own ``/gate`` and ``/execute`` must + keep agreeing with each other. + * When no envelope can be built at all, the digest still goes + on the wire (over the ``no_impact`` sentinel) and the + envelope stays ABSENT. The backend then takes its + fail-CLOSED arm ("cannot recompute") instead of the legacy + "no digest at all" arm, which is a grant bound to nothing. + """ + if business_impact is not None or action_digest is not None: + return business_impact, action_digest, None + + derived = _derive_mcp_impact(tool_name, arguments) + if derived is None: + return ( + None, + compute_action_digest(BusinessImpact.no_impact()), + None, + ) + + impact, wire, digest = derived + token: Token[BusinessImpact | None] | None = None + if get_call_impact() is None: + token = set_call_impact(impact) + return wire, digest, token + + def _call_tool_gated( + self, + *, + runtime: Any, + tool_name: str, + arguments: dict[str, Any] | None, + execute_input: dict[str, Any], + approval_id: str | None, + business_impact: dict[str, Any] | None, + action_digest: str | None, + passthrough: dict[str, Any], + scope_already_gated: bool, + ) -> Any: + """The gated half of :meth:`call_tool`. + + Split out so the caller can hold ONE envelope for the whole + logical action: the envelope is installed on the call context + before the pre-flight ``/gate`` runs and released after this + method returns, whatever it returns. + """ # ── Pre-flight: /execute needs a prior /gate ─────────────── # # The backend's ``/api/v1/execute`` runs an existence check on @@ -452,13 +685,30 @@ def call_tool( # has no such wrapper, so without this pre-flight the /execute # round trip was refused and the MCP server was never called. # - # The pre-flight is CONDITIONAL, and the condition is the - # server-minted execution id: when one is already in scope a - # ``/gate`` has already run for this logical action, and - # running it again would reserve the budget a second time - # (that path charges money at the gate). The contextvar is - # exactly what ``runtime.execute`` reads to decide whether to - # reuse the id, so the two agree by construction. + # The pre-flight runs for EVERY tool call. A tool call is its + # own action with its own budget, so it gets its own ``/gate``, + # its own reservation and its own execution id. + # + # The condition used to be "no server-minted execution id in + # scope". That id is set by ``/gate`` and by ``/execute`` and is + # NEVER reset in ``src/``, so the second and later tool calls + # in one scope ran NO ``/gate``: no budget check, no + # reservation, no spend reserved for those calls. The id says + # "some action was gated", not "THIS action was gated". + # + # Two cases are genuinely covered by an ambient reservation, + # and both are about THIS action: + # + # * the approval re-entry — the caller carries an + # ``approval_id``, and the backend's require_approval arm + # echoes the approval row's execution id into + # ``reservation_id``. The ambient id IS the approval's + # execution, so a second ``/gate`` would mint a DIFFERENT + # id and the approval's consume (``WHERE execution_id = + # $3``) would miss its row (APPROVAL_REPLAY_REJECTED). + # * the ``@protect`` scope — the scope reserved for itself, + # and a second ``/gate`` inside it would reserve twice and + # orphan the first (NR-059). # # A refusal from the pre-flight propagates: the MCP client is # NOT called. Same ordering and same error handling as the @@ -466,7 +716,27 @@ def call_tool( # pre-flight → /execute → body). from nullrun.context import get_server_minted_execution_id - if get_server_minted_execution_id() is None: + # Via-edge. The BOX is the enforcement surface there: its + # ``/enforce`` call is what prices AND charges the call, and + # there is no cloud ``/track`` to book it through afterwards. + # So every tool call needs its own ask — one box call, one + # charge, per tool call. + # + # The "a /gate already ran for this action" short-circuit is + # correct for the cloud (one /gate reserves, one /execute + # reuses that reservation, one /track settles it) and wrong + # for the edge (one /enforce per tool call). Skipping the ask + # on the second tool call would leave the box unconsulted, and + # with the SDK-2 fix the /execute then fails CLOSED because + # nobody asked — so the second and later MCP calls in an edge + # scope must ask the box themselves. + _transport = getattr(runtime, "_transport", None) + edge_mode = getattr(_transport, "edge", None) is not None + + ambient_id = get_server_minted_execution_id() + covered = (approval_id is not None) or scope_already_gated + + if edge_mode or ambient_id is None or not covered: runtime.check_workflow_budget() # Every MCP tool call routed through the runtime contacts diff --git a/src/nullrun/transport.py b/src/nullrun/transport.py index 7ac8559..83bb5e5 100644 --- a/src/nullrun/transport.py +++ b/src/nullrun/transport.py @@ -11,7 +11,6 @@ import logging import os import random -import tempfile import threading import time import uuid @@ -42,10 +41,17 @@ NullRunMcpDestructiveBlockedError, NullRunMcpReadonlyBypassBlockedError, NullRunTransportError, + NullRunUnknownActionError, RateLimitError, TransportErrorSource, ) from nullrun.observability import metrics +from nullrun.observability.redaction import api_key_fingerprint +from nullrun.wal_paths import ( + WAL_PATH_ENV_VAR, + default_wal_path, + legacy_wal_path, +) if TYPE_CHECKING: # Forward-referenced to avoid transport.py ⇄ transport_websocket.py cycle. @@ -670,6 +676,21 @@ def _wait_on(cancel: threading.Event, seconds: float) -> bool: return not cancel.wait(seconds) +def _sleep(seconds: float) -> None: + """The clock seam for the non-cancel sleep path. + + A module-level function, like ``_wait_on``, for one reason: + ``time.sleep`` is ONE attribute on ONE module object, so a test that + patches it — anywhere — captures the sleep of EVERY thread in the + process. The webhook-delivery singleton idles on ``time.sleep(0.5)``, + and that value lands in a retry-delay list that is supposed to hold + the transport's own waits, turning a floor assertion into a + load-sensitive flake. This seam lets a test observe or stub the + transport's wait and only the transport's wait. + """ + time.sleep(seconds) + + def _interruptible_sleep(seconds: float, cancel: threading.Event | None) -> bool: """Sleep, but wake early when the SDK is shutting down. @@ -686,7 +707,7 @@ def _interruptible_sleep(seconds: float, cancel: threading.Event | None) -> bool misbehaving, which is when someone is most likely to be killing it. """ if cancel is None: - time.sleep(seconds) + _sleep(seconds) return True return _wait_on(cancel, seconds) @@ -935,6 +956,11 @@ class DecisionSource: """ GATEWAY = "gateway" + # NO PRODUCER. Nothing in ``src/`` ever returns this value: the + # SDK has no in-process decision cache (ADR-180 / NR-116). The + # member is kept so that code importing it keeps working, but it + # reads as a live decision cache and it is not one. A pin test + # asserts that no producer appears. CACHED = "cached" FALLBACK = "fallback" LOCAL = "local" @@ -1006,9 +1032,10 @@ class ExecuteConfig: timeout: float = 5.0 # Max retries for execute calls max_retries: int = 10 - # Cache TTL for CACHED mode (seconds) + # UNUSED. There is no decision cache in the SDK (NR-116): no + # producer reads these two fields. They are kept for import + # compatibility and a pin test asserts that nothing reads them. cache_ttl: float = 60.0 - # Cache max size cache_max_size: int = 10000 @@ -1032,6 +1059,7 @@ def __init__( secret_key: str | None = None, config: FlushConfig | None = None, redis_client: Any = None, + organization_id: str | None = None, ): self.api_url = api_url.rstrip("/") @@ -1060,6 +1088,27 @@ def __init__( self.api_key = api_key self.secret_key = secret_key # HMAC signing key + # Metadata only. The WAL path is scoped by the API KEY, not by + # this value — see `_resolve_wal_path` and `nullrun.wal_paths`. + self.organization_id = organization_id + # SDK-4 (security audit 2026-11-06). The WAL path is resolved + # ONCE, here, and NEVER changes for the life of this transport. + # + # It used to be re-derived on every call, and the default was a + # FIXED path in the shared temp directory: every user and every + # org on the host resolved to `/nullrun.wal`. Two + # runtimes then wrote to the same file and the atomic + # whole-file replace in `_persist_to_wal` destroyed the other + # runtime's undelivered events — money that had already been + # spent was never booked. + # + # Resolving it once is what makes the scope safe. A path that + # changed when the org id arrived (it arrives AFTER + # authentication, which happens after construction) would + # orphan every file written before that moment, and no test + # would catch it: the path is "correct" at every instant and + # the spend is lost anyway. + self._wal_base_path = self._resolve_wal_path() self.config = config or FlushConfig() # Allow env-var override of batch size and flush interval. if "NULLRUN_BATCH_SIZE" in os.environ: @@ -1182,8 +1231,13 @@ def __init__( ) self._stopped = False # Track if stop was called # 0.7.0 thin client: no local policy cache. Backend is authoritative. - _masked = api_key[:8] + "***" if api_key and len(api_key) >= 8 else "***" - logger.debug(f"Transport initialized: api_url={self.api_url}, api_key={_masked}") + # SDK-5: a fingerprint, not a prefix. The transport and the + # status string used to print different numbers of key + # characters; they now print the same fingerprint. + _fingerprint = api_key_fingerprint(api_key) or "unknown" + logger.debug( + f"Transport initialized: api_url={self.api_url}, api_key_fp={_fingerprint}" + ) # OpenTelemetry tracer (lazy-loaded: only if opentelemetry is installed) self._tracer = None @@ -1232,12 +1286,57 @@ def _wal_max_bytes(self) -> int: except ValueError: return self._WAL_MAX_BYTES_DEFAULT - def _wal_path(self) -> str: - """Resolve WAL path. Honours ``NULLRUN_WAL_PATH``; defaults to platform tempdir.""" - env_path = os.environ.get("NULLRUN_WAL_PATH") + def _resolve_wal_path(self) -> str: + """Resolve the WAL base path. Called ONCE, from the constructor. + + ``NULLRUN_WAL_PATH`` wins outright — it is the operator's + explicit choice. Otherwise the default is scoped per user AND + per api key, so two runtimes on one host do not share a file + (SDK-4, security audit 2026-11-06). + """ + env_path = os.environ.get(WAL_PATH_ENV_VAR) if env_path: return env_path - return os.path.join(tempfile.gettempdir(), "nullrun.wal") + return default_wal_path(api_key=self.api_key, org_id=self.organization_id) + + def _wal_path(self) -> str: + """The WAL base path of this transport. Fixed for its lifetime. + + Resolved once in the constructor on purpose: a path that changed + mid-life would orphan the files written under the previous one. + """ + return self._wal_base_path + + def _warn_about_legacy_wal(self) -> None: + """Report the unscoped WAL an older SDK left in the temp dir. + + The file is NOT read, NOT migrated and NOT deleted. It was + shared by every user and every org on the host, so its rows may + belong to another tenant — replaying them into this org's WAL + would book a stranger's spend against this account. Reporting + the path is the only safe thing to do with it; an operator with + the right credentials decides what it holds. + """ + if os.environ.get(WAL_PATH_ENV_VAR): + # The operator pinned the path. The legacy file is not ours. + return + legacy = legacy_wal_path() + try: + size = os.path.getsize(legacy) + except OSError: + return + if size == 0: + return + logger.warning( + "Legacy unscoped WAL found at %s (%d bytes). It is NOT read, NOT " + "migrated and NOT deleted: it was shared by every user and org on " + "this host, so its events may belong to another tenant. Inspect it " + "with `nullrun-wal --wal %s status` and replay it under the org " + "that owns it.", + legacy, + size, + legacy, + ) def _rotate_wal_if_needed(self) -> None: """Rotate ```` to ``.1`` if it exceeds the size cap.""" @@ -1283,6 +1382,23 @@ def _wal_lock_path(self) -> str: """Path of the advisory lock guarding mutations of the WAL files.""" return f"{self._wal_path()}.lock" + def _wal_v3_path(self) -> str: + """Path of the v3 single-shot consume WAL. + + SDK-1 (security audit 2026-11-06). A SEPARATE file from the + base WAL because the two are replayed through DIFFERENT + endpoints. + + The base WAL holds legacy events, and they are replayed through + ``/api/v1/track/batch``. A v3 consume payload must go to + ``/api/v1/track``: the batch path does not run the CONSUME + script, so a reservation replayed through it is never settled + and the money is never booked. Replaying a v3 payload through + the batch path is the same loss as dropping it, with more + machinery. + """ + return f"{self._wal_path()}.v3" + @property def _dlq_max_bytes(self) -> int: """Effective DLQ size cap. Same shape as ``_wal_max_bytes``.""" @@ -1824,6 +1940,186 @@ def _replay_from_wal(self) -> None: + (" (unflushed remainder re-persisted)" if repersisted else "") ) + # ------------------------------------------------------------------ + # SDK-1 (security audit 2026-11-06): the v3 consume WAL. + # + # The money path. A v3 `/track` carries a CONSUME: it settles the + # reservation the matching `/gate` made. If that call is refused or + # never reaches the backend, the tool call has ALREADY run and the + # money has ALREADY been spent — but nothing booked it. That gap is + # what this file closes. + # + # The invariant: after the call, the event is either BOOKED or + # RECOVERABLE. Never dropped. + # ------------------------------------------------------------------ + + def persist_v3_track(self, payload: dict[str, Any]) -> bool: + """Durably record a v3 /track payload that could not be delivered. + + The payload is written VERBATIM — ``reservation_id`` and + ``idempotency_key`` included — because the replay sends it back + exactly as it was built. + + The ``idempotency_key`` is not decoration. The backend DELETEs + the reservation key after the first successful consume, so a + replay that arrives WITHOUT the key is answered with 503 + ``RESERVATION_NOT_FOUND`` and the row could never leave the + file. With the key the backend replays its stored answer (200, + ``idempotent_replay: true``) and the row clears. + + APPENDED, never rewritten: the file holds every undelivered + payload, and a rewrite would drop the ones an earlier failure + left behind. + + Returns True when the payload is durably on disk. + """ + row = _v3_wal_row(payload) + path = self._wal_v3_path() + if not self._write_events_atomic(path, [row], mode="a"): + logger.error( + "Could not persist the v3 consume payload for reservation_id=%s " + "to %s. The spend is NOT recoverable — it is lost.", + payload.get("reservation_id"), + path, + ) + return False + logger.warning( + "v3 consume for reservation_id=%s was not delivered; the payload is " + "recoverable in %s and will be replayed to /api/v1/track.", + payload.get("reservation_id"), + path, + ) + return True + + def _read_wal_rows(self, path: str) -> list[dict[str, Any]]: + """Read the JSON lines of a WAL file under the cross-process lock. + + A line that does not parse is skipped: it was never durable (a + torn tail) and there is nothing to recover from it. + """ + with self._wal_file_lock() as acquired: + if not acquired: + logger.warning( + "WAL read skipped: another process holds the WAL lock." + ) + return [] + try: + with open(path) as f: + lines = f.readlines() + except FileNotFoundError: + return [] + except OSError as e: + logger.warning(f"Failed to read WAL {path}: {e}") + return [] + rows: list[dict[str, Any]] = [] + for line in lines: + try: + row = json.loads(line.strip()) + except json.JSONDecodeError: + continue + if isinstance(row, dict): + rows.append(row) + return rows + + def _replay_v3_from_wal(self) -> None: + """Replay the v3 consumes that could not be delivered. + + Each payload is re-sent to ``/api/v1/track`` (single) — NOT to + the batch endpoint. The batch path does not run the CONSUME + script, so a reservation replayed through it is never settled + and the money is never booked. + + A row is removed ONLY when the backend accepted it. A payload + that is refused again stays in the file: an undelivered spend + that nobody retries is a lost spend. The loop stops at the + first refusal so the spends keep their order. + """ + path = self._wal_v3_path() + rows = self._read_wal_rows(path) + if not rows: + return + delivered: set[str] = set() + for row in rows: + payload = row.get("payload") + if not isinstance(payload, dict): + # Not a row this SDK wrote. Leave it alone — it is + # evidence, and the SDK cannot know what it means. + logger.warning( + "WAL replay: %s holds a row this SDK cannot read; leaving it " + "in place.", + path, + ) + continue + try: + self.track_single(payload) + except (NullRunBackendError, NullRunDecision) as exc: + # The backend ANSWERED and refused this row. The refusal + # is about this consume, not about the backend, so the + # next row still gets its chance — one stuck row must + # not block every spend behind it. + # + # A `NullRunDecision` (a budget verdict, e.g. + # CONSUME_OVERBUDGET) is a refusal too, and it used to + # fall through to the arm below, which STOPS the replay + # — so one overbudget row blocked every spend behind it + # from being booked. The rows are independent + # reservations; one stuck row must not block the rest. + logger.error( + "WAL replay: /track still refuses the consume for " + "reservation_id=%s (%s). The payload stays in %s and is " + "retried on the next start.", + payload.get("reservation_id"), + exc, + path, + ) + continue + except Exception as exc: # noqa: BLE001 — every failure is a keep + # The backend did not answer (unreachable, timeout). Every + # remaining row would fail the same way, so the replay + # stops here and the file keeps every payload. + logger.error( + "WAL replay: /track is unreachable (%s). The payloads stay " + "in %s and are retried on the next start.", + exc, + path, + ) + break + delivered.add(_v3_row_key(row)) + metrics.inc_transport("v3_track_replayed") + if delivered: + self._remove_v3_rows(delivered, path) + logger.info( + "Replayed %d of %d v3 consume payload(s) from WAL", + len(delivered), + len(rows), + ) + + def _remove_v3_rows(self, delivered: set[str], path: str) -> None: + """Drop the rows the backend accepted. Everything else stays. + + The file is re-read before the rewrite so a row another process + appended in the meantime survives. + """ + rows = self._read_wal_rows(path) + kept = [row for row in rows if _v3_row_key(row) not in delivered] + if len(kept) == len(rows): + return + if not kept: + try: + os.remove(path) + except FileNotFoundError: + return + except OSError as e: + logger.warning(f"Failed to remove emptied v3 WAL {path}: {e}") + return + if not self._write_events_atomic(path, kept, mode="w"): + logger.warning( + "WAL replay: could not rewrite %s. The accepted payloads stay " + "in the file; the next replay re-sends them and the backend " + "answers them from its idempotency record.", + path, + ) + def track(self, event: dict[str, Any]) -> None: """ Add event to buffer. Non-blocking. @@ -1849,8 +2145,14 @@ def start(self) -> None: """Start background flush thread.""" if self._running: return + # SDK-4: report the unscoped WAL an older SDK left behind. It is + # not read and not migrated — see `_warn_about_legacy_wal`. + self._warn_about_legacy_wal() # Replay any events from WAL that were persisted due to previous crash self._replay_from_wal() + # SDK-1: replay the v3 consumes that could not be delivered. They + # go to /api/v1/track (single), not to the batch endpoint. + self._replay_v3_from_wal() # Held refusals are recovered separately and are NOT replayed onto the # send path. `_replay_from_wal` recovers undelivered events, which is # the right thing for them; these were already refused, so re-sending @@ -3161,6 +3463,22 @@ def _post_batch() -> httpx.Response: reason = action.get("reason", "") if action_type: handle_action(action_type, workflow_id, reason) + except NullRunUnknownActionError as item_err: + # The control plane sent an action type this SDK + # cannot execute (SDK-12). That is a control-plane + # INTEGRITY event, not a malformed payload: it is + # logged at ERROR so it is not filed under "skipped + # malformed action", and the loop continues so one + # unknown action cannot abort the rest of the batch. + logger.error( + "Control plane sent an action type this SDK cannot " + "execute: %r (workflow=%s). The action was NOT " + "executed — this is a server/SDK version mismatch. " + "(%s)", + action.get("type"), + action.get("workflow_id"), + item_err, + ) except Exception as item_err: logger.warning("Skipping malformed action %r: %s", action, item_err) for msg in (data or {}).get("messages", []) or []: @@ -3278,14 +3596,31 @@ def execute( # (defeating the mode) or ask the box a question it has no # endpoint for. Re-asking would also double-charge: the charge # and the decision are one event, done once. + # + # So the answer comes from the box's own call — the pre-flight + # (`Transport.check` → `edge.enforce`) recorded it under the + # event id, and it is read back here. What this branch must + # NOT do is DECIDE. It used to return a client-side + # `{"decision": "allow"}` without the box ever being asked + # (SDK-2): the whole edge decision surface was client-side + # only, so an expired, revoked or exhausted lease still + # allowed the call, and the caller could not tell the box's + # answer from the SDK's own. + # + # With no recorded answer the box was never asked about THIS + # call, and the answer is a refusal. The cloud has the same + # rule and the same reason: a direct `/execute` with no prior + # `/gate` is answered 404 EXECUTION_NOT_FOUND, because the + # backend's `execution:{id}` binding does not exist. if self.edge is not None: - from nullrun.edge import EDGE_LEASE - return { - "decision": "allow", - "decision_source": self.edge.last_source or EDGE_LEASE, - "execution_id": execution_id, - "via_edge": True, - } + from nullrun.context import get_operation_id + + answer = self.edge.take_answer(get_operation_id()) + if answer is None: + return self.edge.not_enforced(execution_id) + answer["via_edge"] = True + answer.setdefault("execution_id", execution_id) + return answer gate_request = { "organization_id": organization_id, @@ -4984,6 +5319,30 @@ def _safe_json(response: httpx.Response, endpoint: str) -> Any: ) from exc +def _v3_wal_row(payload: dict[str, Any]) -> dict[str, Any]: + """Wrap a v3 consume payload into a WAL row. + + The wrapper marks the row's KIND. The base WAL and the v3 WAL are + separate files with separate replay routes, so the kind is + belt-and-braces — but it costs nothing and it means a row can + never be mistaken for a legacy event. + """ + return {"_nullrun_wal": "v3_track", "_v": 1, "payload": payload} + + +def _v3_row_key(row: dict[str, Any]) -> str: + """The identity of a v3 WAL row: the reservation it settles. + + Two rows for the same reservation are the same consume. The + idempotency key is part of the identity because two attempts at the + same consume can carry different keys. + """ + payload = row.get("payload") + if not isinstance(payload, dict): + return "" + return f"{payload.get('reservation_id')}|{payload.get('idempotency_key')}" + + def _parse_v3_error_envelope( response: httpx.Response, endpoint: str, @@ -5002,6 +5361,26 @@ def _parse_v3_error_envelope( body = response.json() except Exception: body = None + # The backend's OWN error code, as the wire spelled it. + # + # `_extract_error_envelope` is the single normaliser for the three + # envelope shapes the backend emits, and the uncategorised parser + # dispatched on its result — so this is the same string the class + # was chosen from. It is exposed on the exception because the + # decision "is this refusal fail-closed?" is made BY CODE, not by + # exception class: the class hierarchy puts a backend refusal + # (`NullRunBackendError`) under `NullRunInfrastructureError`, which + # is a SIBLING of `NullRunDecision`, so no `isinstance` check can + # tell them apart. + backend_code, _message, _details = _extract_error_envelope( + body if isinstance(body, dict) else {}, response.text + ) + exc.backend_error_code = backend_code or None # type: ignore[attr-defined] + # The HTTP status of the ANSWER. This attribute is the marker that + # separates "the backend refused" from "the backend never answered": + # the network path raises `NullRunTransportError` without it. + # ADR-008's fail-OPEN covers the second case only. + exc.wire_status_code = response.status_code # type: ignore[attr-defined] if isinstance(body, dict): # ADR-062 §2.2. A refusal whose category is ABSENT is left # absent, not defaulted: the runtime's strict rule reads the @@ -5011,10 +5390,17 @@ def _parse_v3_error_envelope( category = body.get("category") if category is not None: exc.wire_category = category # type: ignore[attr-defined] + # The TOP LEVEL first. The `/track` envelope carries the code + # there and NOT inside `details` + # (``handlers.rs::track_handler``), so the old details-first + # order reported `None` for every /track refusal — the one + # place the SDK most needs to read the code. The gate's own + # envelope carries it in BOTH places, so the order does not + # change what that path reports. + details_block = body.get("details") + details_block = details_block if isinstance(details_block, dict) else {} exc.wire_error_code = ( # type: ignore[attr-defined] - (body.get("details") or {}).get("error_code") - if isinstance(body.get("details"), dict) - else body.get("error_code") + body.get("error_code") or details_block.get("error_code") ) # Server-authored text, carried the same way ``check()`` # carries it on the dict it returns. ADR-063 §1.3(e): @@ -5049,12 +5435,11 @@ def _parse_v3_error_envelope_uncategorised( Mapping table lives at ``_V3_ERROR_CODE_MAP`` below — keep the helper as a thin dispatcher. - DEF-NR-TOOLBLOCKED-PARSER: the ``catalog is - NullRunToolBlockedError or catalog is NullRunBlockedException`` - arm further down is the dedicated branch for the typed v3 - envelope. It used to be documented by a comment inside - ``Transport.check``, which never had such a branch — the tag was - filed against the wrong function for as long as it existed. + DEF-NR-TOOLBLOCKED-PARSER: the subclass test on + NullRunBlockedException further down is the dedicated branch for + the typed v3 envelope. It used to be documented by a comment + inside ``Transport.check``, which never had such a branch — the tag + was filed against the wrong function for as long as it existed. Re-filed here, on the function that actually holds the code. """ # Lazy imports: the exception classes import the transport @@ -5071,7 +5456,6 @@ def _parse_v3_error_envelope_uncategorised( NullRunConsumeOverbudgetError, NullRunProtocolError, NullRunRateLimitRedisError, - NullRunToolBlockedError, NullRunWorkflowInactiveError, RateLimitError, ) @@ -5138,6 +5522,7 @@ def _parse_v3_error_envelope_uncategorised( backend_code == "CHAIN_MAX_DURATION_EXCEEDED" or backend_code == "CHAIN_CROSS_ORG" or backend_code == "CHAIN_ORG_MISMATCH" + or backend_code == "CHAIN_KEY_MISMATCH" ): return NullRunChainError( full_message, @@ -5173,6 +5558,7 @@ def _parse_v3_error_envelope_uncategorised( "APPROVAL_EXPIRED", "APPROVAL_DIGEST_MISMATCH", "APPROVAL_TOOL_DIGEST_MISMATCH", + "APPROVAL_TOOL_MISMATCH", "APPROVAL_REPLAY_REJECTED", ): # dispatch so callers can branch on the precise grant-consume @@ -5190,11 +5576,16 @@ def _parse_v3_error_envelope_uncategorised( # NullRunApprovalNotYetApprovedError:`` for wait/poll, # ``except NullRunApprovalReplayRejectedError:`` for # retry-loop detection, etc. + # + # DEC-5 / DEC-7 added the seventh: ``APPROVAL_TOOL_MISMATCH`` + # (NR-A017, HTTP 409 — a conflict, not a permission denial). + # The ``status_code`` passed through is the one the backend + # actually sent, so the 409 reaches the caller verbatim. catalog = _V3_ERROR_CODE_MAP[backend_code] return catalog( # type: ignore[call-arg] workflow_id=str(details.get("workflow_id") or "unknown"), reason=full_message, - status_code=status, # 403 per backend mapping + status_code=status, # 403 per backend mapping (409 for APPROVAL_TOOL_MISMATCH) approval_id=details.get("approval_id"), ) @@ -5263,6 +5654,44 @@ def _parse_v3_error_envelope_uncategorised( return catalog(full_message) if catalog is NullRunProtocolError: return catalog(full_message) + if catalog is RateLimitError: + # RateLimitError has NO message-only form: it requires + # ``source`` + ``endpoint``. The generic + # ``catalog(full_message, **forwarded)`` fallback below + # would raise TypeError, and the TypeError escapes the + # parser into the catch-all in Transport.execute — the + # refusal would be swallowed into a synthetic + # ``{"decision": "block"}`` dict and the caller would see + # a block where the backend said "you are rate limited". + # + # The 429 codes that reach this arm are RATE_LIMIT_PLAN_TIER + # (the PLAN's aggregate per-org cap — ADR-029 §13) and + # WAITLIST_RATE_LIMITED. ``upgrade_url`` is the plan-change + # link the backend sends for the plan tier; the caller + # surfaces it instead of guessing the pricing page. + retry_after = retry_after_ms / 1000.0 if retry_after_ms else None + return RateLimitError( + full_message, + source=TransportErrorSource.GATEWAY_ERROR, + endpoint=endpoint, + retry_after=retry_after, + upgrade_url=body.get("upgrade_url") if isinstance(body, dict) else None, + body=body, + ) + if catalog is NullRunChainError: + # The chain family carries its own attributes (chain_id, + # backend_code). The generic fallback drops them: the base + # constructor only accepts error_code / user_action / + # retryable / docs_url / cause, and everything else in the + # wire ``details`` is filtered out before the call. + # ``backend_code`` keeps the wire string, which is what + # tells CHAIN_NOT_FOUND (404) from CHAIN_ENDED (409). + return NullRunChainError( + full_message, + chain_id=details.get("chain_id"), + backend_code=backend_code, + status_code=status, + ) # NullRunAuthError — surface the wire error_code (one of # v3.38's API_KEY_REVOKED / API_KEY_EXPIRED / API_KEY_DISABLED # / API_KEY_INVALID / API_KEY_MISSING / API_KEY_MALFORMED) on @@ -5304,27 +5733,38 @@ def _parse_v3_error_envelope_uncategorised( # as BaseException (the helper declares -> Exception). allowed = {"error_code", "user_action", "retryable", "docs_url", "cause"} forwarded = {k: v for k, v in details.items() if k in allowed} - if ( - catalog is NullRunToolBlockedError - or catalog is NullRunBlockedException - ): - # subclasses require positional ``workflow_id`` + ``reason`` - # (no defaults), so the generic ``catalog(full_message, ...)`` - # fallback below raises TypeError when given a string for - # ``workflow_id``. Affects 7 catalog entries: TOOL_BLOCKED, - # LOOP_DETECTED, MODEL_REQUIRED, POLICY_UNCONFIGURED, - # TOO_MANY_PENDING_APPROVALS, BUSINESS_IMPACT_INVALID, - # VALIDATION_FAILED. Pre-fix the TypeError escaped the parser - # and got swallowed by the catch-all ``except Exception: pass`` - # in Transport.execute, surfacing the synthetic-block dict + if issubclass(catalog, NullRunBlockedException): + # Every NullRunBlockedException subclass requires + # ``workflow_id`` + ``reason`` (no defaults), so the generic + # ``catalog(full_message, ...)`` fallback below raises + # TypeError when given a string for ``workflow_id``. + # + # The test is ``issubclass``, not identity, and that is the + # whole point of this arm. An identity test + # (``catalog is NullRunBlockedException``) matched only the + # two exact classes — NullRunToolBlockedError and the base — + # so EVERY typed subclass of the blocked family fell through + # to the generic fallback and raised TypeError: + # NullRunApprovalDbUnavailableError (8 catalog entries), + # NullRunApprovalReplayRejectedError (via + # APPROVAL_ALREADY_DECIDED), and the three NullRunMcp*Error + # classes. The TypeError escaped the parser and got swallowed + # by the catch-all ``except Exception: pass`` in + # Transport.execute, surfacing the synthetic-block dict # ``{"decision": "block", "explanation": "Gateway returned - # 403"}`` instead of the typed NR-T001 / NR-Lxxx catalog line. - # ``tool_name`` is forwarded for NullRunToolBlockedError - # (the only BlockedException subclass that surfaces it on the - # wire envelope); the parent constructor drops it for plain - # NullRunBlockedException so it's a no-op there. ``forwarded`` - # (error_code / user_action / retryable / docs_url / cause) is - # passed through so the catalog value's defaults win. + # 403"}`` instead of the typed NR-Axxx / NR-MCPxx catalog + # line. The caller could not branch on the refusal at all. + # + # ``tool_name`` is forwarded for the subclasses that surface + # it on the wire envelope; the parent constructor drops it + # for plain NullRunBlockedException so it's a no-op there. + # ``forwarded`` (error_code / user_action / retryable / + # docs_url / cause) is passed through so the catalog value's + # defaults win. + # + # Order matters: NullRunBudgetError is a subclass of + # NullRunBlockedException and has its OWN arm above, so the + # budget family keeps its dedicated constructor. instance = catalog( # type: ignore[call-arg] workflow_id=str(details.get("workflow_id") or "unknown"), reason=full_message, @@ -5384,6 +5824,7 @@ def _build_v3_error_code_map() -> dict[str, type[Exception]]: NullRunApprovalNotYetApprovedError, NullRunApprovalReplayRejectedError, NullRunApprovalToolDigestMismatchError, + NullRunApprovalToolMismatchError, NullRunAuthError, NullRunBackendError, NullRunBlockedException, @@ -5436,6 +5877,13 @@ def _build_v3_error_code_map() -> dict[str, type[Exception]]: # 403 — chain security + workflow state "CHAIN_CROSS_ORG": NullRunChainError, "CHAIN_ORG_MISMATCH": NullRunChainError, + # ADR-180 (2026-11-07): a chain belongs to the API key that + # created it. The backend emits this code when another key of + # the same organization presents a chain it does not own. It + # is a chain refusal, so it carries the chain class — without + # this entry the dispatcher falls through to the base class and + # a caller branching on NullRunChainError cannot catch it. + "CHAIN_KEY_MISMATCH": NullRunChainError, "PARENT_EXECUTION_NOT_FOUND": NullRunChainError, "PARENT_EXECUTION_ORG_MISMATCH": NullRunChainError, "PARENT_EXECUTION_KEY_MISMATCH": NullRunChainError, @@ -5489,6 +5937,12 @@ def _build_v3_error_code_map() -> dict[str, type[Exception]]: "APPROVAL_EXPIRED": NullRunApprovalExpiredError, "APPROVAL_DIGEST_MISMATCH": NullRunApprovalDigestMismatchError, "APPROVAL_TOOL_DIGEST_MISMATCH": NullRunApprovalToolDigestMismatchError, + # DEC-5 / DEC-7 (audit 2026-11-02): the approval binds a TOOL + # SET, and the executing tool must be a member. The operator + # approved an action for one tool and the agent is running a + # different one — a money approval used to bind only the + # AMOUNT, so an approval for tool A authorised tool B. + "APPROVAL_TOOL_MISMATCH": NullRunApprovalToolMismatchError, "APPROVAL_REPLAY_REJECTED": NullRunApprovalReplayRejectedError, # Distinct from BUDGET_HARD_BLOCKED: the operator explicitly # approved the grant at /gate, but the period-bound counter @@ -5716,18 +6170,270 @@ def _build_v3_error_code_map() -> dict[str, type[Exception]]: # catch-all hiding drift: previously every one of the eleven # codes above landed here untyped. "INTERNAL_ERROR": NullRunBackendError, + # ----------------------------------------------------------------- + # 2026-11-07 (SDK-15) — thirty-four codes the backend registers and + # this catalog never learned about. + # + # The parity guard in + # ``backend/tests/nr007_sdk_error_code_parity.rs`` could not see + # them: it was not wired into any workflow, so the drift grew from + # the twelve the audit named to thirty-four by the time the guard + # was finally run against a live backend enum. Every one of them + # fell through to the status-based fallback, so the refusal + # arrived as a generic ``NullRunBackendError`` (or, on a 4xx, as + # an authentication error) with no diagnostic class at all. + # + # The mapping is by WIRE BUCKET, and the bucket comes from the + # backend's own registry — ``GateErrorCode::http_status`` — so the + # SDK's class and the backend's status cannot disagree: + # + # - 402/429 money → NullRunBudgetError ("stop spending") + # - 403 refusal → the refusal's own class + # - 429 rate limit → RateLimitError (retry_after and + # upgrade_url ride along) + # - 4xx client contract → NullRunProtocolError / the typed + # binding class + # - 5xx "the store did + # not answer" → NullRunBackendError (retryable), or + # the subsystem's own typed infra class + # ----------------------------------------------------------------- + # 402 — the anti-DoS reserve scan could not enumerate the org's + # reservations, so the reservation was refused. The wire puts it + # in the money bucket: the call did not get its money. It is NOT + # an outage, so it belongs to the budget family and not to the + # infra family. + "RESERVED_SCANNED_PARTIAL": NullRunBudgetError, + # 429 — the period budget was exhausted between reserve and + # consume. The caller must stop and wait for the period, exactly + # like every other exhausted-boundary code — a money outcome, so a + # budget error and not a rate limit. + "BUDGET_EXCEEDED": NullRunBudgetError, + # 429 — the PLAN's aggregate per-org cap refused the call + # (ADR-029 §13). Same 429 bucket as RATE_LIMIT_EXCEEDED and the + # same client behaviour (back off and retry), with a distinct wire + # string so the OPERATOR can tell the two caps apart. The + # caller's own remedy is a plan change, which is what the + # exception's ``upgrade_url`` is for. + "RATE_LIMIT_PLAN_TIER": RateLimitError, + # 429 — the address waitlist refused the call. Same 429 bucket. + "WAITLIST_RATE_LIMITED": RateLimitError, + # 503 — the rate limiter's store could not be reached, so the gate + # could not decide. Fail-CLOSED, and the rate-limit subsystem's + # own typed infra class — NOT the generic backend error, so a + # caller can tell "the limiter is down" from "the backend is + # down". + "RATE_LIMITER_UNAVAILABLE": NullRunRateLimitRedisError, + # 503 — the AUTH store could not be reached. The registry says + # 503, not 401: the store did not answer, so this is an outage + # and not a rejection. NullRunAuthError means "the key was + # REJECTED", which is a different fact with a different remedy, + # so the generic infra class is the honest one here. + "AUTH_BACKEND_UNAVAILABLE": NullRunBackendError, + # 503 — the org context could not be resolved (the key's org + # binding could not be read). Same shape as the code above: the + # store did not answer. + "ORG_CONTEXT_UNRESOLVED": NullRunBackendError, + # 403 — the key authenticates but lacks the scope this endpoint + # needs (ADR-143 / P2-08). The auth family, and the granular wire + # code is preserved on ``exc.wire_code``, so a caller can tell + # "wrong scope" from "bad key". + "INSUFFICIENT_SCOPE": NullRunAuthError, + # 404 — the chain_id names no live chain. The same "no binding + # exists" class as EXECUTION_NOT_FOUND: the client mints a new id + # and carries on. + "CHAIN_NOT_FOUND": NullRunChainError, + # 409 — the chain ENDED (duration cap, or an explicit end). A new + # chain_id is the remedy. Same class as the chain family: the + # lineage context is what is wrong. + "CHAIN_ENDED": NullRunChainError, + # 422 — the execution has no binding. Only the server can mint an + # execution_id, so the remedy is "re-issue /gate" — the same + # remedy NullRunExecutionNotFoundError already names. The class + # carries the status, so the 422 reaches the caller verbatim. + "EXECUTION_NOT_BOUND": NullRunExecutionNotFoundError, + # 500 — the binding VANISHED (the state is gone). A server-side + # fault: the class that says "the backend could not produce a + # determinate answer", and it is retryable. + "EXECUTION_BINDING_MISSING": NullRunBackendError, + # 503 — Redis could not be READ for the binding. + "EXECUTION_BINDING_READ_FAILED": NullRunBackendError, + # 503 — Redis could not be WRITTEN for the binding. + "EXECUTION_BINDING_WRITE_FAILED": NullRunBackendError, + # 503 — Redis errored while resolving the binding. + "EXECUTION_RESOLVE_REDIS_ERROR": NullRunBackendError, + # 409 — a prior request with this idempotency_key is still being + # processed. The remedy is "retry after the in-flight call + # settles" — a transient race, so the retryable infra class. The + # wire's ``retry_after_ms`` rides along in the details. + "IDEMPOTENCY_IN_FLIGHT": NullRunBackendError, + # 409 — the same key replayed with a different payload. A + # client-side contract violation, the same class as + # IDEMPOTENCY_KEY_MISMATCH: mint a fresh key. + "IDEMPOTENCY_CONFLICT": NullRunProtocolError, + # 503 — the ingestion queue is applying backpressure. + "INGESTION_BACKPRESSURE": NullRunBackendError, + # 503 — the ingestion queue is closed. + "INGESTION_CLOSED": NullRunBackendError, + # 413 — the batch is too large. The transport halves and resends + # on this status before the failure ever reaches the exception + # path; the entry exists so the catalog is complete. + "BATCH_TOO_LARGE": NullRunBackendError, + # 422 — a /track field carries control characters. A payload-shape + # defect on the client's side, the same family as + # WORKFLOW_ID_BODY_MISMATCH. + "CONTROL_CHARS_IN_FIELD": NullRunBackendError, + # 503 — the /track reservation envelope is gone. The /track path + # surfaces this per event as a deterministic refusal; the entry + # keeps the catalog complete. + "RESERVATION_NOT_FOUND": NullRunBackendError, + # 403 — the executed tool arguments DRIFTED from the + # operator-approved ToolCall.params. The same family as the + # business-impact digest mismatch: the operator approved a + # different action than the one now bound, and the remedy is a + # fresh approval. + "APPROVAL_TOOL_ARGUMENT_MISMATCH": NullRunApprovalDigestMismatchError, + # 403 — the server could not obtain the executed payload to + # compare against the approval, so the approval boundary could + # not be verified at all. The closest typed home is the + # approval-infra class: the refusal stays in the approval family + # instead of collapsing into the generic infra class. + "APPROVAL_TOOL_ARGUMENT_UNAVAILABLE": NullRunApprovalDbUnavailableError, + # 403 — the org's MCP registry says this tool does not run. The + # generic tool-block class: "this tool is blocked" is exactly what + # the caller has to act on. + "MCP_TOOL_BLOCKED": NullRunToolBlockedError, + # 403 — the registry's decision is "ask", so the tool needs an + # operator approval that is not there yet. Retryable: the operator + # can still decide. + "MCP_TOOL_APPROVAL_REQUIRED": NullRunMcpApprovalRequiredError, + # 403 — nobody has decided about this tool yet. The same remedy + # as the code above (wait for a decision), so the same class; the + # wire string is what tells the two apart for an operator. + "MCP_TOOL_DECISION_REQUIRED": NullRunMcpApprovalRequiredError, + # 403 — the tool's definition moved under the approval. The same + # family as APPROVAL_TOOL_DIGEST_MISMATCH: the operator approved a + # different capability surface. + "MCP_DEFINITION_DRIFTED": NullRunApprovalToolDigestMismatchError, + # 403 — the tool is gone from the registry. The tool does not run: + # the generic tool-block class. + "MCP_TOOL_REMOVED": NullRunToolBlockedError, + # 503 — the per-org MCP registry could not be read, so the gate + # could not determine whether the operator allows this tool. + # Fail-CLOSED and retryable: ask again. + "MCP_REGISTRY_UNAVAILABLE": NullRunBackendError, + # 503 — the call could not be RESERVED, so the budget was never + # enforced for it. Deliberately NOT a budget error: 402 means + # "your budget is exhausted", this says "this server could not + # check the budget", and the remedy (fix the config) and the SDK + # behaviour (retry, do not stop) are both different. + "RESERVE_DISABLED": NullRunBackendError, + # 503 — the budget projection (the estimate) could not be read. + # The same shape as RESERVE_DISABLED. + "BUDGET_PROJECTION_UNAVAILABLE": NullRunBackendError, + # 500 — the workflow's budget configuration is invalid. A + # server-side defect: the call cannot be priced. + "WORKFLOW_BUDGET_CONFIG_INVALID": NullRunBackendError, + # 503 — the ToolBlock backstop's policy cache could not be read, + # so the gate could not determine whether to block. Fail-CLOSED + # and retryable. + "TOOL_BLOCKED_BACKSTOP_UNAVAILABLE": NullRunBackendError, } _V3_ERROR_CODE_MAP: dict[str, type[Exception]] = _build_v3_error_code_map() +# SDK-1 (security audit 2026-11-06). The refusal rule for the /track +# path. +# +# WHERE THE VOCABULARY COMES FROM. The backend owns the error +# vocabulary: every refusal it can emit is a variant of `GateErrorCode` +# (`backend/src/proxy/http/gate/error_codes.rs`), and that registry +# classifies each one as Denied / Budget / Halt / Infra. The SDK's own +# catalog (`_V3_ERROR_CODE_MAP`) is the projection of that registry, and +# the backend keeps the two in sync with a parity test that reads +# `GateErrorCode::all()` — `backend/tests/nr007_sdk_error_code_parity.rs` +# (green). So the codes in the catalog are not a hand-kept list in the +# SDK: they are the classes the registry maps its `Infra` family onto. +# +# The catalog still decides the CLASS a refusal raises as — a +# `NullRunBackendError` for RESERVATION_NOT_FOUND, a +# `NullRunRateLimitRedisError` for a limiter Redis that did not answer, +# a `NullRunAuthenticationError` for a request refused before it could +# be judged, a `NullRunDecision` for a budget or approval verdict. The +# catalog does NOT decide whether the SDK may report the consume as +# booked. That question is answered by the wire, one function below. +# +# A refusal means the CONSUME DID NOT LAND. The spend is real and +# unbooked, so the SDK must not report it as a successful track. +# +# NOT a refusal: `NullRunDecision` is a verdict — but it is one, and it +# raises in its own branch. NOT a refusal: a request that never reached +# the backend. That is a transport failure, and the ADR-008 fail-open +# row for the network case applies. +# +# --------------------------------------------------------------------------- +# SDK-1 (security audit 2026-11-06) — the class list above was WRONG. +# +# It named three classes and left nine backend codes OUTSIDE them, so +# those nine were dropped exactly like a network error: the +# `PROTOCOL_*` family, `IDEMPOTENCY_CONFLICT`, `IDEMPOTENCY_KEY_MISMATCH` +# and the three `RATE_LIMIT_*` codes. The backend rate-limits `/track`, +# so a 429 there is LIVE, and a 429 was a dropped spend. +# +# The list was also the wrong SHAPE. A refusal is not a class — it is +# the fact that the backend ANSWERED. The taxonomy of the answer (a +# budget verdict, a rate limit, a protocol error) does not change what +# the SDK owes the caller: the consume did not land, so the spend is +# unbooked, and reporting that as a successful track is a lost spend +# with a success label on it. +# +# So the rule is now ONE question — "did the backend answer?" — asked of +# the WIRE, not of the exception class. ADR-008's fail-OPEN covers +# exactly one thing: the gate could not be REACHED. There is then no +# answer to honour and a dead backend must not freeze the agent loop. +# Every other case is a refusal. +# --------------------------------------------------------------------------- + + +def is_backend_refusal(exc: BaseException) -> bool: + """True when the backend ANSWERED the consume and refused it. + + ADR-008's fail-OPEN covers exactly ONE thing: the gate could not be + REACHED. There is then no answer to honour, and a dead backend must + not freeze the agent loop. Every other case is a refusal, and a + refusal the SDK reports as a successful track is a lost spend with a + success label on it. + + The marker is the WIRE, never the exception class. The class + hierarchy puts a backend refusal (`NullRunBackendError`) under + `NullRunInfrastructureError`, a SIBLING of `NullRunDecision`, so no + `isinstance` check can tell a 503 ``RESERVATION_NOT_FOUND`` from a + budget verdict. The wire status is stamped on the exception by + `_parse_v3_error_envelope`; the network path raises + `NullRunTransportError` without it. + + A named error code counts as an answer even when no status was + stamped: the backend named a refusal, so there IS an answer. + + The function answers ONE question. `NullRunDecision` is a refusal + too, so the caller must test that class FIRST. + """ + status = getattr(exc, "wire_status_code", None) + if isinstance(status, int): + return True + code = getattr(exc, "backend_error_code", None) or getattr(exc, "wire_error_code", None) + return bool(code) and isinstance(code, str) + + def _parse_error_envelope( response: httpx.Response, endpoint: str, ) -> Exception: """Translate a non-2xx ``httpx.Response`` into the right exception - subclass per the canonical ``contracts/errors.ts`` envelope. + subclass per the control-plane ``ApiErrorResponse`` envelope + (``backend/src/proxy/http/errors.rs`` — the ``ErrorSlug`` values + the backend actually emits). 4xx/5xx/429 are mapped to distinct ``RateLimitError`` / ``NullRunAuthenticationError`` / ``NullRunTransportError(GATEWAY_ERROR)`` diff --git a/src/nullrun/transport_websocket.py b/src/nullrun/transport_websocket.py index ef60ac1..5b8aaf5 100644 --- a/src/nullrun/transport_websocket.py +++ b/src/nullrun/transport_websocket.py @@ -296,11 +296,63 @@ async def _handle_message(self, message: str) -> None: """ try: data = json.loads(message) - msg_type = data.get("type", "") + if not isinstance(data, dict): + # A control-plane frame is a JSON object by contract. A + # non-object cannot carry a ``type`` and is not a + # message we know — drop it instead of crashing the + # receive loop on ``.get``. + logger.warning( + "WS control frame is not a JSON object " + f"({type(data).__name__}) — dropping" + ) + return # Check for HMAC signature and verify if present signature = data.get("signature") timestamp = data.get("timestamp") + signed_payload_hex = data.get("signed_payload") + + # This client CAN verify a signature: it holds the API key + # and the HMAC secret. The server signs EVERY control-plane + # frame it sends for such a key + # (``ws_control.rs::send_signed_or_raw``), so a frame that + # arrives unsigned — or with a signature but no timestamp — + # did not come from the server. It is refused here, before + # any branch reads a field out of it. + # + # A client WITHOUT a secret (a legacy key) cannot verify + # anything, so the unsigned path stays open for it: the kill + # switch must keep working even without HMAC, and the + # privileged branch refuses unauthenticated frames on its + # own. + if self.api_key and self.secret_key and not (signature and timestamp): + # Local import to avoid a module-level cycle: + # observability imports nothing from us, so this + # is safe and lazy. + from nullrun.observability import metrics + + claimed_type = data.get("type", "") + metrics.inc_transport("ws_unverified_frame_total") + if claimed_type == "approval_resolved": + # Keep the approval-specific signal. The frame is + # refused either way, but the operator has to be + # able to see that somebody pushed an approval this + # client cannot verify. + metrics.inc_transport("ws_unverified_approval_total") + # The server's own refusal frame is unsigned BY DESIGN: + # it is sent before the connection is authenticated, so + # there is no secret to sign it with yet. Naming its code + # keeps the operator's reason visible even though the + # frame itself is refused. + claimed_code = data.get("code", "") + logger.error( + "Refusing an unauthenticated WS frame — this client can " + "verify signatures, so an unsigned frame did not come " + f"from the server (type={claimed_type!r}, " + f"code={claimed_code!r})" + ) + return + if signature and timestamp and self.api_key and self.secret_key: # This is a signed message - verify the signature msg_timestamp = int(timestamp) if isinstance(timestamp, (int, str)) else 0 @@ -319,7 +371,6 @@ async def _handle_message(self, message: str) -> None: # # See memory/ws-signed-message-byte-mismatch for the # original failure this design rule encodes. - signed_payload_hex = data.get("signed_payload") if isinstance(signed_payload_hex, str) and signed_payload_hex: try: verify_payload = bytes.fromhex(signed_payload_hex) @@ -391,7 +442,7 @@ async def _handle_message(self, message: str) -> None: # or (b) something is forging traffic. Both are # actionable and the operator needs to know. logger.error( - f"Invalid HMAC signature for {msg_type} message - " + f"Invalid HMAC signature for {data.get('type', '')} message - " "rejecting. This usually means the secret_key is out " "of sync with the server (check for a key_rotated " "event you may have missed) or the control plane is " @@ -405,38 +456,73 @@ async def _handle_message(self, message: str) -> None: metrics.inc_transport("hmac_verify_failures_total") return - # FIX-C (counterpart of backend fix(ws-control) in - # NULLRUN): when the message is signed and carries a - # `signed_payload` field, dispatching from the outer - # body fields would let an attacker splice forged values - # into the outer body while reusing a captured - # (signed_payload, signature) pair. The signature is - # computed over the bytes inside signed_payload, not the - # outer body, so the *only* trusted source is signed_payload - # itself. We parse it once and use the parsed dict for all - # state-dispatch decisions. + # FIX-C / SDK-5: ONE source of truth. + # + # The HMAC covers the bytes inside ``signed_payload`` — NOT + # the flattened outer body. The outer body therefore carries + # a WRITABLE copy of every payload field, and the signature + # check still PASSES (it verifies bytes the attacker never + # touched). So the branch type and every field must be read + # from the VERIFIED payload, never from the outer body. + # + # ``authenticated`` says: this frame's contents were proven + # to come from the server. It is True only when the signature + # verified. It gates the privileged branch (the approval + # push, which releases a parked gate). # - # For non-signed messages (legacy servers, or policy - # events that don't need per-payload signing) we fall back - # to the outer body — there is no signing, no attacker - # model. + # For frames that could not be authenticated at all (an + # unsigned legacy frame) we fall back to the outer body — + # there is no signing to trust, and the branches that need + # authentication refuse such a frame explicitly. trusted: dict[str, Any] | None = None + authenticated = False if signature and timestamp and self.api_key and self.secret_key: + # The signature verified over the bytes we were given. + authenticated = True if isinstance(signed_payload_hex, str) and signed_payload_hex: try: - trusted = json.loads(bytes.fromhex(signed_payload_hex).decode("utf-8")) + parsed = json.loads( + bytes.fromhex(signed_payload_hex).decode("utf-8") + ) except (ValueError, json.JSONDecodeError): - # Malformed signed_payload — the signature - # check above will already have rejected this - # message, so this branch should be unreachable - # in practice. We keep the fall-through to - # outer body to avoid a hard crash if the - # two checks ever drift. - trusted = None + # The signature verified, but the signed bytes do + # not parse. The outer body is NOT what was + # signed, so it is NOT a fallback source — the + # frame is dropped. + # + # (A legacy pre-FIX-C server signs the whole wire + # body and publishes no ``signed_payload``; that + # path verified over the body itself and is + # handled by the branch below.) + logger.error( + "Signed WS frame carries an unparseable signed_payload " + "— rejecting. The outer body is not what was signed, so " + "it cannot be used as a source." + ) + from nullrun.observability import metrics + + metrics.inc_transport("hmac_verify_failures_total") + return + if not isinstance(parsed, dict): + logger.error( + "Signed WS frame carries a non-object signed_payload " + "— rejecting." + ) + from nullrun.observability import metrics + + metrics.inc_transport("hmac_verify_failures_total") + return + trusted = parsed + + # The ONE source of truth for the dispatch below: the + # verified payload when there is one, the outer body + # otherwise. + source: dict[str, Any] = trusted if trusted is not None else data + msg_type = source.get("type", "") if msg_type == "initial_state": # Initial state with all workflow states - workflows = data.get("workflows", []) + workflows = source.get("workflows", []) logger.debug(f"Received initial state: {len(workflows)} workflows") for wf in workflows: # Trust the inner workflows[] entries the same @@ -461,13 +547,13 @@ async def _handle_message(self, message: str) -> None: elif msg_type == "state_change": # Workflow state change notification # Check if this message requires acknowledgment - await self._handle_state_change_with_ack(data, trusted) + await self._handle_state_change_with_ack(source, trusted) elif msg_type == "policy_invalidated": # Policy was updated via dashboard - SDK should clear its cache - organization_id = data.get("organization_id", "") - policy_id = data.get("policy_id", "") - new_version = data.get("new_version", 0) + organization_id = source.get("organization_id", "") + policy_id = source.get("policy_id", "") + new_version = source.get("new_version", 0) logger.info( f"Policy invalidated: {policy_id} v{new_version}, org: {organization_id}" ) @@ -479,9 +565,9 @@ async def _handle_message(self, message: str) -> None: elif msg_type == "key_rotated": # HMAC secret key was rotated - SDK should re-fetch from /auth/verify - organization_id = data.get("organization_id", "") - key_id = data.get("key_id", "") - new_version = data.get("new_version", 0) + organization_id = source.get("organization_id", "") + key_id = source.get("key_id", "") + new_version = source.get("new_version", 0) logger.info(f"Key rotated: {key_id} v{new_version}, org: {organization_id}") if self.on_key_rotated: try: @@ -508,11 +594,36 @@ async def _handle_message(self, message: str) -> None: # resolved_at: i64 Unix seconds, # message_id: Option, # } - approval_id = data.get("approval_id", "") - outcome = data.get("outcome", "") - execution_id = data.get("execution_id", "") - workflow_id = data.get("workflow_id", "") - message_id = data.get("message_id") + # + # SDK-5: this branch RELEASES a parked gate, so it is the + # privileged branch. It runs ONLY on an authenticated + # frame: the fields below come from the verified + # ``signed_payload`` (via ``source``), never from the + # writable outer body, and a frame the SDK cannot + # authenticate is REFUSED — counted and logged — so the + # waiter stays parked and fails closed on the approval + # timeout. + if not authenticated: + logger.error( + "Refusing an unauthenticated approval_resolved frame — it " + "would release a parked gate, and the SDK cannot tell it " + "from a genuine one. If this is expected, the API key has " + "no HMAC secret: rotate it (the backend counts the same " + "frames as ws_unsigned_messages_total)." + ) + # Local import to avoid a module-level cycle: + # observability imports nothing from us, so this + # is safe and lazy. + from nullrun.observability import metrics + + metrics.inc_transport("ws_unverified_approval_total") + return + + approval_id = source.get("approval_id", "") + outcome = source.get("outcome", "") + execution_id = source.get("execution_id", "") + workflow_id = source.get("workflow_id", "") + message_id = source.get("message_id") logger.info( f"Approval {outcome}: id={approval_id} exec={execution_id} wf={workflow_id}" ) @@ -542,7 +653,12 @@ async def _handle_message(self, message: str) -> None: ) if self.on_approval_resolved: try: - self.on_approval_resolved(data) + # The VERIFIED payload, not the outer body: the + # runtime reads approval_id / outcome / + # execution_id / note from this dict, and an + # outer body can be spliced by anyone who can + # inject bytes into the socket. + self.on_approval_resolved(source) except Exception as e: logger.warning(f"Approval resolved callback error: {e}") @@ -553,7 +669,7 @@ async def _handle_message(self, message: str) -> None: # ``initial_state`` — there is no "catch up" # semantics. We clear the version-dedup cache and # let ``_reconnect_loop`` reopen the connection. - reason = data.get("reason", "overflow") + reason = source.get("reason", "overflow") logger.warning( f"Server requested resync (reason={reason}); " "clearing local state and reconnecting" @@ -574,13 +690,13 @@ async def _handle_message(self, message: str) -> None: elif msg_type == "subscribed": # Subscription confirmation - organization_id = data.get("organization_id") + organization_id = source.get("organization_id") logger.debug(f"Subscribed to organization: {organization_id}") elif msg_type == "error": # Error message from server - code = data.get("code", "unknown") - message = data.get("message", "Unknown error") + code = source.get("code", "unknown") + message = source.get("message", "Unknown error") logger.warning(f"WebSocket error: {code} - {message}") else: @@ -603,7 +719,7 @@ async def _handle_message(self, message: str) -> None: "SDK to handle this type if it's expected to be " "in production soon.", msg_type, - sorted(data.keys()), + sorted(source.keys()), ) # Bump a metric so an SRE can alert on a spike of # unknown-type messages — that signals a real diff --git a/src/nullrun/wal_admin.py b/src/nullrun/wal_admin.py index a25b5b6..2d22c17 100644 --- a/src/nullrun/wal_admin.py +++ b/src/nullrun/wal_admin.py @@ -38,9 +38,13 @@ from collections import Counter from typing import Any -__all__ = ["main", "read_dlq", "dlq_paths", "default_wal_path"] +from nullrun.wal_paths import ( + WAL_PATH_ENV_VAR, + legacy_wal_path, + scoped_wal_paths, +) -_DEFAULT_WAL = os.path.join(tempfile.gettempdir(), "nullrun.wal") +__all__ = ["main", "read_dlq", "dlq_paths", "default_wal_path", "scoped_wal_paths"] # Refusals that replaying cannot fix, so replay refuses them by default. # @@ -67,15 +71,53 @@ def _is_terminal_without_replay(reason: str) -> bool: return reason in _TERMINAL_WITHOUT_REPLAY -def default_wal_path() -> str: - """The WAL path this host would use, honouring ``NULLRUN_WAL_PATH``. +def default_wal_path() -> str | None: + """The WAL this host would use, or ``None`` when that is not one path. - Mirrors ``Transport._wal_path``. Duplicated rather than imported so the - CLI can answer "where is my WAL" without constructing a Transport, which - would create an httpx client and touch the permissions of files it has no - business touching yet. + The order: + + 1. ``NULLRUN_WAL_PATH`` — the operator's explicit choice. It wins + outright. + 2. The single scoped WAL this user owns, when there is exactly one. + 3. ``None``. + + ``None`` is the honest answer when the user owns several scoped WALs + (one per API key). Guessing one of them would point the operator's + ``replay`` at a file that may hold another tenant's spend; the CLI + lists them instead and asks for ``--wal``. + + Before SDK-4 (security audit 2026-11-06) this returned one fixed path + — ``/nullrun.wal`` — shared by every user and every org on + the host. """ - return os.environ.get("NULLRUN_WAL_PATH") or _DEFAULT_WAL + env_path = os.environ.get(WAL_PATH_ENV_VAR) + if env_path: + return env_path + scoped = scoped_wal_paths() + if len(scoped) == 1: + return scoped[0] + return None + + +def legacy_wal_status() -> str | None: + """One line about the unscoped WAL an older SDK left behind, if any. + + It is never read and never migrated: it was shared by every user and + every org on the host, so its rows may belong to another tenant. + """ + legacy = legacy_wal_path() + try: + size = os.path.getsize(legacy) + except OSError: + return None + if size == 0: + return None + return ( + f"Legacy unscoped WAL at {legacy} ({size} bytes). It is NOT read and " + "NOT migrated: it was shared by every user and org on this host, so " + "its events may belong to another tenant. Inspect it with " + f"`nullrun-wal --wal {legacy} status`." + ) def dlq_paths(wal_path: str) -> dict[str, str]: @@ -144,7 +186,30 @@ def _row_event(row: dict[str, Any]) -> dict[str, Any] | None: def _cmd_status(args: argparse.Namespace) -> int: - wal_path = args.wal + if args.wal: + return _status_one(args.wal) + # No explicit path. The user may own several scoped WALs (one per API + # key), so list them all rather than guessing one — a guess could point + # the operator at another tenant's file. + scoped = scoped_wal_paths() + if not scoped: + print("No WAL found for this user.") + legacy = legacy_wal_status() + if legacy: + print(f"\n{legacy}") + return 0 + if len(scoped) > 1: + print(f"{len(scoped)} WAL(s) for this user:\n") + for path in scoped: + _status_one(path) + print() + legacy = legacy_wal_status() + if legacy: + print(legacy) + return 0 + + +def _status_one(wal_path: str) -> int: paths = dlq_paths(wal_path) print(f"WAL: {wal_path}") for name, path in paths.items(): @@ -249,6 +314,25 @@ def _unconfirmed_event_ids(transport: Any, wal_path: str) -> set[str]: def _cmd_replay(args: argparse.Namespace) -> int: from nullrun.transport import Transport + if not args.wal: + # `default_wal_path()` is the honest answer to "which one?": the + # single WAL this user owns, or None when there are several. A + # guess between several would point the replay at a file that may + # hold another tenant's spend. + one = default_wal_path() + if one: + hint = f" Pass --wal {one}." + else: + hint = ( + f" This host owns {len(scoped_wal_paths())} scoped WAL(s), and " + "`status` lists them." + ) + print( + f"error: no WAL to replay. Pass --wal .{hint}", + file=sys.stderr, + ) + return 1 + paths = dlq_paths(args.wal) rows, corrupt = read_dlq(paths["dlq"]) if corrupt: @@ -425,8 +509,12 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument( "--wal", - default=default_wal_path(), - help="WAL path (default: $NULLRUN_WAL_PATH, else the tempdir).", + default=None, + help=( + "WAL path. Default: $NULLRUN_WAL_PATH, else the one scoped WAL " + "this user owns. Without it, `status` lists every WAL this user " + "owns (one per API key) and `replay` refuses to guess." + ), ) sub = parser.add_subparsers(dest="command") diff --git a/src/nullrun/wal_paths.py b/src/nullrun/wal_paths.py new file mode 100644 index 0000000..5a9c223 --- /dev/null +++ b/src/nullrun/wal_paths.py @@ -0,0 +1,151 @@ +"""Where the SDK's write-ahead log lives by default. + +SDK-4 (security audit 2026-11-06): the WAL used to live at a FIXED +path in the shared OS temp directory — ``/nullrun.wal`` — for +every user and every org on the host. Two runtimes therefore resolved +to the SAME file, and the atomic whole-file replace in +``Transport._persist_to_wal`` destroyed the other runtime's undelivered +events. Spend that had already happened was then never booked. + +The default is now scoped per user AND per API KEY:: + + /nullrun--.wal + +Why the API KEY and not the org id +---------------------------------- +The path is resolved ONCE, when the transport is constructed, and never +changes afterwards. The org id is only known AFTER authentication, +which happens later; a path that changed at authentication would +orphan every file written before it, and no test would catch it — the +path is "right" at every instant and the money is lost anyway. + +The API key is known at construction and never changes, and one API key +belongs to exactly one organisation, so a key-scoped WAL is an +org-scoped WAL in money terms. Two organisations therefore never share +a WAL. Two workers of the SAME organisation using DIFFERENT keys never +share one either — that is the common multi-worker case, and it is +covered by the default. + +The digest is not decoration. Org ids and keys come from the server, +and a readable slug alone would let two different values collapse onto +one path — the exact collision this module exists to remove. + +``NULLRUN_WAL_PATH`` still wins outright. It MUST be unique per +runtime: pointing two runtimes at one path collides exactly like the +old shared default did, and the SDK cannot detect it. +""" + +from __future__ import annotations + +import getpass +import hashlib +import os +import tempfile + +__all__ = [ + "WAL_PATH_ENV_VAR", + "LEGACY_WAL_BASENAME", + "user_token", + "scope_token", + "default_wal_path", + "key_scoped_wal_path", + "legacy_wal_path", + "scoped_wal_paths", +] + + +# The env var that overrides the default path entirely. +WAL_PATH_ENV_VAR = "NULLRUN_WAL_PATH" + +# The unscoped path the SDK used before SDK-4. Never written to any +# more; still reported so an operator can find and inspect the events +# an older SDK left there. +LEGACY_WAL_BASENAME = "nullrun.wal" + +# How many hex characters of the digest are kept. Sixteen is enough to +# make a collision a non-event and short enough to read out loud. +_DIGEST_LEN = 16 + +# The prefix every scoped WAL file carries. The operator greps for it. +_SCOPED_PREFIX = "nullrun-" + + +def _digest(value: str) -> str: + """A short, stable digest of ``value``.""" + return hashlib.sha256(value.encode("utf-8")).hexdigest()[:_DIGEST_LEN] + + +def user_token() -> str: + """A short token identifying the OS user this process runs as. + + POSIX uid where the platform exposes one. Elsewhere a digest of the + account name: the raw name is not a secret, but it does not belong + in a temp directory other users can list. + """ + getuid = getattr(os, "getuid", None) + if callable(getuid): + return f"u{getuid()}" + try: + account = getpass.getuser() + except Exception: # noqa: BLE001 — no account name is not fatal here + account = "" + return "u" + _digest(account) + + +def scope_token( + api_key: str | None = None, org_id: str | None = None +) -> str: + """A short token identifying whose spend a WAL file holds. + + The API key when there is one — it is known before the org is, and + one key belongs to exactly one org, so the two scopes describe the + same money. The org id is the fallback for a transport that has no + key (it cannot book anything). Both are reduced to a digest: the + value itself never reaches a path that every user on the host can + list. + """ + if api_key: + return "k" + _digest(api_key) + if org_id: + return "o" + _digest(org_id) + return "x" + + +def default_wal_path( + api_key: str | None = None, org_id: str | None = None +) -> str: + """The default WAL base path for one user and one api key.""" + return os.path.join( + tempfile.gettempdir(), + f"{_SCOPED_PREFIX}{user_token()}-{scope_token(api_key, org_id)}.wal", + ) + + +def key_scoped_wal_path(api_key: str | None) -> str: + """The WAL base path for one user and one api key.""" + return default_wal_path(api_key=api_key) + + +def legacy_wal_path() -> str: + """The unscoped path the SDK used before SDK-4.""" + return os.path.join(tempfile.gettempdir(), LEGACY_WAL_BASENAME) + + +def scoped_wal_paths() -> list[str]: + """Every WAL base path this user owns, sorted. + + Only the base files are listed; the sibling files (``.1``, + ``.inflight``, ``.holdover``, ``.dlq``, ``.lock``) hang off a base + and are not separate WALs. + """ + temp_dir = tempfile.gettempdir() + prefix = f"{_SCOPED_PREFIX}{user_token()}-" + try: + names = sorted(os.listdir(temp_dir)) + except OSError: + return [] + return [ + os.path.join(temp_dir, name) + for name in names + if name.startswith(prefix) and name.endswith(".wal") + ] diff --git a/tests/conftest.py b/tests/conftest.py index 1bc0cb0..6b442f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -254,19 +254,18 @@ def _fast_wait_on(cancel, seconds): monkeypatch.setattr(_transport_mod, "_wait_on", _fast_wait_on) except Exception: pass - # Stub the modules that captured a module-level reference at import time. - try: - import nullrun.transport as _transport_mod - - monkeypatch.setattr(_transport_mod.time, "sleep", _fast_sleep) - except Exception: - pass - try: - import nullrun.breaker.circuit_breaker as _cb_mod - - monkeypatch.setattr(_cb_mod.time, "sleep", _fast_sleep) - except Exception: - pass + # ONE patch site for the global sleep. `nullrun.transport.time` and + # `nullrun.breaker.circuit_breaker.time` ARE the stdlib `time` + # module — the same object — so patching through those paths patched + # the attribute patched above, a second and a third time. Measured: + # `nullrun.transport.time is time` is True, and no module in src/ or + # tests/ does `from time import sleep` — every one of them does + # `import time` and calls `time.sleep(...)`, so the single patch + # above already reaches all of them. A test that needs the + # TRANSPORT's own wait patches the transport's seam + # (`nullrun.transport._sleep`), never the global: the global carries + # the sleep of every thread in the process, and a foreign value in a + # retry-delay list is a flake with no cause in the retry path. yield diff --git a/tests/test_2026_09_10_toolblocked_parser.py b/tests/test_2026_09_10_toolblocked_parser.py index 7671441..5a93577 100644 --- a/tests/test_2026_09_10_toolblocked_parser.py +++ b/tests/test_2026_09_10_toolblocked_parser.py @@ -37,11 +37,26 @@ Post-fix: - Added a dedicated dispatch branch BEFORE the final catalog - fallback. Detects ``catalog is NullRunToolBlockedError`` or - ``catalog is NullRunBlockedException`` and calls the - constructor with the right (workflow_id, reason, status_code, - tool_name) signature. ``forwarded`` is passed through so - catalog value's defaults win. + fallback. Detects the blocked family and calls the constructor + with the right (workflow_id, reason, status_code, tool_name) + signature. ``forwarded`` is passed through so catalog value's + defaults win. + +SDK-15 (2026-11-07) widened the branch. It used to test the class +by IDENTITY — ``catalog is NullRunToolBlockedError or catalog is +NullRunBlockedException`` — so every typed SUBCLASS of the blocked +family (the approval family, the MCP family) fell through to the +generic fallback and raised the same TypeError this file was written +to kill: nineteen codes, all of them flattened into the synthetic +block by the transport's catch-all. The branch now matches by +SUBCLASS (``issubclass(catalog, NullRunBlockedException)``), which +covers the two exact classes and every subclass of them. + +The source pins below were re-pointed at that contract. They used to +assert the branch's exact bytes (an identity test on two class +names); a byte-window that has been widened by a deliberate fix must +be re-pointed at the STRUCTURAL boundary, not satisfied by keeping +dead code alive to match the old window. These tests pin BOTH the source shape AND the runtime behavior so a future refactor that reverts to the broken generic fallback @@ -69,7 +84,16 @@ # The dedicated dispatch branch DEF-NR-TOOLBLOCKED-PARSER added. Used as # the content needle for locating the dispatching parser — see # ``_parser_fn_containing``. -_BRANCH_NEEDLE = "catalog is NullRunToolBlockedError" +# +# The needle is the branch's SUBCLASS TEST, which is the one line that +# exists only in that branch. Two needles were tried and rejected: +# a class NAME is self-satisfying (the function-local import block +# names it even after the branch is deleted, so the pin would go green +# against a parser with no branch at all), and the branch's +# constructor call is not unique to it (the NullRunBudgetError arm +# above builds the same ``workflow_id=`` argument, so the pin stayed +# green when the blocked branch was gutted). +_BRANCH_NEEDLE = "issubclass(catalog, NullRunBlockedException)" def _read(path: Path) -> str: @@ -161,23 +185,27 @@ class TestDefNrToolblockedParserSourcePin: def test_dedicated_branch_present(self): src = _read(TRANSPORT_PY) - # The fix added an ``if catalog is NullRunToolBlockedError - # or catalog is NullRunBlockedException:`` branch BEFORE - # the generic ``catalog(full_message, **forwarded)`` - # fallback. + # The fix added a dispatch branch BEFORE the generic + # ``catalog(full_message, **forwarded)`` fallback. It + # matches the blocked family BY SUBCLASS (SDK-15): an + # identity test on two class names left every typed + # subclass of the family — the approval family, the MCP + # family, nineteen codes — falling through to the generic + # fallback and raising the TypeError this branch exists to + # prevent. assert re.search( - r"catalog\s+is\s+NullRunToolBlockedError\s*\n\s+or\s+catalog\s+is\s+NullRunBlockedException", + r"issubclass\(\s*catalog,\s*NullRunBlockedException\s*\)", src, ), ( "DEF-NR-TOOLBLOCKED-PARSER: the dedicated " - "NullRunToolBlockedError / NullRunBlockedException " - "dispatch branch must be present in " - "_parse_v3_error_envelope. Pre-fix the generic " - "catalog-fallback called " - "``catalog(full_message, **forwarded)`` which " - "raised TypeError because " - "NullRunBlockedException.__init__ requires positional " - "(workflow_id, reason)." + "NullRunBlockedException dispatch branch must be present " + "in _parse_v3_error_envelope, and it must match the " + "family BY SUBCLASS. Pre-fix the generic catalog-fallback " + "called ``catalog(full_message, **forwarded)`` which " + "raised TypeError because NullRunBlockedException." + "__init__ requires positional (workflow_id, reason); and " + "an identity test on two class names left every subclass " + "of the family with the same TypeError." ) def test_branch_in_parser(self): @@ -188,24 +216,30 @@ def test_branch_in_parser(self): src = _read(TRANSPORT_PY) # Locate the parser function body that holds the dedicated # branch. Resolved by the BRANCH, not by the exception name: - # resolving on ``NullRunToolBlockedError`` is self-satisfying, - # because the function-local import block names it even after - # the branch is deleted. Verified — deleting the branch left + # resolving on a class name is self-satisfying, because the + # function-local import block names it even after the branch + # is deleted. Verified — deleting the branch left # this pin green that way. The branch text is the only needle # that is actually absent when the branch is gone. fn_body = _parser_fn_containing(src, _BRANCH_NEEDLE) assert _BRANCH_NEEDLE in fn_body, ( - "DEF-NR-TOOLBLOCKED-PARSER: NullRunToolBlockedError " - "must be referenced inside the v3-envelope parser that " - "performs the dispatch (the dedicated dispatch branch " - "lives there)." + "DEF-NR-TOOLBLOCKED-PARSER: the blocked-family " + "constructor call must be inside the v3-envelope parser " + "that performs the dispatch (the dedicated dispatch " + "branch lives there)." ) def test_import_includes_blocked_exception_classes(self): """The function-local import block in the dispatching parser - must include both ``NullRunToolBlockedError`` and - ``NullRunBlockedException`` — otherwise NameError at - runtime even though the branch is present.""" + must include ``NullRunBlockedException`` — otherwise NameError + at runtime even though the branch is present. + + The branch names the BASE class of the family (it matches by + subclass), so the base is what it must import. The concrete + subclasses are named by the CATALOG, which is built in + ``_build_v3_error_code_map`` with its own import block — a + class the parser never names must not be pinned here. + """ src = _read(TRANSPORT_PY) # Locate the function-local import block (the one inside the # function that holds the branch, NOT the module-level one). @@ -223,11 +257,6 @@ def test_import_includes_blocked_exception_classes(self): "v3-envelope parser" ) imported = import_block.group(1) - assert "NullRunToolBlockedError" in imported, ( - "DEF-NR-TOOLBLOCKED-PARSER: NullRunToolBlockedError " - "must be imported in the function-local block — the " - "dedicated dispatch branch references it." - ) assert "NullRunBlockedException" in imported, ( "DEF-NR-TOOLBLOCKED-PARSER: NullRunBlockedException " "must be imported in the function-local block — the " @@ -271,11 +300,10 @@ def test_branch_uses_correct_constructor_signature(self): ``catalog(full_message, **forwarded)`` form for these classes.""" src = _read(TRANSPORT_PY) - # Extract the body of the new branch (between - # ``catalog is NullRunToolBlockedError`` and the next - # ``return cast(Exception, instance)``). + # Extract the body of the new branch (between the subclass + # test and the next ``return cast(Exception, instance)``). m = re.search( - r"if \(\s*\n\s*catalog is NullRunToolBlockedError.*?" + r"if issubclass\(catalog,\s*NullRunBlockedException\).*?" r"return cast\(Exception, instance\)", src, re.DOTALL, diff --git a/tests/test_actions.py b/tests/test_actions.py index c213d85..ff65039 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -17,7 +17,10 @@ handle_action, register_action_handler, ) -from nullrun.breaker.exceptions import NullRunBlockedException +from nullrun.breaker.exceptions import ( + NullRunBlockedException, + NullRunUnknownActionError, +) class TestActionHandlerInit: @@ -262,32 +265,58 @@ def test_block_does_not_propagate_exception(self): # =========================================================================== -# B14: unknown action type must NOT silently BLOCK +# B14 + SDK-12: an unknown action type must NOT silently BLOCK, and +# must not be silently RETURNED either # =========================================================================== -# Pre-fix: an unknown action type (e.g. server schema regression +# Pre-B14: an unknown action type (e.g. server schema regression, # version mismatch, or attacker-controlled input) silently degraded # to ``ActionType.BLOCK`` and triggered ``_default_block``, which # raises ``NullRunBlockedException``. That made the SDK into a DoS # amplifier: one malformed message stopped the whole workflow. -# Post-fix: log at ERROR, record a forensic event with the unknown -# action type, and DO NOT invoke any handler. Workflow continues. - - -class TestUnknownActionTypeFailOpen: - """Unknown action types must fail open, not silently BLOCK.""" +# +# B14's fix kept the DoS property but left the action neither executed +# nor refused — it was logged and RETURNED, so the caller believed it +# had run. SDK-12 closes that: the unknown type now RAISES a typed +# infrastructure error (``NullRunUnknownActionError``). It is +# deliberately NOT a block, so B14's property survives. +# +# The full contract is pinned in tests/test_sdk12_unknown_action.py. +# What follows is the in-place rewrite of the B14-era pins: they +# asserted "must not raise", which is now the opposite of the truth, +# so they are REPLACED by the stronger pair ("raises the typed error" +# AND "is not a block"), never relaxed. + + +class TestUnknownActionTypeRefused: + """An unknown action type must be REFUSED — and must not BLOCK.""" + + def test_unknown_action_raises_a_typed_error(self): + """SDK-12: the action was neither executed nor refused before. + + The caller must be able to tell "it ran" from "it could not + run", so the unknown type raises. + """ + handler = ActionHandler() + with pytest.raises(NullRunUnknownActionError): + handler.handle("totally_made_up_action", "wf-mystery", "test reason") def test_unknown_action_does_not_raise_blocked_exception(self): - """Unknown action type must not raise NullRunBlockedException. + """B14's property survives: an unknown type must not BLOCK. - Pre-fix this raised ``NullRunBlockedException`` because + Pre-B14 this raised ``NullRunBlockedException`` because ``ActionType(action.lower )`` raised ``ValueError`` which was caught and silently fell through to ``ActionType.BLOCK`` - → ``_default_block`` → raise. Post-fix the method returns - cleanly and the workflow continues. + → ``_default_block`` → raise. That made the SDK into a DoS + amplifier: one malformed message stopped the whole workflow. + The refusal is therefore an infrastructure error, not a block. """ handler = ActionHandler() - # Must not raise. - handler.handle("totally_made_up_action", "wf-mystery", "test reason") + with pytest.raises(NullRunUnknownActionError) as info: + handler.handle("totally_made_up_action", "wf-mystery", "test reason") + assert not isinstance(info.value, NullRunBlockedException), ( + "an unknown action type must not become a block — that was " + "the DoS amplifier B14 removed" + ) def test_unknown_action_records_forensic_event(self): """Unknown action type is still recorded in action history. @@ -298,7 +327,8 @@ def test_unknown_action_records_forensic_event(self): event in history. """ handler = ActionHandler() - handler.handle("not_a_real_action", "wf-mystery", "real reason") + with pytest.raises(NullRunUnknownActionError): + handler.handle("not_a_real_action", "wf-mystery", "real reason") history = handler.get_action_history() assert len(history) == 1 @@ -317,7 +347,8 @@ def test_unknown_action_logs_at_error_level(self, caplog): handler = ActionHandler() with caplog.at_level(logging.ERROR, logger="nullrun.actions"): - handler.handle("bogus", "wf-x", "r") + with pytest.raises(NullRunUnknownActionError): + handler.handle("bogus", "wf-x", "r") error_records = [r for r in caplog.records if r.levelno >= logging.ERROR] assert any("bogus" in r.getMessage() for r in error_records), ( @@ -331,12 +362,10 @@ def test_known_actions_still_work_after_unknown_action(self): Regression guard: a malformed action in the stream must not prevent subsequent KILL/PAUSE/etc. from being delivered. - Pre-fix the silent-BLOCK raised an exception that the - ``except BaseException`` swallowed, but a future change to - that catch could break this — pin it. """ handler = ActionHandler() - handler.handle("malformed_first", "wf-mix", "first") + with pytest.raises(NullRunUnknownActionError): + handler.handle("malformed_first", "wf-mix", "first") # Now a real KILL — must still be recorded and still raise. handler.handle(ActionType.KILL, "wf-mix", "second") @@ -413,15 +442,16 @@ def test_clear_history_empties_list(): def test_handle_unknown_action_does_not_invoke_handler(): - """B14: unknown action logs ERROR + records BLOCK but - does NOT invoke any handler (fail-open). Pre-fix this degraded - to BLOCK → DoS amplifier. + """B14 + SDK-12: an unknown action logs ERROR, records a forensic + event, invokes NO handler, and refuses with a typed error. + Pre-B14 this degraded to BLOCK → DoS amplifier. """ h = ActionHandler() handler_mock = MagicMock() h.register_handler(ActionType.BLOCK, handler_mock) - # ``"weird"`` is not in ActionType — should fail-open. - h.handle("weird", "wf-1", reason="x") + # ``"weird"`` is not in ActionType — it must not run a handler. + with pytest.raises(NullRunUnknownActionError): + h.handle("weird", "wf-1", reason="x") handler_mock.assert_not_called() @@ -431,7 +461,8 @@ def test_handle_unknown_action_records_block_event(caplog): h = ActionHandler() with caplog.at_level(logging.ERROR, logger="nullrun.actions"): - h.handle("unknown_action_type", "wf-1", reason="x") + with pytest.raises(NullRunUnknownActionError): + h.handle("unknown_action_type", "wf-1", reason="x") history = h.get_action_history() assert any(e.action_type == "block" for e in history) @@ -455,9 +486,15 @@ def test_handle_action_lowercases_input(): def test_handle_kill_does_not_propagate_killed_interrupt(): """``WorkflowKilledInterrupt`` from the handler is SWALLOWED by the - dispatch loop (BaseException caught and logged). The kill signal - has already been recorded in history by the time the dispatch - wraps the handler call — re-raising would lose the audit entry. + dispatch loop. + + The old rationale for the swallow — "re-raising would lose the + audit entry" — was factually wrong: the history entry is written + BEFORE the handler is called, so a raise cannot lose it. The real + reason the refusal is swallowed is that the authoritative + enforcement of a kill is the GATE (ADR-063 §3 — the gate refuses + the next /check); re-raising here would abort the caller's action + loop over an action that was delivered correctly. """ h = ActionHandler() h.handle("kill", "wf-1", reason="x") # no raise diff --git a/tests/test_api_key_fingerprint.py b/tests/test_api_key_fingerprint.py new file mode 100644 index 0000000..beb67c9 --- /dev/null +++ b/tests/test_api_key_fingerprint.py @@ -0,0 +1,152 @@ +"""SDK-5 — no character of the API key reaches a user-visible string. + +Before the fix the SDK printed three different fragments of the key: +``api_key[:10]`` in the status snapshot, ``api_key[:10]`` in the error +context, and ``api_key[:8] + "***"`` in the transport's debug log. A +prefix of a credential is a disclosure, and "the full key never leaves +the SDK" was not a claim anyone could check. + +After the fix every one of those places prints the SAME 8-character +SHA-256 fingerprint of the key — a correlation handle, not a fragment. + +Every key in this file is fake. +""" + +from __future__ import annotations + +import logging + +import pytest + +from nullrun.observability.redaction import api_key_fingerprint + +# A fake key. It is long enough to have a prefix and a suffix and it +# is not a credential. +FAKE_KEY = "nr_live_FAKEKEYFORTESTSONLY_0123456789" + +# sha256(FAKE_KEY)[:8] — hardcoded so the test does not read the +# helper it is checking. +FAKE_KEY_FINGERPRINT = "956dceaa" + + +@pytest.fixture(autouse=True) +def _isolated_wal(tmp_path, monkeypatch): + """The transport touches the WAL on construction. Keep it in tmp.""" + monkeypatch.setenv("NULLRUN_WAL_PATH", str(tmp_path / "sdk.wal")) + + +class TestTheFingerprintHelper: + def test_it_is_the_sha256_prefix(self): + import hashlib + + expected = hashlib.sha256(FAKE_KEY.encode()).hexdigest()[:8] + assert api_key_fingerprint(FAKE_KEY) == expected + + def test_it_is_eight_characters(self): + assert len(api_key_fingerprint(FAKE_KEY)) == 8 + + def test_a_missing_key_has_no_fingerprint(self): + assert api_key_fingerprint(None) is None + assert api_key_fingerprint("") is None + + def test_two_different_keys_do_not_share_a_fingerprint(self): + assert api_key_fingerprint(FAKE_KEY) != api_key_fingerprint( + "nr_live_OTHERKEYFORTESTSONLY_012345678" + ) + + +class TestTheStatusString: + """The audit's verification: ``NullRunStatus.__str__`` must not + contain more than the agreed number of key characters — and the + agreed number is zero.""" + + def test_the_status_carries_no_character_of_the_key(self, make_runtime): + rt = make_runtime(api_key=FAKE_KEY) + s = rt.status() + + rendered = str(s) + assert FAKE_KEY not in rendered + assert "nr_live_" not in rendered + assert "FAKEKEY" not in rendered + assert "0123456789" not in rendered + + def test_the_status_carries_the_fingerprint(self, make_runtime): + rt = make_runtime(api_key=FAKE_KEY) + s = rt.status() + + assert s.api_key_prefix == FAKE_KEY_FINGERPRINT + assert FAKE_KEY_FINGERPRINT in str(s) + + def test_no_prefix_of_the_key_survives(self, make_runtime): + """Every 4-character window of the key must be absent. A test + that only checks the whole key would pass on a slice.""" + rt = make_runtime(api_key=FAKE_KEY) + rendered = str(rt.status()) + + key = FAKE_KEY + for i in range(len(key) - 3): + window = key[i : i + 4] + assert window not in rendered, f"the key window {window!r} leaked" + + +class TestTheErrorContext: + def test_the_error_context_carries_the_same_fingerprint( + self, make_runtime, monkeypatch + ): + from nullrun.observability.error_hooks import ErrorContext + + rt = make_runtime(api_key=FAKE_KEY) + ctx = ErrorContext(stage="init", api_key_prefix=api_key_fingerprint(FAKE_KEY)) + assert ctx.api_key_prefix == FAKE_KEY_FINGERPRINT + assert rt.status().api_key_prefix == ctx.api_key_prefix + + def test_the_emitted_error_context_carries_no_key_character( + self, make_runtime, monkeypatch + ): + """The runtime's own error emitter is the path that ships the + context to the operator's hook.""" + from nullrun.observability.error_hooks import ErrorContext, register_hook + + seen: list[ErrorContext] = [] + + def _hook(err, ctx) -> None: + seen.append(ctx) + + unregister = register_hook(_hook) + try: + rt = make_runtime(api_key=FAKE_KEY) + rt._emit_sdk_error( + RuntimeError("boom"), stage="auth", tool_name="charge_card" + ) + finally: + unregister() + + assert seen, "the error hook was never called" + for ctx in seen: + assert ctx.api_key_prefix == FAKE_KEY_FINGERPRINT + assert "FAKEKEY" not in repr(ctx) + + +class TestTheTwoPlacesAgree: + """The transport used to print eight characters, the status ten.""" + + def test_the_transport_log_and_the_status_carry_the_same_fingerprint( + self, make_runtime, caplog + ): + from nullrun.transport import Transport + + with caplog.at_level(logging.DEBUG, logger="nullrun.transport"): + Transport(api_url="https://api.test.nullrun.io", api_key=FAKE_KEY) + + transport_lines = [ + r.getMessage() for r in caplog.records if r.name == "nullrun.transport" + ] + assert transport_lines, "the transport logged nothing" + joined = "\n".join(transport_lines) + + assert f"api_key_fp={FAKE_KEY_FINGERPRINT}" in joined + assert "nr_live_" not in joined + assert "FAKEKEY" not in joined + + rt = make_runtime(api_key=FAKE_KEY) + assert rt.status().api_key_prefix == FAKE_KEY_FINGERPRINT diff --git a/tests/test_decision_cache_surface_has_no_producer.py b/tests/test_decision_cache_surface_has_no_producer.py new file mode 100644 index 0000000..c9980d0 --- /dev/null +++ b/tests/test_decision_cache_surface_has_no_producer.py @@ -0,0 +1,208 @@ +"""The SDK has no decision cache, and its surface says it has one. + +``DecisionSource.CACHED`` and ``ExecuteConfig.cache_ttl`` / +``cache_max_size`` sit in the public surface with no producer +anywhere in ``src/``. The SDK has no in-process gate cache +(NR-116 / ADR-180): every call is its own ``/gate`` call with its +own reservation. A vestigial member that reads as a live decision +cache is a trap -- a future "restore the cache" change would land +green, and a reader would assume a decision can be reused. + +This file pins the ABSENCE of a producer. That is a textual +property of the tree, so it is checked with an AST walk rather than +a substring search -- a substring search would also match the +comment that documents the defect. + +What this does NOT prove: that no future change adds a producer. +It proves that no producer exists now, and it fails the moment one +appears. +""" + +from __future__ import annotations + +import ast +import pathlib + +import pytest + +from nullrun.transport import DecisionSource, ExecuteConfig + +_SRC_ROOT = pathlib.Path(__file__).resolve().parent.parent / "src" / "nullrun" + + +def _modules() -> list[pathlib.Path]: + return sorted(p for p in _SRC_ROOT.rglob("*.py")) + + +def _parsed() -> list[tuple[pathlib.Path, ast.Module]]: + out: list[tuple[pathlib.Path, ast.Module]] = [] + for path in _modules(): + tree = ast.parse(path.read_text(encoding="utf-8")) + out.append((path, tree)) + return out + + +def _rel(path: pathlib.Path) -> str: + return path.relative_to(_SRC_ROOT).as_posix() + + +def _set_elements(value: ast.expr) -> list[ast.expr] | None: + """The elements of a set literal, whether it is written bare or + wrapped in ``frozenset(...)`` / ``set(...)``.""" + if isinstance(value, ast.Set): + return list(value.elts) + if isinstance(value, ast.Call) and isinstance(value.func, ast.Name): + if value.func.id in {"frozenset", "set"} and value.args: + first = value.args[0] + if isinstance(first, ast.Set): + return list(first.elts) + return None + + +def _cached_occurrences() -> list[tuple[str, str]]: + """Every ``"cached"`` string constant in the package, with the + STRUCTURAL ROLE it plays. + + Two roles are surface and legal: the ``DecisionSource`` enum + member, and the runtime's provenance allowlist. Every other + appearance is reported as ``unclassified`` -- a literal nobody can + account for is a producer until proven otherwise.""" + found: list[tuple[str, str]] = [] + for path, tree in _parsed(): + rel = _rel(path) + roles: dict[int, str] = {} + + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + if node.name != "DecisionSource": + continue + for stmt in node.body: + if not isinstance(stmt, ast.Assign): + continue + targets = [ast.unparse(t) for t in stmt.targets] + if "CACHED" not in targets: + continue + if isinstance(stmt.value, ast.Constant) and stmt.value.value == "cached": + roles[id(stmt.value)] = "enum_member" + + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + targets = [ast.unparse(t) for t in node.targets] + if "_KNOWN_DECISION_SOURCES" not in targets: + continue + elements = _set_elements(node.value) + if elements is None: + continue + for element in elements: + if isinstance(element, ast.Constant) and element.value == "cached": + roles[id(element)] = "allowlist_member" + + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and node.value == "cached": + found.append((rel, roles.get(id(node), "unclassified"))) + return found + + +class TestNoProducerForCached: + """``DecisionSource.CACHED`` is never produced.""" + + def test_the_member_is_never_referenced(self): + """No attribute access ``DecisionSource.CACHED`` anywhere. + + The member exists for import compatibility. Referencing it + would be a producer, or a comparison against a value that + never arrives.""" + offenders: list[str] = [] + for path, tree in _parsed(): + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute): + continue + if node.attr != "CACHED": + continue + offenders.append(f"{_rel(path)}:{node.lineno}: {ast.unparse(node)}") + assert not offenders, ( + "`DecisionSource.CACHED` is referenced in the package; the " + "SDK has no decision cache, so nothing may read it:\n " + + "\n ".join(offenders) + ) + + def test_the_literal_appears_only_as_surface(self): + """The string ``"cached"`` appears exactly twice in the + package, and both occurrences are surface: + + * the enum member in ``transport.py``; + * the runtime's known-provenance allowlist, which accepts a + body that CLAIMS the provenance -- it does not produce it. + + A third occurrence would be a producer, or a new surface that + says the same thing a second way. + + The shape is pinned structurally, not by line number: a line + number moves whenever anything above it is edited, and a pin + that breaks on an unrelated edit is a broken pin.""" + kinds = _cached_occurrences() + assert sorted(kinds) == [ + ("runtime.py", "allowlist_member"), + ("transport.py", "enum_member"), + ], ( + "The string 'cached' must appear exactly twice in the package " + "-- the DecisionSource member and the runtime's provenance " + "allowlist. A new occurrence is either a producer or a second " + "spelling of the same vestigial shape: " + ", ".join( + f"{f}:{k}" for f, k in kinds + ) + ) + + def test_the_allowlist_entry_is_a_set_member_not_a_producer(self): + """The runtime's occurrence is an element of the + ``_KNOWN_DECISION_SOURCES`` set. Pin the shape so the entry + cannot quietly become a return value.""" + found = False + for path, tree in _parsed(): + if _rel(path) != "runtime.py": + continue + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + targets = [ast.unparse(t) for t in node.targets] + if "_KNOWN_DECISION_SOURCES" not in targets: + continue + elements = _set_elements(node.value) + assert elements is not None, ( + "_KNOWN_DECISION_SOURCES must stay a set literal" + ) + assert any( + isinstance(e, ast.Constant) and e.value == "cached" + for e in elements + ), "the allowlist no longer names 'cached'" + found = True + assert found, "_KNOWN_DECISION_SOURCES was not found in runtime.py" + + +class TestNoReaderForTheCacheConfig: + """``ExecuteConfig.cache_ttl`` / ``cache_max_size`` are never read.""" + + @pytest.mark.parametrize("field", ["cache_ttl", "cache_max_size"]) + def test_the_field_is_never_read(self, field): + """No attribute access anywhere in the package. + + The fields stay on the dataclass for import compatibility. A + reader would be the first half of a decision cache.""" + offenders: list[str] = [] + for path, tree in _parsed(): + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and node.attr == field: + offenders.append(f"{_rel(path)}:{node.lineno}: {ast.unparse(node)}") + assert not offenders, ( + f"`ExecuteConfig.{field}` is read in the package; the SDK has " + "no decision cache:\n " + "\n ".join(offenders) + ) + + def test_the_fields_still_exist_on_the_dataclass(self): + """The pin above is about READERS. The surface itself is + public and stays -- removing it would break importers for no + safety gain.""" + assert ExecuteConfig().cache_ttl == 60.0 + assert ExecuteConfig().cache_max_size == 10000 + assert DecisionSource.CACHED == "cached" diff --git a/tests/test_edge_via_edge.py b/tests/test_edge_via_edge.py index 873d94f..b44dce9 100644 --- a/tests/test_edge_via_edge.py +++ b/tests/test_edge_via_edge.py @@ -14,6 +14,8 @@ from nullrun.edge import ( EDGE_LEASE, + EDGE_NOT_ENFORCED, + EDGE_UNREACHABLE, EdgeConfigurationError, EdgeTransport, edge_transport_from_env, @@ -123,14 +125,29 @@ def test_a_lease_decision_is_not_a_fallback_decision(): assert is_fallback_decision_source(EDGE_LEASE) is False -def test_an_unreachable_box_is_a_fallback_so_strict_still_closes(): - """E3. The unreachable answer is NOT a lease answer. - - Same shape, opposite direction: this one must read as synthetic, or - the runtime would treat "the box did not answer" as the lease having - spoken. +def test_an_unreachable_box_is_not_a_fallback_source(): + """E3 / SDK-3. The unreachable answer is NOT a lease answer AND NOT + a fallback answer. + + Same shape, opposite direction: it must not read as synthetic + either. `is_fallback_decision_source` is the runtime's fail-OPEN + predicate — a body whose source is a fallback source means "the + engine never answered", and ADR-008 lets the budget pre-flight + proceed on it (the cloud's `/track` can still bill the call + later). + + The edge is the opposite case, and the difference is not stylistic: + in edge mode the BOX is the enforcement surface, the cloud is not + consulted at all, and there is no `/track` recovery — the cloud the + spend would have been recovered through is the same one that is + down. Reusing FALLBACK made the two surfaces indistinguishable, + and the runtime — which can only see the source — discarded the + edge's refusal and reported "no decision" (SDK-3). """ assert is_fallback_decision_source(DecisionSource.FALLBACK) is True + assert is_fallback_decision_source(EDGE_UNREACHABLE) is False + assert is_fallback_decision_source(EDGE_NOT_ENFORCED) is False + assert is_fallback_decision_source(EDGE_LEASE) is False # ── what the SDK is allowed to say ───────────────────────────────────── @@ -279,8 +296,8 @@ def test_an_unreachable_box_blocks_and_does_not_reach_the_cloud(): The cloud is not configured anywhere in this object, so there is nothing to fall back to even in principle. What is asserted is the - shape the runtime needs: a synthetic block, not an exception and - definitely not an allow. + shape the runtime needs: a block the runtime cannot mistake for a + transport error — not an exception and definitely not an allow. """ def boom(body): raise OSError("connection refused") @@ -288,21 +305,22 @@ def boom(body): t = _transport(boom) result = t.enforce("gpt-4o", 0, 10) assert result["decision"] == "block" - assert result["decision_source"] == DecisionSource.FALLBACK + assert result["decision_source"] == EDGE_UNREACHABLE assert result["error_code"] == "EDGE_UNREACHABLE" - assert is_fallback_decision_source(result["decision_source"]) is True + assert is_fallback_decision_source(result["decision_source"]) is False def test_a_broken_box_is_not_laundered_into_a_budget_answer(): """5xx means the box is broken, which is not the same as 'no money'. - It takes the transport-error path so STRICT closes. A 5xx presented - as a lease refusal would be a decision the lease never made. + It takes the transport-error path so the refusal stays a refusal. A + 5xx presented as a lease refusal would be a decision the lease + never made. """ t = _transport(lambda body: _FakeResponse({"allowed": False}, status_code=503)) result = t.enforce("gpt-4o", 0, 10) assert result["decision"] == "block" - assert result["decision_source"] == DecisionSource.FALLBACK + assert result["decision_source"] == EDGE_UNREACHABLE assert result["error_code"] == "EDGE_UNREACHABLE" @@ -314,7 +332,8 @@ def responder(body): t = _transport(responder) result = t.enforce("gpt-4o", 0, 10) - assert result["decision_source"] == DecisionSource.FALLBACK + assert result["decision"] == "block" + assert result["decision_source"] == EDGE_UNREACHABLE def test_a_broken_observer_cannot_change_the_decision(): @@ -360,10 +379,69 @@ def must_not_be_called(*a, **kw): def test_execute_does_not_reach_the_cloud_in_via_edge_mode(monkeypatch): - """E4. `/execute` has no box equivalent — and must not look for one. + """E4 / SDK-2. `/execute` has no box equivalent — and must not look + for one. The box's `/enforce` already decided and charged, so a second call would either hit the cloud or double-charge. + + What `/execute` MAY do is read back what the box said about THIS + event. What it must not do is DECIDE — pre-fix it returned a + client-side `{"decision": "allow"}` without the box ever being + asked, so an expired, revoked or exhausted lease still allowed the + call. + """ + monkeypatch.setenv("NULLRUN_EDGE_URL", "http://127.0.0.1:18090") + monkeypatch.setenv("NULLRUN_EDGE_LEASE_ID", "lease-1") + monkeypatch.setenv("NULLRUN_EDGE_TOKEN", "box-token") + monkeypatch.setenv("NULLRUN_API_KEY", "nr_org_key") + + t = Transport(api_url="https://api.nullrun.io") + + def must_not_be_called(*a, **kw): + raise AssertionError("the cloud was called in via-edge mode") + + t._client.post = must_not_be_called # type: ignore[method-assign] + + # The box is asked once, about one event. + t.edge._post = lambda body: _FakeResponse( # type: ignore[method-assign] + {"allowed": True, "spent_millicents": 10, "remaining_millicents": 90} + ) + t.edge.enforce("gpt-4o", 0, 10, event_id="evt-1") + + from nullrun.context import set_operation_id + + token = set_operation_id("evt-1") + try: + result = t.execute( + organization_id="org-1", + execution_id="exec-1", + trace_id="trace-1", + tool="read_file", + input_data={}, + ) + finally: + from nullrun.context import _operation_id_var + + _operation_id_var.reset(token) + + assert result["decision"] == "allow" + assert result["decision_source"] == EDGE_LEASE + assert result["via_edge"] is True + + +def test_execute_with_no_box_answer_is_refused(monkeypatch): + """SDK-2. The box was never asked about this call, so it is refused. + + This is the shape of the hole the fix closes. Pre-fix `/execute` + answered "allow" with no box answer at all, so the whole edge + decision surface was client-side only — a box that said no, or was + never asked, was indistinguishable from a grant in force. + + The cloud applies the same rule for the same reason: a direct + `/execute` with no prior `/gate` is answered 404 + EXECUTION_NOT_FOUND, because the backend's `execution:{id}` + binding does not exist. """ monkeypatch.setenv("NULLRUN_EDGE_URL", "http://127.0.0.1:18090") monkeypatch.setenv("NULLRUN_EDGE_LEASE_ID", "lease-1") @@ -376,7 +454,6 @@ def must_not_be_called(*a, **kw): raise AssertionError("the cloud was called in via-edge mode") t._client.post = must_not_be_called # type: ignore[method-assign] - t.edge.last_source = EDGE_LEASE result = t.execute( organization_id="org-1", @@ -385,8 +462,45 @@ def must_not_be_called(*a, **kw): tool="read_file", input_data={}, ) - assert result["decision"] == "allow" - assert result["via_edge"] is True + assert result["decision"] == "block" + assert result["decision_source"] == EDGE_NOT_ENFORCED + assert result["error_code"] == "EDGE_NOT_ENFORCED" + assert is_fallback_decision_source(result["decision_source"]) is False + + +def test_a_box_answer_is_consumed_once_and_never_reused(): + """SDK-2. A recorded answer authorises ONE action. + + `take_answer` pops. That is what makes a stale answer impossible: + an answer that was never consumed cannot be handed to a later + action, and a later action that never asked the box gets "no + answer" and is refused. + """ + t = _transport( + lambda body: _FakeResponse({"allowed": True, "spent_millicents": 10}) + ) + t.enforce("gpt-4o", 0, 10, event_id="evt-1") + + first = t.take_answer("evt-1") + assert first is not None + assert first["decision"] == "allow" + + # Second take: nothing left. The box's answer was for that one + # action. + assert t.take_answer("evt-1") is None + + +def test_an_answer_for_another_event_does_not_authorise_this_one(): + """The answer is keyed by event, not by "the last thing that + happened". A different event is a different call and a different + charge.""" + t = _transport( + lambda body: _FakeResponse({"allowed": True, "spent_millicents": 10}) + ) + t.enforce("gpt-4o", 0, 10, event_id="evt-1") + + assert t.take_answer("evt-2") is None + assert t.take_answer("evt-1") is not None def test_a_direct_transport_has_no_edge_and_says_nothing_about_one(monkeypatch): @@ -462,3 +576,97 @@ def test_a_source_is_not_enough_if_the_decision_is_nonsense(): rt = NullRunRuntime.__new__(NullRunRuntime) with pytest.raises(NullRunMalformedGateResponseError): rt._require_gate_decision({"decision": "probably", "decision_source": EDGE_LEASE}) + + +def test_the_runtime_accepts_the_edge_refusal_sources(): + """SDK-3. A refusal the runtime cannot parse is a refusal the + runtime discards. + + `_require_gate_decision` rejects any body whose `decision_source` + it does not know, and it runs BEFORE the decision is read. So the + two edge refusal sources must be on the list — otherwise the + runtime raises "malformed response" on a perfectly well-formed + refusal, and the agent sees a crash instead of a block. + """ + from nullrun.runtime import NullRunRuntime + + rt = NullRunRuntime.__new__(NullRunRuntime) # no I/O, no client + for source in (EDGE_LEASE, EDGE_UNREACHABLE, EDGE_NOT_ENFORCED): + assert rt._require_gate_decision( + {"decision": "block", "decision_source": source} + ) == "block" + + +# ── the refusal reaches the caller ──────────────────────────────────── + + +def _edge_runtime(monkeypatch): + """A runtime whose enforcement path is a box that cannot answer.""" + monkeypatch.setenv("NULLRUN_EDGE_URL", "http://127.0.0.1:18090") + monkeypatch.setenv("NULLRUN_EDGE_LEASE_ID", "lease-1") + monkeypatch.setenv("NULLRUN_EDGE_TOKEN", "box-token") + monkeypatch.setenv("NULLRUN_API_KEY", "nr_org_key") + + from nullrun.runtime import NullRunRuntime + + return NullRunRuntime( + api_key="nr_org_key", + api_url="https://api.nullrun.io", + _test_mode=True, + polling=False, + ) + + +def test_an_unreachable_box_blocks_the_budget_preflight(monkeypatch): + """SDK-3. The refusal must reach the caller. + + Pre-fix the runtime's fail-OPEN predicate read the edge's + unreachable answer as a fallback source, so the pre-flight returned + "no decision" — and the caller reads that as "no block". The agent + carried on with the tool call unenforced. + + Here the block must surface as a raised exception. `None` is the + shape that means "nothing was decided", and the whole point is that + something WAS decided: by nobody, which is a refusal. + """ + from nullrun.breaker.exceptions import NullRunBlockedException + from nullrun.context import workflow + + rt = _edge_runtime(monkeypatch) + + def boom(body): + raise OSError("connection refused") + + rt._transport.edge._post = boom # type: ignore[method-assign] + + with workflow("wf-edge"): + with pytest.raises(NullRunBlockedException) as excinfo: + rt.check_workflow_budget() + + assert excinfo.value.error_code == "EDGE_UNREACHABLE", ( + "the refusal must name the edge, not the budget" + ) + + +def test_an_unreachable_box_is_not_reported_as_a_budget_error(monkeypatch): + """The refusal must not be laundered into "your budget ran out". + + An operator reading a budget error goes and raises a cap. The box + being unreachable is an infrastructure fact, and the error code + says so. + """ + from nullrun.breaker.exceptions import NullRunBudgetError + from nullrun.context import workflow + + rt = _edge_runtime(monkeypatch) + + def boom(body): + raise OSError("connection refused") + + rt._transport.edge._post = boom # type: ignore[method-assign] + + with workflow("wf-edge"): + with pytest.raises(Exception) as excinfo: + rt.check_workflow_budget() + + assert not isinstance(excinfo.value, NullRunBudgetError) diff --git a/tests/test_error_code_catalog.py b/tests/test_error_code_catalog.py new file mode 100644 index 0000000..8b2bf22 --- /dev/null +++ b/tests/test_error_code_catalog.py @@ -0,0 +1,211 @@ +"""The wire-code catalog must name a class the dispatcher can build. + +Audit 2026-11-03, **SDK-15**: the SDK's v3 error-code catalog lagged +the backend's registry by thirty-four codes. The parity guard in +``backend/tests/nr007_sdk_error_code_parity.rs`` could not see the +drift because it was wired into no workflow. + +A catalog entry is a PROMISE: "when the backend sends this code, the +caller gets this class". Two ways the promise breaks: + +1. **The code is missing.** The dispatcher falls back to the HTTP + status, so a money refusal (402) arrives as a generic backend error + and a refusal (403) arrives as an authentication error. The caller + cannot branch on what happened. +2. **The code is present but the dispatcher cannot build its class.** + The typed classes have heterogeneous constructors, so a class whose + constructor the dispatcher does not know raises ``TypeError`` inside + the parser. The ``TypeError`` escapes into the catch-all in + ``Transport.execute``, and the caller receives a synthetic + ``{"decision": "block"}`` dict instead of the typed refusal. The + refusal is not lost — it is flattened into "the gateway said no", + which is exactly the information the caller needed. + +The second failure mode is what the SDK-15 dispatch fix closed: the +blocked-family arm tested ``catalog is NullRunBlockedException`` by +IDENTITY, so every typed SUBCLASS of the blocked family (the approval +family, the MCP family) fell through to the generic fallback and +raised ``TypeError``. + +The property under test, over the whole catalog: + + for every code in the catalog, the real dispatcher returns an + instance of the catalog's class — for every status, because the + code names the refusal and the status is only its HTTP rendering. + +That last clause is the anti-vacuity canary. A check that only +asserted "the catalog is non-empty" would pass against a dispatcher +that returned the same class for everything. A check that only +asserted "an exception came back" would pass against the synthetic +block. This one asserts the exact class. +""" + +from __future__ import annotations + +import httpx +import pytest + +from nullrun.breaker.exceptions import ( + NullRunApprovalDbUnavailableError, + NullRunMcpApprovalRequiredError, +) +from nullrun.transport import _V3_ERROR_CODE_MAP, _parse_v3_error_envelope + +# Every status the backend's own registry uses +# (``GateErrorCode::http_status``). The class a wire code maps to must +# NOT depend on the status: a code names the refusal, the status is +# the HTTP rendering of it. If a status ever changed the class, the +# caller would have to know the status to know the class — which is +# the status-based fallback the catalog exists to replace. +STATUSES = (402, 403, 404, 409, 413, 422, 429, 500, 503) + +# The thirty-four codes SDK-15 added. Pinned by name so a later +# removal of any one of them is a RED test, not a silent regression. +# The cross-repo guard +# (``backend/tests/nr007_sdk_error_code_parity.rs``) would catch it +# too, but that guard is not wired into any workflow yet — a guard +# that does not run is not a guard. +SDK_15_CODES = ( + "APPROVAL_TOOL_ARGUMENT_MISMATCH", + "APPROVAL_TOOL_ARGUMENT_UNAVAILABLE", + "AUTH_BACKEND_UNAVAILABLE", + "BATCH_TOO_LARGE", + "BUDGET_EXCEEDED", + "BUDGET_PROJECTION_UNAVAILABLE", + "CHAIN_ENDED", + "CHAIN_NOT_FOUND", + "CONTROL_CHARS_IN_FIELD", + "EXECUTION_BINDING_MISSING", + "EXECUTION_BINDING_READ_FAILED", + "EXECUTION_BINDING_WRITE_FAILED", + "EXECUTION_NOT_BOUND", + "EXECUTION_RESOLVE_REDIS_ERROR", + "IDEMPOTENCY_CONFLICT", + "IDEMPOTENCY_IN_FLIGHT", + "INGESTION_BACKPRESSURE", + "INGESTION_CLOSED", + "INSUFFICIENT_SCOPE", + "MCP_DEFINITION_DRIFTED", + "MCP_REGISTRY_UNAVAILABLE", + "MCP_TOOL_APPROVAL_REQUIRED", + "MCP_TOOL_BLOCKED", + "MCP_TOOL_DECISION_REQUIRED", + "MCP_TOOL_REMOVED", + "ORG_CONTEXT_UNRESOLVED", + "RATE_LIMITER_UNAVAILABLE", + "RATE_LIMIT_PLAN_TIER", + "RESERVATION_NOT_FOUND", + "RESERVED_SCANNED_PARTIAL", + "RESERVE_DISABLED", + "TOOL_BLOCKED_BACKSTOP_UNAVAILABLE", + "WAITLIST_RATE_LIMITED", + "WORKFLOW_BUDGET_CONFIG_INVALID", +) + + +def _response(code: str, status: int) -> httpx.Response: + """A refusal in the shape the gate actually sends. + + The v3 envelope is ``{"error_code", "error_message", "details"}`` + and the gate's ``details`` block carries its own ``error_code`` + (see the refusal construction in ``gate/orchestrator.rs``). The + test reproduces that, because a ``details`` block WITHOUT an + ``error_code`` would skip the forwarding path and make the test + pass for the wrong reason. + """ + return httpx.Response( + status, + json={ + "error_code": code, + "error_message": "boom", + "details": {"error_code": code}, + }, + ) + + +def test_the_thirty_four_sdk_15_codes_are_catalogued(): + """Every code SDK-15 added is still in the catalog.""" + missing = [code for code in SDK_15_CODES if code not in _V3_ERROR_CODE_MAP] + assert not missing, f"SDK-15 codes dropped from the catalog: {missing}" + + +@pytest.mark.parametrize("code", sorted(_V3_ERROR_CODE_MAP)) +def test_every_catalog_code_dispatches_to_its_mapped_class(code: str): + """The catalog's promise holds: code in, that class out. + + A ``TypeError`` from a mismatched constructor fails this test as an + ERROR, which is the point: the TypeError is the defect the + dispatch fix closed, and it used to be swallowed further down by + the transport's catch-all. + """ + expected = _V3_ERROR_CODE_MAP[code] + for status in STATUSES: + exc = _parse_v3_error_envelope(_response(code, status), "/api/v1/gate") + assert isinstance(exc, expected), ( + f"{code} (status {status}) dispatched to " + f"{type(exc).__name__}, catalog says {expected.__name__}" + ) + + +@pytest.mark.parametrize("code", sorted(_V3_ERROR_CODE_MAP)) +def test_every_catalog_code_keeps_its_wire_code_on_the_exception(code: str): + """The wire code survives on the exception. + + ``exc.wire_error_code`` is the granular string the backend sent. + It is the field a cookbook branches on when it needs the exact + refusal rather than the class. + """ + exc = _parse_v3_error_envelope(_response(code, 403), "/api/v1/gate") + assert getattr(exc, "wire_error_code", None) == code + + +def test_a_typed_subclass_of_the_blocked_family_is_not_flattened(): + """The specific defect SDK-15's dispatch fix closed. + + ``APPROVAL_DB_UNAVAILABLE`` maps to + ``NullRunApprovalDbUnavailableError``, a SUBCLASS of + ``NullRunBlockedException``. The blocked-family arm used to test + the class by identity, so the subclass fell through to the generic + fallback, which called ``catalog(full_message, ...)`` against a + constructor that requires ``workflow_id`` — a ``TypeError`` that + the transport's catch-all turned into a synthetic block. + + The canary is the sibling class in the same family: if the arm + matched nothing at all, both would be flattened and this test + would still pass on the first assertion alone. The second + assertion pins that the two classes are DISTINCT — so the first + one cannot pass by accident. + """ + exc = _parse_v3_error_envelope(_response("APPROVAL_DB_UNAVAILABLE", 503), "/api/v1/gate") + assert isinstance(exc, NullRunApprovalDbUnavailableError) + + sibling = _parse_v3_error_envelope(_response("MCP_APPROVAL_REQUIRED", 403), "/api/v1/gate") + assert isinstance(sibling, NullRunMcpApprovalRequiredError) + assert not isinstance(sibling, NullRunApprovalDbUnavailableError) + + +def test_the_plan_tier_rate_limit_carries_the_retry_and_the_upgrade_url(): + """A 429 that is NOT ``RATE_LIMIT_EXCEEDED`` still builds a + ``RateLimitError``. + + ``RateLimitError`` has no message-only constructor: it requires + ``source`` and ``endpoint``. A catalog entry for a 429 code that + the dispatcher routes to the generic fallback therefore raises + ``TypeError`` instead of building the class — the same + "the catalog promises a class the dispatcher cannot build" defect, + one family over. + """ + response = httpx.Response( + 429, + json={ + "error_code": "RATE_LIMIT_PLAN_TIER", + "error_message": "plan aggregate exceeded", + "details": {"error_code": "RATE_LIMIT_PLAN_TIER"}, + "retry_after_ms": 2000, + "upgrade_url": "https://app.nullrun.io/settings/billing", + }, + ) + exc = _parse_v3_error_envelope(response, "/api/v1/gate") + assert isinstance(exc, _V3_ERROR_CODE_MAP["RATE_LIMIT_PLAN_TIER"]) + assert exc.retry_after == 2.0 + assert exc.upgrade_url == "https://app.nullrun.io/settings/billing" diff --git a/tests/test_error_envelope.py b/tests/test_error_envelope.py index b76ee4e..71ee53e 100644 --- a/tests/test_error_envelope.py +++ b/tests/test_error_envelope.py @@ -2,11 +2,10 @@ tests/test_error_envelope.py. Verifies ``_parse_error_envelope`` maps 4xx / 5xx / 429 to the -right exception subclass per the canonical ``contracts/errors.ts`` +right exception subclass per the control-plane ``ApiErrorResponse`` envelope. Reference: - contracts/errors.ts:1-39 backend/src/proxy/http/errors.rs:1-85 """ diff --git a/tests/test_exception_hierarchy.py b/tests/test_exception_hierarchy.py index 6268393..ea42e7c 100644 --- a/tests/test_exception_hierarchy.py +++ b/tests/test_exception_hierarchy.py @@ -38,10 +38,13 @@ NullRunBudgetError, # Config / auth NullRunConfigError, + NullRunDecision, NullRunError, + NullRunGateIndeterminateError, NullRunToolBlockedError, # Transport NullRunTransportError, + NullRunUnknownActionError, RateLimitError, TransportErrorSource, WorkflowKilledInterrupt, @@ -237,6 +240,69 @@ def test_rate_limit_is_NR_R001(self): assert info.value.error_code == "NR-R001" assert info.value.retryable is True + def test_indeterminate_gate_is_NR_P004(self): + """SDK-11 — the refusal for a decision the SDK could not obtain. + + The code is the operator's only handle on this class, so it is + pinned like every other one. ``retryable`` is True: the fault + is the SDK's own and may be transient. + """ + with pytest.raises(NullRunGateIndeterminateError) as info: + raise NullRunGateIndeterminateError("no decision") + assert info.value.error_code == "NR-P004" + assert info.value.retryable is True + + def test_indeterminate_gate_is_not_a_transport_error(self): + """ADR-126's two-step opt-in opens TRANSPORT errors. + + An indeterminate decision must not be convertible into "the + body runs" by flipping a switch, so it must not be a + transport error. + """ + assert not issubclass( + NullRunGateIndeterminateError, NullRunTransportError + ), ( + "the indeterminate refusal must not be a transport error — " + "that is the one class ADR-126's opt-in can open" + ) + + def test_unknown_action_is_NR_P005(self): + """SDK-12 — the refusal for an action the SDK cannot execute. + + ``retryable`` is False: the same unknown action on the same SDK + version fails identically. The fix is a version alignment, not + a retry. + """ + with pytest.raises(NullRunUnknownActionError) as info: + raise NullRunUnknownActionError("unknown action type") + assert info.value.error_code == "NR-P005" + assert info.value.retryable is False + + def test_unknown_action_is_not_a_transport_error(self): + """The same rule as the indeterminate refusal (ADR-126). + + "The body runs" is not an acceptable answer to "the control + plane sent an instruction this SDK cannot execute". + """ + assert not issubclass( + NullRunUnknownActionError, NullRunTransportError + ), ( + "the unknown-action refusal must not be a transport error — " + "that is the one class ADR-126's opt-in can open" + ) + + def test_unknown_action_is_not_a_decision(self): + """It is an infrastructure error, NOT a policy decision. + + A decision would mean "the policy engine refused this"; an + unknown action type means "the control plane sent something + this SDK cannot execute". The workflow is NOT blocked. + """ + assert not issubclass(NullRunUnknownActionError, NullRunDecision), ( + "an unknown action type must not be a decision — a decision " + "reads as a policy verdict and as a block" + ) + # --------------------------------------------------------------------------- # 5. Transport-error → code mapping diff --git a/tests/test_fail_open_class.py b/tests/test_fail_open_class.py new file mode 100644 index 0000000..571671b --- /dev/null +++ b/tests/test_fail_open_class.py @@ -0,0 +1,292 @@ +"""tests/test_fail_open_class.py — the fail-open switch is bounded per tool (SDK-8). + +The audit's finding, verbatim +(``RELIABILITY_AUDIT_2026-11-02_EVIDENCE.md`` §SDK-8): + + **SDK-8 — `sensitive_fail_open_enabled` is one global boolean.** + *Location:* `src/nullrun/runtime.py` ≈L2159–2236. *Mechanism:* a + single switch decides whether "sensitive" operations fail open, with + no classification of which operation is sensitive. *Failure:* + enabling it for one tool disables enforcement for **every** tool. + *Verify:* enable it and check a non-sensitive tool — it must still + fail closed. *Fix:* classify per tool; the switch is a kill-switch + for the whole system, not a per-tool option. + +What the SDK actually had +------------------------- + +The switch was read in exactly ONE place — ``decorators.py``'s +``_run_tool_policy_gate``, on the local ``@protect`` path — and it was +read as a bare boolean: ``fail_open = runtime.sensitive_fail_open_enabled()``. +Turning it on therefore opened EVERY protected tool, whatever the tool +was. The MCP path never consulted it (it fails CLOSED through +``fallback_mode=STRICT``), so the switch's reach was "every ``@protect`` +tool in the process". + +The SDK *did* have a per-tool registry — ``is_sensitive_tool`` / +``add_sensitive_tool`` / ``register_sensitive_tools`` — but it answers a +DIFFERENT question: "is this tool destructive?". Its built-in list holds +25 names, four of which are plain Python identifiers +(``send_email``, ``send_sms``, ``send_slack``, ``send_discord``) and 21 +of which are dotted MCP-style names that can never match a local +function's ``__name__``. Reusing it as the fail-open reach would have +made the SDK's own list of destructive operations an allowlist for +running without a policy decision. + +The design (ADR-126) +--------------------- + +A NEW, explicitly-named classification bounds the switch: + +* ``register_fail_open_tools(names)`` / ``remove_fail_open_tool(name)`` + / ``get_fail_open_tools()`` / ``is_fail_open_allowed(name)``; +* the set is EMPTY by default, so the switch alone opens NOTHING; +* the classification is CODE-level on purpose — it cannot be inherited + by a deploy pipeline or a CI environment the way an env var can. + +The switch alone now satisfies the audit's verify clause in its +strongest form: with it ON, a tool that is not classified still fails +CLOSED. + +The doubles here are shallow — a mock MCP client and the real runtime +with the real transport — so a change that removes the classification +bound turns these red rather than being stubbed around. +""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from nullrun.toolbox.mcp import MCPAdapter + +BASE_URL = "https://api.test.nullrun.io" +EXECUTE_URL = f"{BASE_URL}/api/v1/execute" +GATE_URL = f"{BASE_URL}/api/v1/gate" + +_FLAG = "NULLRUN_SENSITIVE_FAIL_OPEN" +_ACK = "NULLRUN_ALLOW_SENSITIVE_FAIL_OPEN" + + +@pytest.fixture(autouse=True) +def _clean_flag_env(monkeypatch): + """Both vars start unset, so a developer's shell cannot decide + whether these tests pass.""" + monkeypatch.delenv(_FLAG, raising=False) + monkeypatch.delenv(_ACK, raising=False) + monkeypatch.delenv("NULLRUN_ENV", raising=False) + monkeypatch.delenv("NULLRUN_API_URL", raising=False) + + +# --------------------------------------------------------------------------- +# The classification itself +# --------------------------------------------------------------------------- + + +class TestTheClassificationIsEmptyByDefault: + def test_a_fresh_runtime_classifies_nothing(self, make_runtime): + rt = make_runtime(api_url=BASE_URL) + assert rt.get_fail_open_tools() == set() + assert rt.is_fail_open_allowed("charge_card") is False + + def test_the_lookup_does_not_read_the_environment(self, make_runtime, monkeypatch): + """The switch being ON is not a classification. Two separate + things, and the audit's finding was that they were one.""" + rt = make_runtime(api_url=BASE_URL) + monkeypatch.setenv(_FLAG, "1") + assert rt.is_fail_open_allowed("charge_card") is False + + def test_the_builtin_sensitive_list_is_not_a_fail_open_allowlist(self, make_runtime): + """The sensitivity registry answers "is this destructive?"; the + fail-open registry answers "may this run unchecked?". Merging + them would put the SDK's own destructive-tool list into the + allowlist for running without a policy decision.""" + rt = make_runtime(api_url=BASE_URL) + + assert rt.is_sensitive_tool("send_email") is True, ( + "precondition: the built-in registry still knows the " + "destructive tools" + ) + assert rt.is_fail_open_allowed("send_email") is False, ( + "a destructive tool must never be fail-open-allowed by " + "default -- the two registries must stay separate" + ) + + +class TestRegisterAndRemove: + def test_registering_makes_the_tool_allowed(self, make_runtime): + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + assert rt.is_fail_open_allowed("charge_card") is True + assert rt.get_fail_open_tools() == {"charge_card"} + + def test_registration_is_case_insensitive(self, make_runtime): + """Same as the sensitivity registry: the tool's name comes from + ``fn.__name__`` and the operator may write it either way.""" + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["Charge_Card"]) + assert rt.is_fail_open_allowed("charge_card") is True + assert rt.is_fail_open_allowed("CHARGE_CARD") is True + + def test_removing_takes_the_tool_back_out(self, make_runtime): + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + rt.remove_fail_open_tool("charge_card") + assert rt.is_fail_open_allowed("charge_card") is False + assert rt.get_fail_open_tools() == set() + + def test_removing_an_unclassified_tool_is_a_no_op(self, make_runtime): + """``discard`` semantics: no KeyError on a tool that was never + registered.""" + rt = make_runtime(api_url=BASE_URL) + rt.remove_fail_open_tool("never_registered") + assert rt.get_fail_open_tools() == set() + + def test_the_returned_set_is_a_copy(self, make_runtime): + """A caller that mutates the returned set must not reach into + the runtime's state.""" + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + + snapshot = rt.get_fail_open_tools() + snapshot.add("smuggle_in") + + assert rt.is_fail_open_allowed("smuggle_in") is False + assert rt.get_fail_open_tools() == {"charge_card"} + + def test_the_classification_is_per_runtime(self, make_runtime): + """Two runtimes are two operators' decisions. A shared + class-level registry would leak one deployment's opt-out into + another's.""" + first = make_runtime(api_url=BASE_URL) + second = make_runtime(api_url=BASE_URL) + + first.register_fail_open_tools(["charge_card"]) + + assert first.is_fail_open_allowed("charge_card") is True + assert second.is_fail_open_allowed("charge_card") is False + + +# --------------------------------------------------------------------------- +# The switch + the classification, end to end on the @protect path +# --------------------------------------------------------------------------- + + +class TestTheTwoStepOptOut: + def test_the_classification_alone_does_not_open_the_body( + self, make_runtime, mock_api, monkeypatch + ): + """Classified, but the switch is OFF: the body must not run. + + The classification is the BOUND, not the grant. Without the + grant the tool stays fail-CLOSED even though the operator + named it.""" + from nullrun.breaker.exceptions import NullRunBlockedException + from nullrun.decorators import protect + + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + + respx.post(EXECUTE_URL).mock( + side_effect=httpx.ConnectError("connection refused") + ) + + ran: list[int] = [] + + @protect + def charge_card(amount: int) -> str: + ran.append(amount) + return "charged" + + with pytest.raises(NullRunBlockedException): + charge_card(100) + + assert ran == [], "the classification alone opened the tool" + + def test_switch_plus_classification_runs_the_body( + self, make_runtime, mock_api, monkeypatch + ): + """The documented dev/test bypass: both steps together.""" + from nullrun.decorators import protect + + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + monkeypatch.setenv(_FLAG, "1") + + respx.post(EXECUTE_URL).mock( + side_effect=httpx.ConnectError("connection refused") + ) + + ran: list[int] = [] + + @protect + def charge_card(amount: int) -> str: + ran.append(amount) + return "charged" + + charge_card(100) + + assert ran == [100] + + +# --------------------------------------------------------------------------- +# The MCP path +# --------------------------------------------------------------------------- + + +class _Ann: + def __init__(self, read=None, destructive=None, open_world=None): + self.readOnlyHint = read + self.destructiveHint = destructive + self.openWorldHint = open_world + + +class _Tool: + def __init__(self, name, annotations=None): + self.name = name + self.annotations = annotations + + +class _MockMcpClient: + def __init__(self, tools): + self._tools = {t.name: t for t in tools} + self.calls: list[tuple[str, dict]] = [] + + def list_tools(self): + return list(self._tools.values()) + + def call_tool(self, name, arguments=None, **kwargs): + self.calls.append((name, arguments or {})) + return f"ok:{name}" + + +class TestTheMcpPathIsNotOpened: + """The switch is a LOCAL-path switch. The MCP path fails CLOSED + through ``fallback_mode=STRICT`` and never consults it — so the + switch must not open the MCP path either, however the operator + configures it.""" + + def test_the_mcp_client_is_not_called_when_the_engine_is_unreachable( + self, make_runtime, mock_api, monkeypatch + ): + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["create_issue"]) + monkeypatch.setenv(_FLAG, "1") + + # The pre-flight /gate answers (the conftest's mock); the + # /execute the gate that decides the call is unreachable. + respx.post(EXECUTE_URL).mock( + side_effect=httpx.ConnectError("connection refused") + ) + + client = _MockMcpClient([_Tool("create_issue", _Ann(destructive=True))]) + adapter = MCPAdapter(server_name="github", mcp_client=client, runtime=rt) + + with pytest.raises(Exception): + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert client.calls == [], ( + "the fail-open switch opened the MCP path -- the switch is " + "consulted only on the local @protect path" + ) diff --git a/tests/test_langchain_enforcement.py b/tests/test_langchain_enforcement.py index b825493..d763b5d 100644 --- a/tests/test_langchain_enforcement.py +++ b/tests/test_langchain_enforcement.py @@ -29,28 +29,126 @@ ``@protect`` boundary — which can abort — raises it. This module pins all three links in that chain: the swallow behaviour, the stash, and the raise. + +SDK-14 (reliability audit 2026-11-02) +------------------------------------- +This file used to define its OWN copy of the enforcement family:: + + ENFORCEMENT = (NullRunBudgetError, WorkflowKilledInterrupt, WorkflowPausedException) + +and assert ``issubclass(case, production_family)`` — one direction +only. Two consequences, both measured: + +* **Widening the production family was invisible.** A refusal class + added to ``ENFORCEMENT_EXCEPTIONS`` and then mishandled by the + callback (dropped into the broad fail-OPEN arm) changed nothing in + this suite: the three hand-written cases were still subclasses of + the family, so every assertion still held. The family is the + mechanism that keeps a refusal from being laundered into "the gate + was unreachable, carry on" — and a member nobody drives is a promise + nobody checks. +* **The copy could rot silently.** Nothing tied the three named + classes to the refusals the SDK actually raises. + +The family now lives in ONE place — the production tuple — and this +file holds only what cannot be derived: the CONSTRUCTORS (every +exception class has its own signature) and the correspondence +INVARIANTS that keep the two in step. A drift in EITHER direction is +RED, never silent. """ from __future__ import annotations +from collections.abc import Callable + import pytest +from nullrun.breaker.categories import NullRunUnclassifiedRefusalError from nullrun.breaker.exceptions import ( + ENFORCEMENT_EXCEPTIONS, + NullRunAuthenticationError, + NullRunBackendError, NullRunBudgetError, + NullRunDecision, + NullRunGateIndeterminateError, + NullRunInfrastructureError, + NullRunMalformedGateResponseError, + NullRunRateLimitRedisError, + NullRunToolBlockedError, + RateLimitError, + TransportErrorSource, WorkflowKilledInterrupt, WorkflowPausedException, ) from nullrun.instrumentation.langgraph import ( _ENFORCEMENT_EXCEPTIONS, + _INFRASTRUCTURE_ERRORS, + _TRANSPORT_ERRORS, + NullRunCallback, drain_deferred_enforcement, record_deferred_enforcement, ) -ENFORCEMENT = ( - NullRunBudgetError, - WorkflowKilledInterrupt, - WorkflowPausedException, -) +# SDK-14: the CONSTRUCTORS, not the predicate. +# +# A case needs an INSTANCE, and every exception class has its own +# signature — so the list of what to drive cannot be derived from the +# production tuple mechanically. What CAN be derived is the PREDICATE, +# and it is not written here: the production tuple +# (`ENFORCEMENT_EXCEPTIONS`) is the only copy of it, and the +# correspondence invariants in `TestTheFamilyAndTheCasesCorrespond` +# are what keep this registry and that tuple in step. A drift in +# EITHER direction is red. +_CASE_BUILDERS: dict[type[BaseException], Callable[[], BaseException]] = { + # -- the production family's own members --------------------------- + NullRunDecision: lambda: NullRunDecision("the gate said no"), + RateLimitError: lambda: RateLimitError( + "rate limit exceeded", + TransportErrorSource.GATEWAY_ERROR, + "gate", + ), + WorkflowKilledInterrupt: lambda: WorkflowKilledInterrupt( + workflow_id="w-sdk14", reason="killed by the dashboard" + ), + NullRunMalformedGateResponseError: lambda: NullRunMalformedGateResponseError( + "the gate answered with something that is not a decision" + ), + NullRunGateIndeterminateError: lambda: NullRunGateIndeterminateError( + "the SDK could not determine a decision" + ), + # -- the concrete refusals the gate actually raises ---------------- + NullRunBudgetError: lambda: NullRunBudgetError( + workflow_id="w-sdk14", reason="budget exhausted" + ), + WorkflowPausedException: lambda: WorkflowPausedException( + workflow_id="w-sdk14", reason="paused by the operator" + ), + NullRunToolBlockedError: lambda: NullRunToolBlockedError( + workflow_id="w-sdk14", reason="tool is on the block list", tool_name="shell" + ), + # -- SDK-16: the classes the RUNTIME raises on purpose and the + # enforcement FAMILY does not name ----------------------------- + # A 401 is a credential failure, not a transport condition: no + # retry can fix a revoked key. The SDK's own table says + # fail-CLOSED for it, and the @protect boundary obeyed that while + # the callback swallowed it. + NullRunAuthenticationError: lambda: NullRunAuthenticationError( + "the backend rejected the key (401)" + ), + # The gate ANSWERED and refused, but without a category the SDK + # trusts. ADR-062 §2.2: an unclassifiable refusal must RAISE. + NullRunUnclassifiedRefusalError: lambda: NullRunUnclassifiedRefusalError( + "the gate refused without a category the SDK understands", + wire_category=None, + error_code_wire="NR-UNKNOWN", + ), + # The rate-limit backend is down. The SDK's table calls this a + # named enforcement failure and says the SDK does NOT silently + # fail-OPEN on it. + NullRunRateLimitRedisError: lambda: NullRunRateLimitRedisError( + "the backend cannot reach Redis for the aggregate rate limit (503)" + ), +} @pytest.fixture(autouse=True) @@ -65,10 +163,15 @@ class TestFrameworkContract: def test_langchain_swallows_callback_exceptions(self): """The premise of the whole fix, pinned against real langchain. - If a future langchain-core propagates callback exceptions, the - handoff becomes unnecessary (though still correct) — but the - test must fail loudly so someone re-evaluates the design rather - than the assumption silently rotting. + A DEFAULT handler's exception is swallowed: langchain-core logs + "Error in callback" and continues. That is why the + NullRun handler has to set `raise_error = True` explicitly + (SDK-4) — the swallow is the default, not the exception. + + If a future langchain-core propagates callback exceptions even + without the flag, the handoff becomes unnecessary (though still + correct) — but the test must fail loudly so someone re-evaluates + the design rather than the assumption silently rotting. """ pytest.importorskip("langchain_core") from langchain_core.callbacks import BaseCallbackHandler @@ -91,6 +194,26 @@ def on_llm_start(self, serialized, prompts, **kwargs): ) +def _fails_closed(case: type[BaseException]) -> bool: + """Whether the CALLBACK must re-raise this class. + + The production predicate, read from the instrumentation's own + constants — never spelled out here (SDK-14 removed the copy of the + family for exactly that reason). It models the ORDER of the + callback's arms, and that order is pinned separately in the AST + (`TestEnforcementClassification`): + + 1. the family -> re-raised + 2. a transport failure -> swallowed (ADR-008's fail-OPEN) + 3. other infra errors -> re-raised (fail-CLOSED) + """ + if any(issubclass(case, member) for member in _ENFORCEMENT_EXCEPTIONS): + return True + if any(issubclass(case, member) for member in _TRANSPORT_ERRORS): + return False + return any(issubclass(case, member) for member in _INFRASTRUCTURE_ERRORS) + + def _excepted_types(method) -> list[list[str]]: """Every `except` clause in `method`, in source order, as names. @@ -119,21 +242,286 @@ def _excepted_types(method) -> list[list[str]]: return found -class TestEnforcementClassification: - @pytest.mark.parametrize("exc_cls", ENFORCEMENT) - def test_real_decisions_are_classified_as_enforcement(self, exc_cls): - assert exc_cls in _ENFORCEMENT_EXCEPTIONS, ( - f"{exc_cls.__name__} is a real gate decision and must be caught " - "by the enforcement arm BEFORE the broad transport arm. If it " - "is not, an exhausted budget / KILL becomes a fail-open " - "transport error." +class TestTheFamilyAndTheCasesCorrespond: + """SDK-14: the suite must assert the PRODUCTION predicate. + + The defect this file used to carry was a COPY of the predicate. + These tests are what make the copy impossible to keep: they + compare the production family with the cases this suite actually + drives, in BOTH directions, and they pin that the instrumentation + uses the production tuple itself. + """ + + def test_every_family_member_has_a_case(self): + missing = [ + member.__name__ + for member in ENFORCEMENT_EXCEPTIONS + if member not in _CASE_BUILDERS + ] + assert not missing, ( + "the production enforcement family has members this suite does " + f"not drive: {missing}. A family member is a promise that the " + "refusal REACHES the caller; a member nobody drives is a " + "promise nobody checks. Drive it here before the class ships " + "— or take the class out of the family with a decision." + ) + + def test_every_case_is_covered_by_the_family(self): + uncovered = [ + case.__name__ + for case in _CASE_BUILDERS + if not _fails_closed(case) + ] + assert not uncovered, ( + f"these cases are no longer covered by the production " + f"fail-CLOSED set: {uncovered}. The set was narrowed, or the " + "case is stale — either way the refusal these classes " + "represent would now fall into the broad fail-OPEN arm and " + "be swallowed, and the same refusal would kill the call " + "through the @protect path and vanish through the " + "callback path." + ) + + def test_every_family_member_is_an_exception(self): + """A BaseException-only member would escape the split entirely.""" + not_exceptions = [ + member.__name__ + for member in ENFORCEMENT_EXCEPTIONS + if not issubclass(member, Exception) + ] + assert not not_exceptions, ( + f"{not_exceptions} derive from BaseException only — the broad " + "arm catches Exception, so such a member would never be " + "classified at all." + ) + + def test_the_instrumentation_uses_the_production_tuple(self): + """Identity, not a copy. + + `instrumentation/langgraph.py` imports the production tuple and + falls back to an EMPTY tuple if that import fails. An empty + tuple means "no refusal is an enforcement refusal" — every + refusal is downgraded to a transport error and swallowed by the + broad arm. The alias must BE the production object, so the two + can never drift apart and the fallback can never be in effect. + """ + assert _ENFORCEMENT_EXCEPTIONS is ENFORCEMENT_EXCEPTIONS, ( + "the instrumentation's enforcement alias is not the production " + f"tuple — it is {_ENFORCEMENT_EXCEPTIONS!r}. An empty alias " + "means every refusal is treated as a transport error and " + "swallowed by the broad fail-OPEN arm." ) - assert issubclass(exc_cls, Exception), ( - f"{exc_cls.__name__} must subclass Exception — the transport arm " - "catches Exception, so a BaseException-only type would escape " - "the split entirely." + + +class _GateRefuses: + """A runtime whose budget pre-flight raises one fixed refusal.""" + + def __init__(self, exc: BaseException) -> None: + self.exc = exc + self.checks = 0 + + def check_workflow_budget(self, *a, **k): + self.checks += 1 + raise self.exc + + def track_event(self, *a, **k): + return None + + def track_tool(self, *a, **k): + return None + + +class TestEveryCaseIsDriven: + """SDK-14: a family member must be PROVEN, not asserted. + + `issubclass` says the class is named by the family. It does not say + the refusal REACHES the caller. That difference is the whole point + of the family: a member the callback mishandles is downgraded to a + fail-OPEN transport error, and the call proceeds with no + reservation — the agent gets the call for free. + + So every case is driven through the REAL callback, and the refusal + must (a) escape to the caller and (b) be recorded for the + ``@protect`` boundary. + """ + + @pytest.mark.parametrize( + "case", + list(_CASE_BUILDERS), + ids=[c.__name__ for c in _CASE_BUILDERS], + ) + def test_the_refusal_reaches_the_caller(self, case): + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + exc = _CASE_BUILDERS[case]() + cb = NullRunCallback(runtime=_GateRefuses(exc)) + + with pytest.raises(type(exc)) as caught: + CallbackManager([cb]).on_llm_start({}, ["hi"]) + + assert caught.value is exc, ( + f"{case.__name__}: the caller must receive the refusal itself, " + f"got {type(caught.value).__name__}" ) + @pytest.mark.parametrize( + "case", + list(_CASE_BUILDERS), + ids=[c.__name__ for c in _CASE_BUILDERS], + ) + def test_the_refusal_reaches_the_async_caller(self, case): + """The async path is a DIFFERENT code path in langchain-core.""" + import asyncio + + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import AsyncCallbackManager + + exc = _CASE_BUILDERS[case]() + cb = NullRunCallback(runtime=_GateRefuses(exc)) + + with pytest.raises(type(exc)) as caught: + asyncio.run(AsyncCallbackManager([cb]).on_llm_start({}, ["hi"])) + + assert type(caught.value) is type(exc), ( + f"{case.__name__}: the async caller must receive the gate " + f"refusal, got {type(caught.value).__name__}" + ) + + @pytest.mark.parametrize( + "case", + list(_CASE_BUILDERS), + ids=[c.__name__ for c in _CASE_BUILDERS], + ) + def test_the_refusal_is_recorded_for_the_protect_boundary(self, case): + """The secondary channel: a host framework that ignores + ``raise_error`` still gets the decision at the ``@protect`` + boundary, on the boundary's own thread. + """ + exc = _CASE_BUILDERS[case]() + cb = NullRunCallback(runtime=_GateRefuses(exc)) + + with pytest.raises(type(exc)): + # `run_id` is required: without it the callback returns early + # (it cannot open a span it would never close) and never + # reaches the budget pre-flight. + cb.on_llm_start({}, ["hi"], run_id="run-sdk14") + + assert drain_deferred_enforcement() is exc, ( + f"{case.__name__}: the refusal must also be recorded — it is " + "the only channel that can abort on the @protect boundary's " + "own thread" + ) + + def test_a_transport_failure_still_fails_open(self): + """ADR-008: a dead backend must NOT freeze the agent. + + The counter-test for the family: an exception the family does + NOT name is a transport failure, and fail-OPEN is the + documented policy for it. Without this, "widen the family" would + look like a free improvement — and the family would grow until + a dead backend froze every agent. + + SDK-16 sharpened this: a transport failure IS an + infrastructure error, so the callback's new infrastructure arm + would catch it and refuse — unless the transport arm comes + first. The ORDER is pinned in the AST + (`test_the_transport_arm_precedes_the_infrastructure_arm`); this + test pins the BEHAVIOUR it exists to protect. + """ + boom = NullRunBackendError("boom", endpoint="gate", status_code=500) + assert not isinstance(boom, ENFORCEMENT_EXCEPTIONS), ( + "a transport failure is not a decision — if this ever becomes " + "true the family's rule has changed and this control is no " + "longer a control" + ) + assert isinstance(boom, NullRunInfrastructureError), ( + "a transport failure IS an infrastructure error — that is why " + "the callback needs a transport arm of its own, listed " + "before the infrastructure one. If this ever becomes false " + "the arm order stops mattering and the pin below would be " + "asserting nothing." + ) + + cb = NullRunCallback(runtime=_GateRefuses(boom)) + cb.on_llm_start({}, ["hi"], run_id="run-sdk14") # must not raise + + assert drain_deferred_enforcement() is None, ( + "a transport failure is not a decision — it must not abort a " + "later @protect call either" + ) + + def test_a_cosmetic_failure_is_still_swallowed(self): + """The other half of the widening: a BUG must not abort the agent. + + ADR-122 §3 rejected "widen the family to every class" partly + because a cosmetic failure — a span bookkeeping bug, a broken + runtime double — must not break the user's agent. The callback + keeps that contract: a fault that is not a NullRun error is + logged at debug and swallowed. + + Without this test, "re-raise everything" would look like a free + improvement too, and a typo in the SDK would freeze every + agent that uses LangChain. + """ + cb = NullRunCallback(runtime=_GateRefuses(RuntimeError("a bug in the SDK"))) + cb.on_llm_start({}, ["hi"], run_id="run-sdk14") # must not raise + + assert drain_deferred_enforcement() is None, ( + "a cosmetic failure is not a decision — it must not abort a " + "later @protect call either" + ) + + +class TestTheTableTheCallbackObeys: + """SDK-16: the callback must obey the SDK's OWN fail-OPEN/CLOSED table. + + The table is the module docstring of `nullrun/runtime.py` — the + authoritative statement of what each gate does when it cannot be + reached (ADR-063 §2.4). Its row for `check_workflow_budget` says + the gate fails OPEN for TRANSPORT errors ONLY, and CLOSED for + everything else. + + This is a DRIFT tripwire, not the proof. The proof is the + behavioural tests above, which drive the real callback with the + real classes. The tripwire exists so that editing the table + without editing the callback is a RED test instead of a silent + contradiction — the contradiction SDK-16 was. + """ + + def test_the_table_still_says_closed_for_everything_else(self): + import nullrun.runtime as runtime_mod + + table = runtime_mod.__doc__ or "" + assert ( + "OPEN (skip check, log warning) for TRANSPORT errors ONLY" in table + ), ( + "the fail-OPEN/CLOSED table in nullrun/runtime.py no longer " + "states the check_workflow_budget row this suite enforces. " + "The table is authoritative (ADR-063 §2.4) — if the row " + "changed, the callback's posture must be re-decided, not " + "left to drift." + ) + + def test_the_table_still_names_the_four_classes(self): + import nullrun.runtime as runtime_mod + + table = runtime_mod.__doc__ or "" + for name in ( + "NullRunAuthError", + "NullRunMalformedGateResponseError", + "NullRunGateIndeterminateError", + "NullRunRateLimitRedisError", + ): + assert name in table, ( + f"{name} is no longer named in the SDK's fail-OPEN/CLOSED " + "table. The table is what the callback is measured " + "against, so a class that leaves the table leaves the " + "posture undecided." + ) + + +class TestEnforcementClassification: def test_enforcement_arm_precedes_transport_arm(self): """The except-ordering is load-bearing, so pin it in the AST. @@ -170,6 +558,46 @@ def test_no_bare_baseexception_catch_remains(self): f"decision (fail-OPEN is a bug). Arms: {arms}" ) + def test_the_transport_arm_precedes_the_infrastructure_arm(self): + """SDK-16: the ORDER is what keeps a dead backend fail-OPEN. + + `NullRunTransportError` is a `NullRunInfrastructureError`, so an + infrastructure arm listed BEFORE the transport arm would catch + every network failure and turn it into a refusal: a dead + backend would freeze the agent instead of letting it through. + The transport arm must come first, in both places that + classify an exception — the callback's check path and the + guard around every callback method. + """ + from nullrun.instrumentation.langgraph import ( + NullRunCallback, + _only_a_gate_refusal_escapes, + ) + + for where, arms in ( + ("on_llm_start", _excepted_types(NullRunCallback.on_llm_start)), + ( + "_only_a_gate_refusal_escapes", + _excepted_types(_only_a_gate_refusal_escapes), + ), + ): + assert ["_TRANSPORT_ERRORS"] in arms, ( + f"{where} must classify a transport failure explicitly; " + f"arms: {arms}" + ) + assert ["_INFRASTRUCTURE_ERRORS"] in arms, ( + f"{where} must re-raise a non-transport infrastructure " + f"error (SDK-16); arms: {arms}" + ) + assert arms.index(["_TRANSPORT_ERRORS"]) < arms.index( + ["_INFRASTRUCTURE_ERRORS"] + ), ( + f"{where}: the transport arm must come FIRST. " + "NullRunTransportError is a NullRunInfrastructureError, so " + "the other order turns every network failure into a " + f"refusal and freezes the agent. Arms: {arms}" + ) + class TestDeferredHandoff: def test_record_then_drain_roundtrip(self): @@ -274,6 +702,15 @@ def sensitive_fail_open_enabled(self, *a, **k): """ return False + def is_fail_open_allowed(self, *a, **k): + """The per-tool bound on the switch (ADR-126). + + The stub reports "not classified" — the same default as a fresh + runtime. With the flag OFF the decorator never asks, so this + only matters if the order of the two halves ever changes. + """ + return False + @pytest.fixture def stub_runtime(monkeypatch): diff --git a/tests/test_langchain_refusal_surfaces.py b/tests/test_langchain_refusal_surfaces.py new file mode 100644 index 0000000..1084989 --- /dev/null +++ b/tests/test_langchain_refusal_surfaces.py @@ -0,0 +1,299 @@ +"""tests/test_langchain_refusal_surfaces.py — a gate refusal must REACH +the caller, not vanish into a debug log line. + +SDK-4 (reliability audit 2026-11-02). + +The defect +---------- +``NullRunCallback.on_llm_start`` classified a gate refusal with a +three-class allow-list:: + + _ENFORCEMENT_EXCEPTIONS = ( + NullRunBudgetError, WorkflowKilledInterrupt, WorkflowPausedException, + ) + +Every OTHER refusal the gate can issue fell into the broad +``except Exception`` arm, was logged at DEBUG and DROPPED: + +* ``TOOL_BLOCKED`` -> ``NullRunToolBlockedError`` -> ``NullRunDecision`` +* ``RATE_LIMIT_EXCEEDED`` -> ``RateLimitError`` -> ``NullRunTransportError`` + +The agent therefore carried on after the gate had refused the call, and +the refusal was invisible to the application that is supposed to honour +it. + +The fix +------- +1. The refusal family is named by BASE class (``NullRunDecision``, + ``RateLimitError``, ``WorkflowKilledInterrupt``), so a refusal + subclass added later cannot fall back into the fail-OPEN arm. +2. The refusal is RE-RAISED out of the callback, and the handler sets + ``raise_error = True`` — langchain-core re-raises a handler's + exception in BOTH the sync and the async dispatch path when that + flag is set. Before the fix the refusal could not leave the + callback at all. +3. ``_only_a_gate_refusal_escapes`` guarantees that a gate refusal is + the ONLY exception that can leave a NullRun callback, so opting + into ``raise_error`` cannot break the host chain for a cosmetic + failure. + +WHY THIS FILE DRIVES A REAL CALLBACK MANAGER +------------------------------------------- +A source-pin on the module can prove that ``raise`` appears after the +enforcement arm. It cannot prove that the framework PROPAGATES what +the handler raises — that is langchain-core's contract, and the +contract is exactly what was silently inverted. So this file attaches +the real handler to a real ``CallbackManager`` / ``AsyncCallbackManager`` +and reads what the caller receives. + +The transport arm is checked too: a backend outage must still fail +OPEN (ADR-008). Opting into ``raise_error`` must not turn "the gate +could not be reached" into a hard failure. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from nullrun.breaker.exceptions import ( + NullRunBackendError, + NullRunBudgetError, + NullRunToolBlockedError, + RateLimitError, + TransportErrorSource, + WorkflowKilledInterrupt, +) +from nullrun.instrumentation.langgraph import ( + _ENFORCEMENT_EXCEPTIONS, + NullRunCallback, + drain_deferred_enforcement, +) + + +class _StubRuntime: + """A runtime whose budget pre-flight raises a fixed exception.""" + + def __init__(self, exc: BaseException | None) -> None: + self.exc = exc + self.checks = 0 + + def check_workflow_budget(self, *a, **k): + self.checks += 1 + if self.exc is not None: + raise self.exc + return None + + def track_event(self, *a, **k): + return None + + def track_tool(self, *a, **k): + return None + + +def _tool_blocked() -> NullRunToolBlockedError: + return NullRunToolBlockedError( + workflow_id="w-1", + reason="tool is on the block list", + tool_name="shell", + ) + + +def _rate_limited() -> RateLimitError: + return RateLimitError( + "rate limit exceeded", + source=TransportErrorSource.GATEWAY_ERROR, + endpoint="gate", + ) + + +# Every refusal the gate can issue on the /gate pre-flight, and the two +# classes the audit named explicitly. +REFUSALS = [ + ("TOOL_BLOCKED", _tool_blocked), + ("RATE_LIMIT", _rate_limited), + ("BUDGET", lambda: NullRunBudgetError(workflow_id="w-1", reason="budget exhausted")), + ("KILLED", lambda: WorkflowKilledInterrupt(workflow_id="w-1", reason="killed")), +] + + +@pytest.fixture(autouse=True) +def _clear_deferred(): + drain_deferred_enforcement() + yield + drain_deferred_enforcement() + + +def _callback(exc: BaseException | None) -> NullRunCallback: + return NullRunCallback(runtime=_StubRuntime(exc)) + + +class TestRefusalReachesTheCaller: + @pytest.mark.parametrize("name,make", REFUSALS, ids=[n for n, _ in REFUSALS]) + def test_sync_dispatch_propagates_the_refusal(self, name, make): + """The refusal must leave the callback, not die in a debug log.""" + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + cb = _callback(make()) + + with pytest.raises(BaseException) as caught: + CallbackManager([cb]).on_llm_start({}, ["hi"]) + + assert type(caught.value) is type(make()), ( + f"{name}: the caller must receive the gate refusal, got " + f"{type(caught.value).__name__}" + ) + + @pytest.mark.parametrize("name,make", REFUSALS, ids=[n for n, _ in REFUSALS]) + def test_async_dispatch_propagates_the_refusal(self, name, make): + """The async path is a DIFFERENT code path in langchain-core. + + The sync handler is run through an executor there, and the + exception is read back on the event-loop thread. Both were + measured: without ``raise_error`` the refusal was swallowed on + both. + """ + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import AsyncCallbackManager + + cb = _callback(make()) + + with pytest.raises(BaseException) as caught: + asyncio.run(AsyncCallbackManager([cb]).on_llm_start({}, ["hi"])) + + assert type(caught.value) is type(make()), ( + f"{name}: the async caller must receive the gate refusal, got " + f"{type(caught.value).__name__}" + ) + + def test_the_refusal_is_also_recorded_for_the_protect_boundary(self): + """The secondary channel is kept: a host framework that ignores + ``raise_error`` still gets the decision at the ``@protect`` + boundary. + """ + from nullrun.instrumentation.langgraph import drain_deferred_enforcement + + cb = _callback(_tool_blocked()) + with pytest.raises(NullRunToolBlockedError): + # `run_id` is required: without it the callback returns early + # (it cannot open a span it would never close) and never + # reaches the budget pre-flight. + cb.on_llm_start({}, ["hi"], run_id="run-1") + + deferred = drain_deferred_enforcement() + assert deferred is not None, ( + "the refusal must also be recorded — it is the only channel " + "that can abort on the @protect boundary's own thread" + ) + assert isinstance(deferred, NullRunToolBlockedError) + + +class TestTransportFailureStillFailsOpen: + """ADR-008: a dead backend must NOT freeze the agent. + + Opting into ``raise_error`` must not turn "the gate could not be + reached" into a hard failure — that would be a different bug, and + a worse one (an outage would take every agent down). + """ + + def test_a_transport_failure_does_not_escape(self): + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + cb = _callback(NullRunBackendError("boom", endpoint="gate", status_code=500)) + + CallbackManager([cb]).on_llm_start({}, ["hi"]) # must not raise + + def test_a_transport_failure_is_not_recorded_as_a_refusal(self): + """A dead backend is not a decision — it must not abort a later + ``@protect`` call either. + """ + cb = _callback(NullRunBackendError("boom", endpoint="gate", status_code=500)) + cb.on_llm_start({}, ["hi"]) + + assert drain_deferred_enforcement() is None + + +class TestOnlyARefusalMayEscape: + """``raise_error = True`` is only safe if nothing else can escape.""" + + def test_the_guard_swallows_a_cosmetic_failure(self): + """A broken span bookkeeping must not abort the user's agent.""" + from nullrun.instrumentation.langgraph import _only_a_gate_refusal_escapes + + class _Obj: + @_only_a_gate_refusal_escapes + def cosmetic(self): + raise RuntimeError("span emission is broken") + + assert _Obj().cosmetic() is None, ( + "a non-refusal failure must be swallowed — it is the reason " + "opting into raise_error is safe" + ) + + def test_the_guard_lets_a_refusal_through(self): + from nullrun.instrumentation.langgraph import _only_a_gate_refusal_escapes + + class _Obj: + @_only_a_gate_refusal_escapes + def refusal(self): + raise _tool_blocked() + + with pytest.raises(NullRunToolBlockedError): + _Obj().refusal() + + def test_a_broken_backend_does_not_break_the_chain(self): + """End to end: a runtime whose event emission fails.""" + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + class _BrokenRuntime(_StubRuntime): + def track_event(self, *a, **k): + raise RuntimeError("span emission is broken") + + cb = NullRunCallback(runtime=_BrokenRuntime(None)) + + CallbackManager([cb]).on_llm_start({}, ["hi"]) # must not raise + + def test_the_handler_opts_into_propagation(self): + assert NullRunCallback.raise_error is True, ( + "langchain-core swallows a callback's exception unless the " + "handler sets raise_error — without it the refusal never " + "reaches the caller" + ) + + +class TestTheRefusalFamilyIsNamedByBaseClass: + """The family must not rot back into a three-class list.""" + + def test_every_refusal_class_is_covered(self): + for name, make in REFUSALS: + exc = make() + assert isinstance(exc, _ENFORCEMENT_EXCEPTIONS), ( + f"{name}: a real gate refusal must be caught by the " + "enforcement arm BEFORE the broad transport arm, or it " + "is downgraded to a fail-OPEN transport error" + ) + + def test_a_transport_failure_is_not_covered(self): + """The gate could not be reached -> ADR-008 fail-OPEN.""" + assert not isinstance( + NullRunBackendError("boom", endpoint="gate", status_code=500), + _ENFORCEMENT_EXCEPTIONS, + ) + + def test_the_rate_limit_refusal_is_covered(self): + """`RateLimitError` is a TRANSPORT-shaped class, not a decision. + + Naming only `NullRunDecision` would still have swallowed the + 429 refusal — the shape is what makes this easy to get wrong. + """ + assert not issubclass(RateLimitError, __import__( + "nullrun.breaker.exceptions", fromlist=["NullRunDecision"] + ).NullRunDecision), ( + "if this ever becomes true, the reason RateLimitError is " + "named explicitly in _ENFORCEMENT_EXCEPTIONS has changed" + ) + assert isinstance(_rate_limited(), _ENFORCEMENT_EXCEPTIONS) diff --git a/tests/test_mcp_action_binding.py b/tests/test_mcp_action_binding.py new file mode 100644 index 0000000..dc50190 --- /dev/null +++ b/tests/test_mcp_action_binding.py @@ -0,0 +1,509 @@ +"""tests/test_mcp_action_binding.py — SDK-7, the MCP approval binding. + +The MCP path is the SDK's SECOND enforcement path (the first is the +``@protect``-decorated local function). Before ADR-125 the adapter +sent NO ``business_impact`` and NO ``action_digest`` on ``/execute``, +and the backend refused the post-approval re-entry outright: + +* ``/execute`` with an ``approval_id`` runs Step 9.5, which + SERVER-DERIVES the action digest from the live + ``gate_req.business_impact`` + (``backend/src/proxy/http/gate/payload_binding.rs:163``). With no + envelope the derive returns ``Ok(None)`` and the orchestrator fails + CLOSED with ``BUSINESS_IMPACT_INVALID`` (HTTP 400, NR-010) — + ``backend/src/proxy/http/gate/orchestrator.rs:2325``. So an + operator approval for an MCP tool could NEVER be spent: the row was + created, the operator approved it, and the re-execute was refused. +* Where the digest arm did run, ``server_validate_tool_argument_binding`` + compares ``BusinessImpact::ToolCall.params`` with the live + ``gate_req.tool_arguments`` — with no envelope it is a no-op, so the + approval bound the tool NAME but not the ARGUMENTS. + +ADR-125 closes both: the adapter derives ONE envelope from the tool +call it is about to make — the tool name plus the EXACT argument bag +it forwards — and its digest, and puts both on the wire. + +The doubles here are deliberately shallow: a mock MCP client and a +fake runtime that records the two wire hops (``check_workflow_budget`` +stands for ``/gate``, ``execute`` for ``/execute``). Nothing between +the adapter and the fake runtime is faked, so a change that drops the +envelope turns these red rather than being stubbed around. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from nullrun.business_impact import BusinessImpact, compute_action_digest +from nullrun.context import ( + clear_server_minted_execution_id, + get_call_impact, + set_call_impact, + set_mcp_tool_context, +) +from nullrun.toolbox.mcp import MCPAdapter + +# The backend's golden hex for ``BusinessImpact::tool_call`` with the +# tool ``stripe.charge`` and the params ``{"region": "EU", +# "amount": 500}`` (backend/src/proxy/gate/business_impact.rs:: +# DIGEST_FIXTURE_HEX_TOOL_CALL, mirrored by +# tests/test_business_impact_tool_call.py). The adapter derives its +# envelope with the SAME production canonicaliser, so the digest it +# puts on the wire for that call IS the backend's digest — the two +# repos cannot drift apart here without this going red. +GOLDEN_HEX_TOOL_CALL = ( + "9975a8b75a436fb78b9d141b9e0c0a90838c1243d78119b304ae6ed0526966a6" +) + + +# --------------------------------------------------------------------------- +# Doubles +# --------------------------------------------------------------------------- + + +class _Ann: + def __init__(self, read=None, destructive=None, open_world=None): + self.readOnlyHint = read + self.destructiveHint = destructive + self.openWorldHint = open_world + + +class _Tool: + def __init__(self, name, annotations=None): + self.name = name + self.annotations = annotations + + +class _MockMcpClient: + """Mock MCP client that records the tool name, the arguments AND + the extra kwargs — the gate fields must never travel into the MCP + server. + + Unlike the double in ``tests/test_mcp_approval_wire.py`` this one + accepts ANY tool name: these tests call tools that are not in the + cached inventory (``stripe.charge``, ``delete_file``) to prove + properties of the ENVELOPE, and the adapter's own + unknown-tool path (``class='invalid'``) is not what is under test + here. + """ + + def __init__(self, tools): + self._tools = {t.name: t for t in tools} + self.calls: list[dict[str, Any]] = [] + + def list_tools(self): + return list(self._tools.values()) + + def call_tool(self, name, arguments=None, **kwargs): + record = {"name": name, "arguments": arguments or {}, "kwargs": kwargs} + self.calls.append(record) + return f"ok:{name}" + + +class _RecordingRuntime: + """Fake runtime recording both wire hops. + + ``check_workflow_budget`` stands for the ``/gate`` pre-flight and + captures the envelope that is in scope at that moment — the + property under test is that the ``/gate`` and the ``/execute`` hash + the SAME bytes (ADR-065), so the capture has to happen inside the + gate hop, not after the call. + """ + + def __init__(self, payload=None): + self.execute_calls: list[dict[str, Any]] = [] + self.gate_calls = 0 + self.gate_impacts: list[Any] = [] + self._payload = payload or { + "decision": "allow", + "decision_source": "gateway", + "explanation": "allow", + } + + def check_workflow_budget(self): + self.gate_calls += 1 + self.gate_impacts.append(get_call_impact()) + + def execute(self, **kwargs): + self.execute_calls.append(kwargs) + return dict(self._payload) + + +def _inventory(): + return [ + _Tool("create_issue", _Ann(read=False, destructive=True, open_world=True)), + _Tool("get_file_contents", _Ann(read=True, destructive=False)), + ] + + +def _adapter(runtime, tools=None): + client = _MockMcpClient(tools if tools is not None else _inventory()) + adapter = MCPAdapter(server_name="github", mcp_client=client, runtime=runtime) + return adapter, client + + +@pytest.fixture(autouse=True) +def _isolate_context(): + """The three contextvars the adapter touches are process-wide, so a + stale value from a previous test would make a test pass for the + wrong reason.""" + + clear_server_minted_execution_id() + set_mcp_tool_context(tool_class=None, annotations=None) + saved_impact = get_call_impact() + yield + clear_server_minted_execution_id() + set_mcp_tool_context(tool_class=None, annotations=None) + set_call_impact(saved_impact) + + +# --------------------------------------------------------------------------- +# T1 — the derived envelope reaches the wire +# --------------------------------------------------------------------------- + + +class TestTheDerivedEnvelopeReachesTheWire: + def test_the_envelope_names_the_tool_and_carries_the_arguments(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + body = runtime.execute_calls[0] + envelope = body["business_impact"] + assert envelope["kind"] == "tool_call" + assert envelope["tool_name"] == "create_issue" + assert envelope["params"] == {"repo": "acme/api"} + + def test_the_envelope_params_are_the_canonical_bag(self): + """The backend compares ``BusinessImpact::ToolCall.params`` + with the live ``tool_arguments`` CANONICALLY + (``server_validate_tool_argument_binding``). Equal dicts are + canonical-equal by construction; a masked or reshaped copy is + not, and would bind the approval to arguments nobody + approved.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + body = runtime.execute_calls[0] + assert body["business_impact"]["params"] == body["tool_arguments"] + + def test_the_digest_is_the_digest_of_the_envelope_on_the_wire(self): + """Not a made-up hex: the digest on the wire is the digest OF + the envelope on the wire. The backend re-derives it from the + live payload and compares it with the value stored on the + approval row.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + body = runtime.execute_calls[0] + expected = compute_action_digest( + BusinessImpact.tool_call("create_issue", {"repo": "acme/api"}) + ) + assert body["action_digest"] == expected + + def test_the_digest_is_the_backend_golden_hex(self): + """The cross-repo half: the adapter's digest for a known call + is the BACKEND's digest, byte for byte.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("stripe.charge", {"region": "EU", "amount": 500}) + + assert runtime.execute_calls[0]["action_digest"] == GOLDEN_HEX_TOOL_CALL + + def test_a_tool_call_without_arguments_still_carries_an_envelope(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("get_file_contents") + + body = runtime.execute_calls[0] + assert body["business_impact"]["params"] == {} + assert body["action_digest"] == compute_action_digest( + BusinessImpact.tool_call("get_file_contents") + ) + + +# --------------------------------------------------------------------------- +# T2 — the digest binds the ARGUMENTS, not just the tool name +# --------------------------------------------------------------------------- + + +class TestTheDigestBindsTheArguments: + def test_changing_an_argument_changes_the_digest(self): + """The hole ADR-125 closes: an approval for + ``delete_file(path=/tmp/x)`` must not be spendable on + ``delete_file(path=/etc/passwd)``.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("delete_file", {"path": "/tmp/x"}) + adapter.call_tool("delete_file", {"path": "/etc/passwd"}) + + first, second = (call["action_digest"] for call in runtime.execute_calls) + assert first != second, "the approval would bind the tool name only" + + def test_changing_the_tool_changes_the_digest(self): + """The tool name is inside the hashed bytes, so a grant for one + tool does not validate a replay of another.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + adapter.call_tool("delete_issue", {"repo": "acme/api"}) + + first, second = (call["action_digest"] for call in runtime.execute_calls) + assert first != second + + +# --------------------------------------------------------------------------- +# T3 — the approval retry carries the SAME envelope +# --------------------------------------------------------------------------- + + +class TestTheApprovalRetryCarriesTheSameEnvelope: + def test_the_retry_derives_the_same_digest_as_the_first_hop(self): + """The operator approves, the caller retries with + ``approval_id=``. The backend compares the digest it + re-derives from the live payload with the digest STORED on the + approval row — which was stamped from the FIRST hop. A retry + that derives a different digest is refused with + APPROVAL_DIGEST_MISMATCH, so the two hops must agree.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + adapter.call_tool( + "create_issue", {"repo": "acme/api"}, approval_id="apr-123" + ) + + first, second = (call["action_digest"] for call in runtime.execute_calls) + assert first == second + assert second == runtime.execute_calls[1]["action_digest"] + + def test_the_retry_carries_the_envelope_too(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool( + "create_issue", {"repo": "acme/api"}, approval_id="apr-123" + ) + + body = runtime.execute_calls[0] + assert body["business_impact"]["params"] == {"repo": "acme/api"} + assert body["approval_id"] == "apr-123" + + +# --------------------------------------------------------------------------- +# T4 — ONE envelope per logical action (ADR-065) +# --------------------------------------------------------------------------- + + +class TestOneEnvelopePerLogicalAction: + def test_the_gate_and_the_execute_carry_the_same_bytes(self): + """The ``/gate`` pre-flight and the ``/execute`` must hash the + SAME envelope. The adapter installs the derived envelope on the + call context before the pre-flight runs, so the pre-flight + reads exactly what the ``/execute`` carries.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert runtime.gate_calls == 1 + gate_impact = runtime.gate_impacts[0] + assert gate_impact is not None, "the /gate saw no envelope at all" + body = runtime.execute_calls[0] + assert gate_impact.to_wire_dict() == body["business_impact"] + assert compute_action_digest(gate_impact) == body["action_digest"] + + def test_the_envelope_is_released_after_the_call(self): + """A leaked envelope would bind the NEXT action's approval to + this call's tool and arguments.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert get_call_impact() is None + + def test_the_envelope_is_released_even_when_the_call_is_refused(self): + runtime = _RecordingRuntime( + payload={ + "decision": "block", + "decision_source": "gateway", + "explanation": "destructive tool blocked", + "workflow_id": "wf-1", + } + ) + adapter, _ = _adapter(runtime) + + from nullrun.breaker.exceptions import NullRunBlockedException + + with pytest.raises(NullRunBlockedException): + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert get_call_impact() is None + + def test_an_envelope_already_in_scope_is_not_overwritten(self): + """The ``@protect`` path owns its own envelope and its own + ``/gate``/``/execute`` pair. Overwriting it would make the two + hops disagree — the exact drift ADR-065 exists to prevent.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + existing = BusinessImpact.tool_call("protected_fn", {"a": 1}) + set_call_impact(existing) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert runtime.gate_impacts[0] is existing + assert get_call_impact() is existing + + +# --------------------------------------------------------------------------- +# T5 — the fail-CLOSED fallback +# --------------------------------------------------------------------------- + + +class TestTheUnbuildableEnvelopeFailsClosed: + def test_a_float_argument_leaves_the_envelope_absent(self): + """The digest layer cannot round-trip an f64 — serde and Python + serialise it differently, so the two repos would never agree on + the hashed bytes. The backend's own ``ToolCallParams::validate`` + rejects it too.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("charge_card", {"amount": 1.5}) + + body = runtime.execute_calls[0] + assert "business_impact" not in body, "an unround-trippable envelope went out" + + def test_a_float_argument_still_carries_a_digest(self): + """The digest is still sent — over the ``no_impact`` sentinel. + The backend's post-approval re-entry then takes its fail-CLOSED + arm ("cannot recompute") instead of the legacy "no digest at + all" arm, which is a grant bound to nothing.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("charge_card", {"amount": 1.5}) + + body = runtime.execute_calls[0] + assert body["action_digest"] == compute_action_digest( + BusinessImpact.no_impact() + ) + + def test_a_non_ascii_tool_name_leaves_the_envelope_absent(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("создать_задачу", {"repo": "acme/api"}) + + body = runtime.execute_calls[0] + assert "business_impact" not in body + assert "action_digest" in body + + +# --------------------------------------------------------------------------- +# T6 — a caller-supplied envelope wins +# --------------------------------------------------------------------------- + + +class TestACallerEnvelopeIsForwardedVerbatim: + def test_a_caller_envelope_is_not_replaced_by_a_derived_one(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + envelope = { + "kind": "tool_call", + "tool_name": "create_issue", + "params": {"repo": "acme/api"}, + "extractor_id": "nullrun.tool_call.path", + "extractor_version": "1", + } + digest = compute_action_digest( + BusinessImpact.tool_call("create_issue", {"repo": "acme/api"}) + ) + + adapter.call_tool( + "create_issue", + {"repo": "acme/api"}, + business_impact=envelope, + action_digest=digest, + ) + + body = runtime.execute_calls[0] + assert body["business_impact"] == envelope + assert body["action_digest"] == digest + + def test_the_gate_never_sees_a_caller_envelope_it_did_not_carry(self): + """A caller envelope is NOT installed on the call context: the + SDK cannot know that the caller's envelope describes THIS + call. The contextvar keeps carrying whatever the caller's own + integration put there.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool( + "create_issue", + {"repo": "acme/api"}, + business_impact={ + "kind": "tool_call", + "tool_name": "create_issue", + "params": {}, + "extractor_id": "nullrun.tool_call.path", + "extractor_version": "1", + }, + action_digest="ab" * 32, + ) + + assert runtime.gate_impacts[0] is None + + +# --------------------------------------------------------------------------- +# T7 — the gate fields never reach the MCP server +# --------------------------------------------------------------------------- + + +class TestTheEnvelopeNeverReachesTheMcpServer: + def test_the_derived_envelope_stays_on_the_gate_side(self): + runtime = _RecordingRuntime() + adapter, client = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + kwargs = client.calls[0]["kwargs"] + assert "business_impact" not in kwargs + assert "action_digest" not in kwargs + assert "approval_id" not in kwargs + + def test_the_tool_payload_is_unchanged(self): + """The envelope is an ADDITIONAL gate field — the MCP server + still receives exactly the arguments it was given.""" + + runtime = _RecordingRuntime() + adapter, client = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert client.calls[0]["arguments"] == {"repo": "acme/api"} + assert client.calls[0]["name"] == "create_issue" diff --git a/tests/test_mcp_adapter.py b/tests/test_mcp_adapter.py index dc4c12a..e0a26fd 100644 --- a/tests/test_mcp_adapter.py +++ b/tests/test_mcp_adapter.py @@ -170,9 +170,14 @@ def test_cache_seconds_minimum(): def test_default_cache_seconds_matches_documented_value(): """Public constant — if we ever change the default, callers - that rely on it (operator docs) break. Pin it.""" + that rely on it (operator docs) break. Pin it. - assert DEFAULT_CACHE_SECONDS == 300 + The value was 300 until the hygiene round of 2026-11-08: the + cached annotations are a POLICY INPUT for the gate's MCP + umbrella policies, so the window was cut to the 30 s floor the + constructor already enforces.""" + + assert DEFAULT_CACHE_SECONDS == 30 # ----------------------------------------------------------------------- diff --git a/tests/test_mcp_annotation_staleness.py b/tests/test_mcp_annotation_staleness.py new file mode 100644 index 0000000..fd33559 --- /dev/null +++ b/tests/test_mcp_annotation_staleness.py @@ -0,0 +1,244 @@ +"""The MCP annotation cache is a POLICY INPUT, not a convenience. + +The gate's ``mcp_destructive_policy`` / ``mcp_readonly_policy`` +umbrella policies are evaluated against the ``destructive`` / +``read_only`` hints the adapter stamps on ``/check``. A cached hint +the server has since changed is a stale policy input, so two +properties are pinned here: + +1. The window is short (30 s by default) and an operator can remove + it entirely with ``cache_seconds=0``. +2. A FAILED refresh never presents the stale snapshot to the gate. + The gate gets the unknown shape and falls back to its own + classifier. + +The snapshot stays readable for DIAGNOSTICS after a failure -- it is +a diagnostic view of the last successful answer, not a claim about +the current one. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import pytest + +from nullrun.context import set_mcp_tool_context +from nullrun.toolbox.mcp import DEFAULT_CACHE_SECONDS, MCPAdapter + +# ----------------------------------------------------------------------- +# Test fixtures / mocks +# ----------------------------------------------------------------------- + + +@dataclass +class _Ann: + """MCP-style annotation object.""" + + readOnlyHint: bool | None = None + destructiveHint: bool | None = None + openWorldHint: bool | None = None + + +@dataclass +class _Tool: + """MCP-style tool entry.""" + + name: str + annotations: _Ann | None = None + + +class _FlakyClient: + """MCP client whose ``tools/list`` can be made to fail.""" + + def __init__(self, tools: list[_Tool]) -> None: + self._tools = {t.name: t for t in tools} + self.list_calls = 0 + self.fail_list = False + + def list_tools(self) -> list[_Tool]: + self.list_calls += 1 + if self.fail_list: + raise RuntimeError("tools/list is down") + return list(self._tools.values()) + + def call_tool( + self, name: str, arguments: dict[str, Any] | None = None, **kwargs: Any + ) -> str: + return f"ok:{name}" + + +class _StubRuntime: + """Allow-all runtime stub.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def check_workflow_budget(self) -> None: + return None + + def execute(self, **kwargs: Any) -> dict[str, Any]: + self.calls.append(kwargs) + return { + "decision": "allow", + "decision_source": "gateway", + "explanation": "stub allow", + "policy_version": 1, + } + + +@pytest.fixture(autouse=True) +def _isolate_mcp_context(): + set_mcp_tool_context(tool_class=None, annotations=None) + yield + set_mcp_tool_context(tool_class=None, annotations=None) + + +@pytest.fixture(autouse=True) +def _default_gate_runtime(): + """``MCPAdapter(runtime=None)`` resolves the global runtime.""" + from nullrun._registry import get_registry + + registry = get_registry() + previous = registry.get() + registry.set(_StubRuntime()) + yield + registry.set(previous) if previous is not None else registry.clear() + + +def _inventory() -> list[_Tool]: + return [ + _Tool( + name="read_notes", + annotations=_Ann( + readOnlyHint=True, destructiveHint=False, openWorldHint=False + ), + ), + ] + + +def _expire(adapter: MCPAdapter) -> None: + """Force the TTL to have elapsed. The adapter exposes no + public invalidate method; the existing suite pokes the private + attribute the same way.""" + adapter._cached_at = 0.0 + + +# ----------------------------------------------------------------------- +# The window itself +# ----------------------------------------------------------------------- + + +def test_the_default_window_is_thirty_seconds(): + """The cached annotations are a policy input. A five-minute + window is a five-minute window in which the gate can be told a + stale ``destructive=False``.""" + assert DEFAULT_CACHE_SECONDS == 30 + + +def test_a_window_below_thirty_seconds_is_refused(): + """An accidental 10 would hammer the upstream and skew the + drift detector. Still refused.""" + with pytest.raises(ValueError, match="cache_seconds"): + MCPAdapter( + server_name="notes", + mcp_client=_FlakyClient([]), + cache_seconds=10, + ) + + +def test_zero_is_accepted_and_removes_the_window(): + """``cache_seconds=0`` is the operator's way to pay one + ``tools/list`` round trip per call and hold no window at all.""" + client = _FlakyClient(_inventory()) + adapter = MCPAdapter( + server_name="notes", mcp_client=client, cache_seconds=0 + ) + adapter.call_tool("read_notes") + adapter.call_tool("read_notes") + assert client.list_calls == 2 + + +# ----------------------------------------------------------------------- +# The failed refresh +# ----------------------------------------------------------------------- + + +def test_a_failed_refresh_stamps_the_unknown_shape_not_the_stale_annotations(): + """The gate must not be told a stale ``destructive=False`` as if + it were the server's current answer.""" + client = _FlakyClient(_inventory()) + adapter = MCPAdapter(server_name="notes", mcp_client=client) + + # The first call populates the cache with destructive=False. + adapter.call_tool("read_notes") + assert client.list_calls == 1 + + # The upstream goes down and the window elapses. + client.fail_list = True + _expire(adapter) + adapter.call_tool("read_notes") + + # The gate was handed the unknown shape, not the cached hint. + from nullrun.context import get_call_mcp_annotations, get_call_mcp_class + + assert get_call_mcp_class() == "invalid" + annotations = get_call_mcp_annotations() + assert annotations is not None + assert annotations["destructive"] is None + assert annotations["read_only"] is None + assert annotations["open_world"] is None + + +def test_a_failed_refresh_keeps_the_snapshot_for_diagnostics(): + """The snapshot is the last successful answer. It stays + readable for diagnostics; it is not a claim about the current + one.""" + client = _FlakyClient(_inventory()) + adapter = MCPAdapter(server_name="notes", mcp_client=client) + adapter.call_tool("read_notes") + + client.fail_list = True + _expire(adapter) + adapter.call_tool("read_notes") + + assert adapter.list_cached_tools() == ["read_notes"] + cached = adapter.cached_annotations("read_notes") + assert cached is not None + assert cached.destructive is False + + +def test_a_successful_refresh_restores_the_annotations(): + """The unknown shape is a statement about a failed refresh, not + a new mode.""" + client = _FlakyClient(_inventory()) + adapter = MCPAdapter(server_name="notes", mcp_client=client) + adapter.call_tool("read_notes") + + client.fail_list = True + _expire(adapter) + adapter.call_tool("read_notes") + + client.fail_list = False + _expire(adapter) + adapter.call_tool("read_notes") + + from nullrun.context import get_call_mcp_annotations, get_call_mcp_class + + assert get_call_mcp_class() == "mcp" + annotations = get_call_mcp_annotations() + assert annotations is not None + assert annotations["destructive"] is False + assert annotations["read_only"] is True + + +def test_a_fresh_cache_does_not_re_list(): + """The window still buys what it bought: one ``tools/list`` per + window, not one per call.""" + client = _FlakyClient(_inventory()) + adapter = MCPAdapter(server_name="notes", mcp_client=client) + adapter.call_tool("read_notes") + adapter.call_tool("read_notes") + adapter.call_tool("read_notes") + assert client.list_calls == 1 diff --git a/tests/test_mcp_approval_wire.py b/tests/test_mcp_approval_wire.py index 29a6a06..2ca0a8e 100644 --- a/tests/test_mcp_approval_wire.py +++ b/tests/test_mcp_approval_wire.py @@ -10,7 +10,11 @@ saw the grant. * The adapter sent neither ``business_impact`` nor ``action_digest``, so the approval row was never digest-bound and the backend could not - detect a tampered payload when the grant was consumed. + detect a tampered payload when the grant was consumed. **ADR-125 + closed this one**: the adapter now DERIVES both from the tool call + it is about to make — the deep properties live in + ``tests/test_mcp_action_binding.py``, and the "absent impact" test + below now asserts the derived envelope instead of its absence. * The tool arguments reached the wire only as the legacy ``input`` field. The backend's canonical argument bag (ADR-060 §4) is ``tool_arguments`` → ``tool_params`` → Null, so with @@ -251,10 +255,22 @@ def test_action_digest_is_forwarded_to_the_execute_body(self): assert runtime.execute_calls[0]["action_digest"] == "cd" * 32 - def test_absent_impact_is_not_invented_on_the_wire(self): - """The adapter must not fabricate a digest the caller did not - supply — a made-up one would be a claim the backend cannot - verify.""" + def test_the_absent_impact_is_derived_from_the_call_itself(self): + """ADR-125 supersedes the old "do not invent a digest" rule. + + The adapter now derives the envelope from the call it is about + to make, so the digest on the wire is the digest OF that + envelope — not a made-up hex. The backend re-derives it from + the live payload and compares it with the digest stored on the + approval row, so a tampered payload is still refused; and + without an envelope the post-approval re-entry was refused + outright (BUSINESS_IMPACT_INVALID), so an operator approval + for an MCP tool could never be spent. + + The deep properties live in ``tests/test_mcp_action_binding.py``. + """ + + from nullrun.business_impact import BusinessImpact, compute_action_digest runtime = _RecordingRuntime() adapter, _ = _adapter(runtime) @@ -262,8 +278,9 @@ def test_absent_impact_is_not_invented_on_the_wire(self): adapter.call_tool("create_issue", {"repo": "acme/api"}) body = runtime.execute_calls[0] - assert "business_impact" not in body - assert "action_digest" not in body + envelope = BusinessImpact.tool_call("create_issue", {"repo": "acme/api"}) + assert body["business_impact"] == envelope.to_wire_dict() + assert body["action_digest"] == compute_action_digest(envelope) # --------------------------------------------------------------------------- @@ -323,9 +340,21 @@ def test_gate_runs_before_execute_when_no_execution_id_is_in_scope(self): assert runtime.gate_calls == 1, "no /gate ran" assert order == ["gate", "execute"] - def test_the_preflight_is_skipped_when_a_gate_already_ran(self): - """The pre-flight RESERVES money at the gate. Running it twice - for one logical call would charge the org twice.""" + def test_an_ambient_execution_id_alone_does_not_cover_this_tool_call(self): + """An execution id in scope says "some action was gated", not + "THIS action was gated". + + The id is set by ``/gate`` and by ``/execute`` and is never + reset in ``src/``, so after the first tool call in a scope it + is still set. Skipping the pre-flight on that basis ran NO + ``/gate`` for the second tool call: no budget check, no + reservation, no spend reserved for it (P2-18). + + The pre-flight RESERVES money at the gate, so running it twice + for ONE logical call would charge the org twice — which is + what the two cells below pin, for the two cases where the + ambient reservation really does cover THIS action. + """ set_server_minted_execution_id("0192f0c1-1111-7000-8000-000000000001") @@ -335,10 +364,70 @@ def test_the_preflight_is_skipped_when_a_gate_already_ran(self): adapter.call_tool("create_issue", {"repo": "acme/api"}) - assert runtime.gate_calls == 0, "a second /gate would charge twice" + assert runtime.gate_calls == 1, ( + "the tool call ran no /gate — its budget was never checked " + "and its spend was never reserved" + ) + assert len(runtime.execute_calls) == 1 + assert order == ["gate", "execute"] + + def test_the_approval_re_entry_skips_the_preflight(self): + """The approval's own execution id must survive the re-fire. + + The backend's require_approval arm echoes the approval row's + execution id into ``reservation_id``, so the ambient id IS the + approval's execution. A second ``/gate`` would mint a DIFFERENT + id and the approval's consume (``WHERE execution_id = $3``) + would miss its row (APPROVAL_REPLAY_REJECTED). + """ + + set_server_minted_execution_id("0192f0c1-1111-7000-8000-000000000001") + + order: list[str] = [] + runtime = _RecordingRuntime(order=order) + adapter, _ = _adapter(runtime) + + adapter.call_tool( + "create_issue", + {"repo": "acme/api"}, + approval_id="appr-1", + ) + + assert runtime.gate_calls == 0, ( + "the approval re-entry ran a second /gate — a second reserve " + "for one logical call charges the org twice" + ) assert len(runtime.execute_calls) == 1 assert order == ["execute"] + def test_the_protect_scope_skips_the_preflight(self): + """The scope reserved for itself, and only for itself. + + The ``@protect`` path runs its own ``/gate`` for the whole + scope. A second ``/gate`` inside the scope reserves twice and + orphans the first (NR-059). + """ + from nullrun.business_impact import BusinessImpact + from nullrun.context import reset_call_impact, set_call_impact + + set_server_minted_execution_id("0192f0c1-1111-7000-8000-000000000001") + token = set_call_impact(BusinessImpact.no_impact()) + try: + order: list[str] = [] + runtime = _RecordingRuntime(order=order) + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert runtime.gate_calls == 0, ( + "the protected scope ran a second /gate — the second " + "reserve orphans the first" + ) + assert len(runtime.execute_calls) == 1 + assert order == ["execute"] + finally: + reset_call_impact(token) + def test_a_blocked_preflight_never_reaches_the_mcp_client(self): """The pre-flight is a gate, not a formality: a block there stops the call before /execute and before the MCP server.""" diff --git a/tests/test_messages.py b/tests/test_messages.py index a214b32..cda1a86 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -38,6 +38,7 @@ "NR-A014", "NR-A015", "NR-A016", # B.1 (2026-09-10): APPROVAL_DB_* sibling family + "NR-A017", # DEC-5 / DEC-7 (2026-11-02): approval tool mismatch "NR-B001", "NR-B002", "NR-B004", @@ -57,6 +58,12 @@ "NR-MCP03", # B.1 (2026-09-10): MCP umbrella approval required "NR-O001", "NR-P001", + # NR-P004: the SDK could not obtain a decision (ADR-129 on the + # /gate seam, ADR-131 on the /execute seam). It had NO entry here + # and none in DEFAULT_MESSAGES, so the refusal rendered as the + # generic fallback — the operator saw "Something went wrong" for + # the one failure that stops the call. + "NR-P004", "NR-R001", "NR-R002", "NR-T001", diff --git a/tests/test_p2_17_edge_last_source.py b/tests/test_p2_17_edge_last_source.py new file mode 100644 index 0000000..adc0c05 --- /dev/null +++ b/tests/test_p2_17_edge_last_source.py @@ -0,0 +1,217 @@ +"""P2-17: `last_source` must tell a box ALLOW from a box REFUSAL. + +The bug this pins: `EdgeTransport.last_source` was set to the single +constant `EDGE_LEASE` on BOTH the allow path and the refusal path, so +the two collapsed into one string. A caller reading the attribute could +not tell which way the box went — the attribute said "the lease spoke" +when the truth was "the lease said no". + +Direction of the failure: this is a *reporting* defect, not a money +one. The decision itself was never wrong — the refusal path returned +`decision: "block"` with the box's own `error_code`. What was wrong is +the provenance the SDK reported about it. So the assertions below are +about the SHAPE of the report, and every one of them also asserts the +decision still goes the way it went before. + +The wire value is deliberately NOT changed. `decision_source` on the +refusal path stays `EDGE_LEASE`: the lease was the authority that +spoke either way, the direction is carried by `decision`, and the +runtime validates `decision_source` against a known set — a source it +does not know is a malformed response, not a refusal. That is why the +new constant is a `last_source` marker only and never reaches the +wire. + +The fake box is the same shape `tests/test_edge_via_edge.py` uses: the +transport's `_post` is replaced by a responder, so the real +`enforce` → `_ask` → decision-building path runs end to end. +""" + +from __future__ import annotations + +import nullrun.edge as edge +from nullrun.edge import ( + EDGE_LEASE, + EDGE_NOT_ENFORCED, + EDGE_UNREACHABLE, + EdgeTransport, +) +from nullrun.transport import is_fallback_decision_source + + +class _FakeResponse: + """What a box answers with. Same shape as the other edge tests.""" + + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +def _transport(responder): + t = EdgeTransport( + base_url="http://127.0.0.1:1", + lease_id="0195f0c0-0000-7000-8000-000000000001", + box_token="box-token", + api_key="nr_org_key", + timeout=0.05, + max_retries=0, + ) + t._post = responder # type: ignore[method-assign] + return t + + +def _allowing_box(body): + return _FakeResponse( + { + "allowed": True, + "lease_id": body["lease_id"], + "granted_millicents": 500_000, + "spent_millicents": 10, + "remaining_millicents": 499_990, + } + ) + + +def _refusing_box(body): + return _FakeResponse({"allowed": False, "error_code": "EDGE_LEASE_EXHAUSTED"}) + + +# ── the collapse itself ─────────────────────────────────────────────── + + +def test_the_allow_path_and_the_refusal_path_do_not_collapse_into_one_source(): + """The defect, stated as a fact about the two paths. + + Both paths are driven for real — a box that says yes and a box that + says no — and the assertion is that the attribute a caller reads + afterwards tells them apart. Pre-fix both read `edge_lease`. + """ + allow_t = _transport(_allowing_box) + allow_result = allow_t.enforce("gpt-4o", 0, 10) + + refuse_t = _transport(_refusing_box) + refuse_result = refuse_t.enforce("gpt-4o", 0, 10) + + # The decisions were never wrong — assert them first, so a test that + # only ever checks provenance cannot be satisfied by an + # implementation that stops deciding. + assert allow_result["decision"] == "allow" + assert refuse_result["decision"] == "block" + + assert allow_t.last_source != refuse_t.last_source, ( + "last_source must distinguish a box ALLOW from a box REFUSAL; " + f"both paths read {allow_t.last_source!r}" + ) + + +def test_the_refusal_source_is_its_own_named_constant(): + """Not merely "different" — named, and the same on both sides. + + A caller cannot act on a value it has no name for, so the refusal + source is a constant the SDK exports, and the refusal path reports + exactly it. + """ + refused = getattr(edge, "EDGE_LEASE_REFUSED", None) + assert refused is not None, ( + "nullrun.edge must export EDGE_LEASE_REFUSED — the source that " + "says the lease REFUSED, as distinct from EDGE_LEASE" + ) + assert refused != EDGE_LEASE + + t = _transport(_refusing_box) + t.enforce("gpt-4o", 0, 10) + assert t.last_source == refused + + +def test_every_edge_source_is_distinct_from_every_other(): + """Four outcomes, four strings. A collapse anywhere is a collapse.""" + refused = getattr(edge, "EDGE_LEASE_REFUSED", None) + assert refused is not None, "nullrun.edge must export EDGE_LEASE_REFUSED" + + sources = {EDGE_LEASE, refused, EDGE_UNREACHABLE, EDGE_NOT_ENFORCED} + assert len(sources) == 4 + + +# ── what the fix must NOT change ────────────────────────────────────── + + +def test_the_refusal_still_carries_the_lease_as_its_wire_decision_source(): + """The wire value is untouched, and that is deliberate. + + `decision_source` says WHO decided. On a refusal the lease was the + authority that spoke — the box answered, with "no" — so the source + is still `EDGE_LEASE`, and `decision` says which way it went. + + Changing it would need a new entry in the runtime's known-source + set; until the runtime knows the name, a body carrying it is a + MALFORMED response, which turns a clean refusal into a crash. + """ + t = _transport(_refusing_box) + result = t.enforce("gpt-4o", 0, 10) + + assert result["decision"] == "block" + assert result["decision_source"] == EDGE_LEASE + assert result["error_code"] == "EDGE_LEASE_EXHAUSTED" + assert "grant" in result["explanation"].lower() + + +def test_the_refusal_still_flows_through_the_runtime_unchanged(): + """The refusal is still one the runtime parses and honours. + + `_require_gate_decision` runs before the decision is read and + rejects any body whose `decision_source` it does not know. This + asserts the refusal survives that check as a `block` — which is + the whole reason the wire value was left alone. + """ + from nullrun.runtime import NullRunRuntime + + rt = NullRunRuntime.__new__(NullRunRuntime) # no I/O, no client + t = _transport(_refusing_box) + result = t.enforce("gpt-4o", 0, 10) + + assert rt._require_gate_decision(result) == "block" + + +def test_the_refusal_source_is_not_read_as_a_synthetic_answer(): + """A box that said no is a verdict, not a degraded transport. + + `is_fallback_decision_source` is the runtime's fail-OPEN predicate: + a source that reads as synthetic means "the engine never answered", + and the refusal would be discarded before the caller saw it. The new + marker must not be mistaken for one. + """ + refused = edge.EDGE_LEASE_REFUSED + assert is_fallback_decision_source(refused) is False + + +# ── the neighbouring sources did not move ────────────────────────────── + + +def test_the_unreachable_box_still_reports_the_unreachable_source(): + """The failure paths keep their own markers.""" + + def boom(body): + raise OSError("connection refused") + + t = _transport(boom) + result = t.enforce("gpt-4o", 0, 10) + + assert result["decision"] == "block" + assert result["decision_source"] == EDGE_UNREACHABLE + assert t.last_source == EDGE_UNREACHABLE + + +def test_the_never_asked_box_still_reports_the_not_enforced_source(): + t = _transport(_allowing_box) + result = t.not_enforced() + + assert result["decision"] == "block" + assert result["decision_source"] == EDGE_NOT_ENFORCED + assert t.last_source == EDGE_NOT_ENFORCED + + +def test_the_new_constant_is_exported_like_its_siblings(): + """Named in `__all__`, so a caller can import it by name.""" + assert "EDGE_LEASE_REFUSED" in edge.__all__ diff --git a/tests/test_p2_18_mcp_preflight_scope.py b/tests/test_p2_18_mcp_preflight_scope.py new file mode 100644 index 0000000..4403464 --- /dev/null +++ b/tests/test_p2_18_mcp_preflight_scope.py @@ -0,0 +1,310 @@ +"""P2-18 — every MCP tool call must run its OWN ``/gate`` pre-flight. + +The adapter ran ``runtime.check_workflow_budget()`` only when +``get_server_minted_execution_id()`` was ``None``. That contextvar is +set by ``/gate`` and by ``/execute`` and is NEVER reset in ``src/``, so +the second and later MCP tool calls in one scope ran NO ``/gate``: +no budget check, no reservation, no spend reserved for that call. + +The pre-flight still has to be SKIPPED in two cases, and this file pins +both — a fix that simply always pre-flights would double-charge: + +* the approval re-entry — the backend's require_approval arm echoes the + approval row's execution id into ``reservation_id`` + (``enforcement/gate_wire_adapter.rs``), so the ambient id IS the + approval's execution. A second ``/gate`` would mint a different id and + the approval's consume (``WHERE execution_id = $3``) would miss its + row (APPROVAL_REPLAY_REJECTED). +* the ``@protect`` scope — one reserve for the whole scope (NR-059). A + second reserve inside the scope orphans the first. + +The cells drive the REAL runtime over respx-mocked wire, so the +assertions are about what went ON THE WIRE, not about a fake's counters. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any + +import httpx +import pytest +import respx + +import nullrun +from nullrun.business_impact import BusinessImpact +from nullrun.context import ( + reset_call_impact, + set_call_impact, + set_server_minted_execution_id, +) +from nullrun.toolbox.mcp import MCPAdapter + +BASE_URL = "https://api.test.nullrun.io" + +# The ambient id the approval re-entry carries. A UUIDv7 shape, because +# the backend mints v7 (ADR-073); the SDK only requires it to parse. +AMBIENT_ID = "0192f0c1-1111-7000-8000-000000000001" + + +class _Ann: + def __init__(self, read=None, destructive=None, open_world=None): + self.readOnlyHint = read + self.destructiveHint = destructive + self.openWorldHint = open_world + + +class _Tool: + def __init__(self, name, annotations=None): + self.name = name + self.annotations = annotations + + +class _MockMcpClient: + """Mock MCP client. Records the tool name and the arguments.""" + + def __init__(self, tools): + self._tools = {t.name: t for t in tools} + self.calls: list[dict[str, Any]] = [] + + def list_tools(self): + return list(self._tools.values()) + + def call_tool(self, name, arguments=None, **kwargs): + self.calls.append({"name": name, "arguments": arguments or {}}) + return f"ok:{name}" + + +@pytest.fixture +def wire(mock_api): + """The two gate hops, captured off the wire. + + The ``/gate`` answer carries a ``reservation_id`` because the + backend's Allow arm stamps the server-minted uuid there + (``gate/internal.rs``). Without it the SDK's capture CLEARS the + ambient id, and the defect this file pins would be invisible. + The id is a UUIDv7 because the backend mints v7 (ADR-073). + + The ``/execute`` answer echoes the request's ``execution_id`` back, + because the ``/execute`` arm reuses the ``/gate`` id (ADR-073) and + the SDK reads that echo. + """ + from nullrun.uuid7 import uuid7_str + + bodies: dict[str, list[Any]] = {"gate": [], "execute": [], "order": []} + + def _gate(request: httpx.Request) -> httpx.Response: + bodies["gate"].append(json.loads(request.content.decode("utf-8"))) + bodies["order"].append("gate") + return httpx.Response( + 200, + json={ + "decision": "allow", + "actions": [], + "local_cost_cents": 0, + "policy_id": "policy-test", + "decision_source": "gateway", + "reservation_id": uuid7_str(), + }, + ) + + def _execute(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content.decode("utf-8")) + bodies["execute"].append(body) + bodies["order"].append("execute") + return httpx.Response( + 200, + json={ + "decision": "allow", + "decision_source": "gateway", + "explanation": "allowed", + "policy_version": 1, + "reservation_id": body.get("execution_id"), + }, + ) + + respx.post(f"{BASE_URL}/api/v1/gate").mock(side_effect=_gate) + respx.post(f"{BASE_URL}/api/v1/execute").mock(side_effect=_execute) + return bodies + + +@pytest.fixture(autouse=True) +def _isolate_context(): + """No ambient execution id or envelope leaks between cells. + + The server-minted execution id is never reset in ``src/``, so a + cell that installs one would change the NEXT cell's pre-flight + decision. Every cell starts from a clean scope. + """ + from nullrun.context import clear_server_minted_execution_id + + clear_server_minted_execution_id() + yield + clear_server_minted_execution_id() + + +def _adapter(runtime): + client = _MockMcpClient( + [_Tool("create_issue", _Ann(read=False, destructive=True, open_world=True))] + ) + adapter = MCPAdapter( + server_name="github", + mcp_client=client, + runtime=runtime, + ) + return adapter, client + + +class _CountingRuntime: + """A runtime double whose transport MAY carry an edge box. + + The adapter's condition reads ``runtime._transport.edge``: the + presence of a box is what makes the pre-flight unconditional. The + double counts the pre-flight calls; the box itself is not the + subject of the edge cell. + """ + + def __init__(self, *, edge: bool): + self.gate_calls = 0 + self.execute_calls: list[dict[str, Any]] = [] + self._transport = SimpleNamespace(edge=object() if edge else None) + + def check_workflow_budget(self): + self.gate_calls += 1 + + def execute(self, **kwargs): + self.execute_calls.append(kwargs) + return { + "decision": "allow", + "decision_source": "gateway", + "explanation": "allowed", + } + + +class TestEveryToolCallIsGated: + def test_two_tool_calls_run_two_gates(self, make_runtime, wire): + """The defect, stated as a fact about the wire. + + Pre-fix the second call issued no ``/gate`` at all: its budget + was never checked and its spend was never reserved. + """ + rt = make_runtime() + adapter, client = _adapter(rt) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert len(wire["gate"]) == 2, ( + "the second tool call ran no /gate — its budget was never " + "checked and its spend was never reserved" + ) + assert len(client.calls) == 2, "the MCP server was called twice" + + def test_each_tool_call_carries_its_own_execution_id(self, make_runtime, wire): + """One reservation per tool call. + + Pre-fix both ``/execute`` bodies carried the FIRST call's + execution id, so the second call settled against a reservation + that was already spent. + """ + rt = make_runtime() + adapter, _ = _adapter(rt) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + ids = [body["execution_id"] for body in wire["execute"]] + assert len(set(ids)) == 2, ( + f"both /execute calls carried the same execution id {ids}: the " + "second tool call reused the first call's reservation" + ) + + def test_every_execute_is_preceded_by_its_own_gate(self, make_runtime, wire): + """The order is the contract: gate, execute, gate, execute. + + A ``/execute`` with no ``/gate`` of its own is refused by the + backend with 404 EXECUTION_NOT_FOUND — the ``execution:{id}`` + binding does not exist. + """ + rt = make_runtime() + adapter, _ = _adapter(rt) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert wire["order"] == ["gate", "execute", "gate", "execute"] + + +class TestTheSkipsThatMustStay: + def test_the_approval_re_entry_does_not_run_a_second_gate(self, make_runtime, wire): + """The approval's own execution id must survive the re-fire. + + The backend's require_approval arm echoes the approval row's + execution id into ``reservation_id``, so the ambient id IS the + approval's execution. A second ``/gate`` would mint a different + id and the approval's consume would miss its row. + """ + rt = make_runtime() + adapter, _ = _adapter(rt) + set_server_minted_execution_id(AMBIENT_ID) + + adapter.call_tool("create_issue", {"repo": "acme/api"}, approval_id="appr-1") + + assert wire["gate"] == [], "the approval re-entry ran a second /gate" + assert len(wire["execute"]) == 1 + assert wire["execute"][0]["execution_id"] == AMBIENT_ID, ( + "the approval re-entry must carry the approval's execution id" + ) + + def test_the_protect_scope_reserves_once(self, make_runtime, wire): + """One reserve for the whole protected scope (NR-059). + + A second reserve inside the scope orphans the first, and the + scope's spend settles against the wrong envelope. + """ + rt = make_runtime() + + @nullrun.protect + def step(): + adapter, _ = _adapter(rt) + adapter.call_tool("create_issue", {"repo": "acme/api"}) + return "ok" + + assert step() == "ok" + assert len(wire["gate"]) == 1, ( + "the protected scope ran more than one /gate — the second " + "reserve orphans the first" + ) + + +class TestTheEdgeCarveOut: + def test_the_edge_asks_the_box_even_when_the_scope_is_covered(self): + """Via-edge asks the box per tool call, covered scope or not. + + The edge has no cloud ``/track`` to book the call through, so + the box ask IS the charge, and the ``/execute`` reads back the + box's answer for ITS event id. A scope that is covered for the + cloud (one reserve for the whole scope) is NOT covered for the + edge: skipping the ask leaves the box unconsulted and the + ``/execute`` then fails CLOSED (SDK-2). + """ + token = set_call_impact(BusinessImpact.no_impact()) + set_server_minted_execution_id(AMBIENT_ID) + try: + cloud = _CountingRuntime(edge=False) + edge_rt = _CountingRuntime(edge=True) + + _adapter(cloud)[0].call_tool("create_issue", {"repo": "acme/api"}) + _adapter(edge_rt)[0].call_tool("create_issue", {"repo": "acme/api"}) + + assert cloud.gate_calls == 0, ( + "a covered cloud scope must NOT pre-flight again — a second " + "/gate would charge the org twice for one logical call" + ) + assert edge_rt.gate_calls == 1, ( + "the edge must ask the box for EVERY tool call — the box " + "is the enforcement surface there" + ) + finally: + reset_call_impact(token) diff --git a/tests/test_p2_19_ws_unsigned_frame.py b/tests/test_p2_19_ws_unsigned_frame.py new file mode 100644 index 0000000..b70301b --- /dev/null +++ b/tests/test_p2_19_ws_unsigned_frame.py @@ -0,0 +1,291 @@ +"""P2-19 — a control-plane frame the client could have verified must be +verified, or refused. + +The receiver verified the signature only when all four of these were +present: ``signature`` and ``timestamp`` on the frame, and ``api_key`` +and ``secret_key`` on the client. A frame that was missing ANY one of +them skipped the verification entirely and was then read from its outer +body — the body the server never signed. + +The hole that matters: a client that HOLDS the API key and the HMAC +secret can verify every frame the server sends it, because the server +signs every frame it sends for such a key +(``ws_control.rs::send_signed_or_raw``). A frame that arrives at such a +client without a signature — or with a signature but no timestamp — did +not come from the server. It must be refused, not delivered. + +The other side of the contract, and the reason this is not a blanket +"unsigned means forged": a client WITHOUT a secret (a legacy key) can +verify nothing, so the unsigned path stays open for it. The kill switch +must keep working there — it can only stop the agent, never release it. + +The cells drive the real ``WebSocketConnection._handle_message``. +""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any + +import pytest + +from nullrun.observability import metrics +from nullrun.transport import generate_hmac_signature +from nullrun.transport_websocket import WebSocketConnection + +API_KEY = "nr_live_test_key" +SECRET = "s3cret" + +KILL_FRAME: dict[str, Any] = { + "type": "state_change", + "workflow_id": "wf-1", + "state": "Killed", + "version": 1, + "message_id": "m-1", +} + + +def _signed_envelope(message: dict, api_key: str, secret_key: str) -> dict: + """The server's ``SignedWsMessage`` shape. + + The payload is flattened into the outer body AND published + hex-encoded as ``signed_payload``; the HMAC covers the INNER + payload bytes only. + """ + timestamp = int(time.time()) + payload_json = json.dumps(message, separators=(",", ":"), sort_keys=True) + signature = generate_hmac_signature( + api_key, secret_key, timestamp, payload_json.encode("utf-8") + ) + envelope = dict(message) + envelope["signature"] = signature + envelope["timestamp"] = timestamp + envelope["api_key"] = api_key + envelope["signed_payload"] = payload_json.encode("utf-8").hex() + return envelope + + +class _StubWS: + """Captures what the SDK writes back (the ACK).""" + + def __init__(self) -> None: + self.sent: list[bytes] = [] + self.closed = False + + async def send(self, data) -> None: + if isinstance(data, str): + self.sent.append(data.encode("utf-8")) + else: + self.sent.append(data) + + async def close(self) -> None: + self.closed = True + + +def _conn( + *, + api_key: str | None = API_KEY, + secret_key: str | None = SECRET, + on_state_change=None, + on_approval_resolved=None, +) -> tuple[WebSocketConnection, _StubWS]: + conn = WebSocketConnection( + url="wss://example.invalid/ws/control/org-1", + headers={}, + api_key=api_key, + secret_key=secret_key, + on_state_change=on_state_change, + on_approval_resolved=on_approval_resolved, + ) + stub = _StubWS() + conn._conn = stub + conn._running = True + return conn, stub + + +class TestAnUnsignedFrameIsRefusedWhenTheClientCanVerify: + """This client holds the key and the secret, so it can verify.""" + + @pytest.mark.asyncio + async def test_a_frame_without_a_signature_is_refused(self): + received: list[dict] = [] + conn, stub = _conn(on_state_change=received.append) + + before = metrics.transport.ws_unverified_frame_total + await conn._handle_message(json.dumps(dict(KILL_FRAME))) + + assert received == [], ( + "an unsigned frame was delivered to a client that can verify " + "signatures — it did not come from the server" + ) + assert metrics.transport.ws_unverified_frame_total == before + 1, ( + "the refusal must be counted — an operator has to be able to " + "see it" + ) + assert stub.sent == [], "a refused frame must not be ACKed" + + @pytest.mark.asyncio + async def test_a_frame_with_a_signature_but_no_timestamp_is_refused(self): + """The audit's case: the frame is missing ONE field. + + Pre-fix the missing timestamp skipped the whole verification + block, and the frame was read from its unsigned outer body. + """ + received: list[dict] = [] + conn, _ = _conn(on_state_change=received.append) + + envelope = _signed_envelope(dict(KILL_FRAME), API_KEY, SECRET) + del envelope["timestamp"] + + before = metrics.transport.ws_unverified_frame_total + await conn._handle_message(json.dumps(envelope)) + + assert received == [], ( + "a frame with a signature but no timestamp skipped " + "verification and was delivered" + ) + assert metrics.transport.ws_unverified_frame_total == before + 1 + + @pytest.mark.asyncio + async def test_a_frame_with_a_timestamp_but_no_signature_is_refused(self): + received: list[dict] = [] + conn, _ = _conn(on_state_change=received.append) + + envelope = dict(KILL_FRAME) + envelope["timestamp"] = int(time.time()) + + before = metrics.transport.ws_unverified_frame_total + await conn._handle_message(json.dumps(envelope)) + + assert received == [], ( + "a frame with a timestamp but no signature skipped " + "verification and was delivered" + ) + assert metrics.transport.ws_unverified_frame_total == before + 1 + + @pytest.mark.asyncio + async def test_a_refused_frame_cannot_block_the_next_real_kill(self): + """A forged frame must not poison the version dedup. + + The dispatcher drops a state event whose version is not newer + than the last one it saw for that workflow. A forged frame with + a huge version would raise that bar, and the next REAL kill + would be dropped as stale — the forgery would silence the kill + switch. + """ + received: list[dict] = [] + conn, _ = _conn(on_state_change=received.append) + + forged = dict(KILL_FRAME) + forged["version"] = 99 + forged["state"] = "Normal" + await conn._handle_message(json.dumps(forged)) + + genuine = _signed_envelope(dict(KILL_FRAME), API_KEY, SECRET) + await conn._handle_message(json.dumps(genuine)) + + assert len(received) == 1, ( + "the real kill never arrived — the forged frame raised the " + "version bar and the genuine frame was dropped as stale" + ) + assert received[0]["state"] == "Killed", ( + "the forged frame was delivered and the genuine kill was " + f"dropped as stale — the SDK dispatched {received[0]!r}" + ) + + @pytest.mark.asyncio + async def test_an_unsigned_approval_still_bumps_the_approval_counter(self): + """The approval-specific signal survives the blanket refusal. + + The operator has to see that somebody pushed an approval this + client cannot verify — that counter is the alertable one. + """ + received: list[dict] = [] + conn, stub = _conn(on_approval_resolved=received.append) + + before = metrics.transport.ws_unverified_frame_total + before_approval = metrics.transport.ws_unverified_approval_total + await conn._handle_message( + json.dumps( + { + "type": "approval_resolved", + "approval_id": "ap-1", + "workflow_id": "wf-1", + "execution_id": "ex-1", + "outcome": "approved", + } + ) + ) + + assert received == [], "an unsigned approval frame was dispatched" + assert metrics.transport.ws_unverified_frame_total == before + 1, ( + "the refusal must be counted" + ) + assert metrics.transport.ws_unverified_approval_total == before_approval + 1, ( + "the approval refusal must stay countable" + ) + assert stub.sent == [], "a refused frame must not be ACKed" + + @pytest.mark.asyncio + async def test_the_refusal_log_names_the_frame_code(self, caplog): + """The operator keeps the server's reason. + + The server's own refusal frame is unsigned BY DESIGN: it is + sent before the connection is authenticated, so there is no + secret to sign it with yet. The SDK refuses that frame — and + the log must still say what the server said. + """ + conn, _ = _conn() + + with caplog.at_level(logging.ERROR): + await conn._handle_message( + json.dumps({"type": "error", "code": "ORGANIZATION_MISMATCH"}) + ) + + assert "ORGANIZATION_MISMATCH" in caplog.text, ( + "the refusal log lost the server's reason — the operator sees " + "the refusal but not why the server closed the socket" + ) + + +class TestTheKillSwitchStillWorks: + """The safety direction must not regress.""" + + @pytest.mark.asyncio + async def test_a_client_without_a_secret_still_receives_the_kill(self): + """A legacy key has no HMAC secret, so nothing can be verified. + + Refusing the frame here would break the kill switch for every + org on a legacy key. The frame can only stop the agent, never + release it. + """ + received: list[dict] = [] + conn, _ = _conn(api_key="k", secret_key=None, on_state_change=received.append) + + await conn._handle_message(json.dumps(dict(KILL_FRAME))) + + assert len(received) == 1, ( + "the kill switch stopped working — a kill must be delivered " + "even when the frame carries no signature" + ) + assert received[0]["state"] == "Killed" + + @pytest.mark.asyncio + async def test_a_signed_kill_is_delivered(self): + """The canary for the negative assertions above. + + The same client, the same frame, signed by the server: it must + arrive. + """ + received: list[dict] = [] + conn, stub = _conn(on_state_change=received.append) + + envelope = _signed_envelope(dict(KILL_FRAME), API_KEY, SECRET) + await conn._handle_message(json.dumps(envelope)) + + assert len(received) == 1, "a genuine signed kill was not delivered" + assert received[0]["state"] == "Killed" + assert received[0]["workflow_id"] == "wf-1" + assert stub.sent, "a genuine killed frame must be ACKed" diff --git a/tests/test_production_environment_detection.py b/tests/test_production_environment_detection.py new file mode 100644 index 0000000..8664cfc --- /dev/null +++ b/tests/test_production_environment_detection.py @@ -0,0 +1,209 @@ +"""SDK-2 — the production detector must not be a substring test. + +`_is_production_environment` decides whether the two fail-open guards +(`NULLRUN_SKIP_BUDGET_CHECK`, `NULLRUN_SENSITIVE_FAIL_OPEN`) arm. Before +the fix it asked "is any marker a SUBSTRING of the hostname?". `"test"` +is a substring of `latest`, `contest`, `attest`, `greatest`; `"localhost"` +is a substring of `notlocalhost.example.com`. So a host that merely +CONTAINED a dev marker was classified non-prod, and the operator's +`NULLRUN_ENV=production` was suppressed by the substring match. The +billing bypass then ran against production with a WARNING instead of a +refusal (L1 / L4). + +The rule after the fix: a host is non-prod only when it is EXACTLY a +loopback / a `.localhost` suffix, or when it carries an explicit +dev-environment LABEL (`staging.`, `test.`). Never by substring. + +The table below is the audit's, evaluated in the context the audit's +exploit skeleton uses -- the operator declared production and the host +merely looks dev: + + latest.nullrun.io -> production + contest.nullrun.io -> production + notlocalhost.example.com -> production + attest.example.com -> production + localhost -> NOT production + 127.0.0.1 -> NOT production + staging.internal -> NOT production + +The last check is the one that matters for the money: with the flag +set against a prod-classified URL the SDK must RAISE NR-S001, not log a +warning. +""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from nullrun.runtime import _is_production_environment + +# A prod-classified host the SDK's own fixtures do not mock. +PROD_LIKE_URL = "https://latest.nullrun.io" + +# Hosts the audit names. Every one of them CONTAINS a dev marker as a +# substring; none of them carries one as a label. +PROD_HOSTS = [ + "latest.nullrun.io", + "contest.nullrun.io", + "notlocalhost.example.com", + "attest.example.com", +] + +# Hosts that really are local / dev. +NON_PROD_HOSTS = [ + "localhost", + "127.0.0.1", + "staging.internal", +] + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + """A developer's shell must not decide whether these pass.""" + monkeypatch.delenv("NULLRUN_ENV", raising=False) + monkeypatch.delenv("NULLRUN_API_URL", raising=False) + + +@pytest.fixture +def mock_prod_like_api(mock_api): + """The auth route for the prod-like host used below. + + `mock_api` only covers `api.test.nullrun.io`, so a runtime built + with any other `api_url` authenticates against an unmocked host and + respx fails the test before the guard under test is consulted. + """ + + def _verify(request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "organization_id": "ws-test", + "workflow_id": "00000000-0000-0000-0000-000000000001", + "plan": "pro", + "features": [], + "limits": {"max_cost_cents": 10000}, + "secret_key": "test-secret-deterministic", + }, + ) + + respx.post(f"{PROD_LIKE_URL}/api/v1/auth/verify").mock(side_effect=_verify) + return mock_api + + +class TestTheAuditTable: + """The operator declared production. The host must not veto it.""" + + @pytest.mark.parametrize("host", PROD_HOSTS) + def test_a_marker_inside_the_name_does_not_veto_production( + self, monkeypatch, host + ): + monkeypatch.setenv("NULLRUN_ENV", "production") + assert _is_production_environment(f"https://{host}") is True, ( + f"{host} was classified non-prod -- a dev marker found as a " + "SUBSTRING of the hostname suppressed NULLRUN_ENV=production" + ) + + @pytest.mark.parametrize("host", NON_PROD_HOSTS) + def test_a_real_dev_host_stays_non_prod(self, monkeypatch, host): + monkeypatch.setenv("NULLRUN_ENV", "production") + assert _is_production_environment(f"https://{host}") is False, ( + f"{host} was classified production -- the local / staging " + "escape hatch must keep working" + ) + + +class TestTheCanonicalProdHost: + def test_the_default_host_is_production(self): + assert _is_production_environment("https://api.nullrun.io") is True + + def test_no_api_url_uses_the_default_host(self): + """The constructor's default is the prod endpoint.""" + assert _is_production_environment(None) is True + + def test_the_api_url_env_var_is_used_when_no_url_is_given(self, monkeypatch): + monkeypatch.setenv("NULLRUN_API_URL", "https://latest.nullrun.io") + monkeypatch.setenv("NULLRUN_ENV", "production") + assert _is_production_environment(None) is True + + def test_a_trailing_slash_does_not_change_the_verdict(self): + assert _is_production_environment("https://api.nullrun.io/") is True + + +class TestNoSubstringMatching: + """Each row is a host whose name CONTAINS a marker as a substring.""" + + @pytest.mark.parametrize( + "host", + [ + "latest.nullrun.io", + "greatest.nullrun.io", + "protest.nullrun.io", + "notlocalhost.example.com", + "mytesthost.internal", + "attest.example.com", + "contest.nullrun.io", + ], + ) + def test_a_marker_inside_a_label_is_not_a_dev_marker(self, monkeypatch, host): + monkeypatch.setenv("NULLRUN_ENV", "production") + assert _is_production_environment(f"https://{host}") is True + + @pytest.mark.parametrize( + "host", + [ + "localhost", + "127.0.0.1", + "0.0.0.0", + "foo.localhost", + "staging.internal", + "test.internal", + "staging.example.com", + ], + ) + def test_an_exact_or_labelled_dev_host_is_non_prod(self, monkeypatch, host): + monkeypatch.setenv("NULLRUN_ENV", "production") + assert _is_production_environment(f"https://{host}") is False + + +class TestProdWinsOverUnknownHosts: + """A host the SDK does not know is still production when the + operator says so -- the guard must not stay disarmed because the + SDK failed to recognise the deployment.""" + + def test_an_unknown_host_is_production_when_declared(self, monkeypatch): + monkeypatch.setenv("NULLRUN_ENV", "production") + assert _is_production_environment("https://nullrun.acme.example") is True + + def test_an_unknown_host_is_non_prod_without_a_declaration(self): + """The pre-existing behaviour for a custom host is unchanged: + it needs an explicit declaration to arm the guard.""" + assert _is_production_environment("https://nullrun.acme.example") is False + + +class TestTheGuardRaisesAgainstAProdUrl: + """The integration half: the predicate must arm the real guard.""" + + def test_skip_budget_check_raises_nr_s001_against_a_prod_url( + self, make_runtime, mock_prod_like_api, monkeypatch + ): + from nullrun.breaker.exceptions import NullRunInfrastructureError + + monkeypatch.setenv("NULLRUN_ENV", "production") + rt = make_runtime(api_url=PROD_LIKE_URL) + monkeypatch.setenv("NULLRUN_SKIP_BUDGET_CHECK", "1") + + with pytest.raises(NullRunInfrastructureError) as exc: + rt.check_workflow_budget() + + assert exc.value.error_code == "NR-S001" + + def test_the_sensitive_fail_open_switch_is_refused_against_a_prod_url( + self, make_runtime, mock_prod_like_api, monkeypatch + ): + monkeypatch.setenv("NULLRUN_ENV", "production") + rt = make_runtime(api_url=PROD_LIKE_URL) + monkeypatch.setenv("NULLRUN_SENSITIVE_FAIL_OPEN", "1") + + assert rt.sensitive_fail_open_enabled() is False diff --git a/tests/test_public_exception_surface.py b/tests/test_public_exception_surface.py index b680bfb..a4a4b2f 100644 --- a/tests/test_public_exception_surface.py +++ b/tests/test_public_exception_surface.py @@ -127,12 +127,18 @@ def test_the_scan_actually_found_something(): "declared ok, nothing ran" failure class. Pin a floor at the known size and assert a few names that must be in the set. """ - # 39 = every exception class in exceptions.py + the one in + # 40 = every exception class in exceptions.py + the one in # categories.py. Enums are deliberately not counted (see # `_is_exception_base`), so this number must not be inflated by # them and must not drift upward by accident either. - assert len(RAISABLE) == 39, ( - f"expected the exception scan to find exactly 39 classes, found " + # 40 = 39 + NullRunApprovalToolMismatchError (DEC-5 / DEC-7, + # 2026-11-02 — the approval tool-binding refusal). + # 41 = 40 + NullRunGateIndeterminateError (SDK-11, 2026-11-05 — + # the refusal for a gate decision the SDK could not determine). + # 42 = 41 + NullRunUnknownActionError (SDK-12, 2026-11-05 — the + # refusal for a control-plane action the SDK cannot execute). + assert len(RAISABLE) == 42, ( + f"expected the exception scan to find exactly 42 classes, found " f"{len(RAISABLE)}. The scan is the premise of every test in " f"this module — if it silently stopped matching, they would " f"all pass vacuously." @@ -141,6 +147,8 @@ def test_the_scan_actually_found_something(): "NullRunError", "NullRunApprovalDeniedError", "NullRunUnclassifiedRefusalError", + "NullRunGateIndeterminateError", + "NullRunUnknownActionError", "WorkflowKilledInterrupt", ): assert must_exist in RAISABLE, ( diff --git a/tests/test_route_track_enforcement_propagates.py b/tests/test_route_track_enforcement_propagates.py index 44d7f68..22193a7 100644 --- a/tests/test_route_track_enforcement_propagates.py +++ b/tests/test_route_track_enforcement_propagates.py @@ -68,12 +68,36 @@ ## The fix Re-raise ``NullRunDecision`` after the existing invalidation and -telemetry. Everything else — network errors, 5xx, protocol errors — -keeps the drop-and-log behavior the table documents. +telemetry. + +## SDK-1 (security audit 2026-11-06) widened it + +The rule above was scoped to ``NullRunDecision`` alone, and the scope +was wrong on the money path: a backend refusal +(``NullRunBackendError``) sits under ``NullRunInfrastructureError``, a +SIBLING of ``NullRunDecision``, so ``isinstance(exc, +NullRunDecision)`` never caught a 503 ``RESERVATION_NOT_FOUND`` or a +500 ``INTERNAL_ERROR``. Those are refusals of the CONSUME: the tool +call has already run, the money has already been spent, and the +backend refused to book it. + +The rule is now one question asked of the WIRE: **did the backend +answer?** ADR-008's fail-OPEN covers exactly one thing — the gate +could not be REACHED, so there is no answer to honour and a dead +backend must not freeze the agent loop. A 5xx is an answer, so it +raises. + +So this file's last test changed contract: a bare 5xx used to be +asserted to drop, and that assertion encoded the defect. It now +asserts that the 5xx raises AND that the payload is recoverable in the +WAL — the fail-OPEN half moves to the case it was always about, a +request that never reached the backend. """ from __future__ import annotations +from pathlib import Path + import httpx import pytest import respx @@ -185,6 +209,9 @@ class TestTransportFailureStillDrops: errors — so it must keep dropping. Widening the fix to ``Exception`` would freeze the agent loop on a dead backend, which is the exact failure the fail-OPEN rows exist to prevent. + + "A dead backend" is a request that never got an answer. A 5xx is + an ANSWER, and it is in the other half. """ @respx.mock @@ -197,17 +224,53 @@ def test_connection_error_does_not_raise(self, make_runtime): rt._transport.flush_now() @respx.mock - def test_server_5xx_does_not_raise(self, make_runtime): - """A 5xx names no enforcement failure, so it stays in the - transport class. The ADR-008 text scopes the "raises the - corresponding exception" rule to responses that *name* an - enforcement failure. + def test_a_connection_error_still_leaves_the_payload_recoverable( + self, make_runtime, tmp_path, monkeypatch + ): + """The fail-OPEN half is not a silent drop. + + The backend never answered, so the consume may or may not have + landed. The SDK must not freeze the agent (ADR-008), and it + must not throw the only copy of the spend away either: the + payload goes to the WAL and the next process replays it. """ + wal = tmp_path / "sdk.wal" + monkeypatch.setenv("NULLRUN_WAL_PATH", str(wal)) + + rt = make_runtime() + respx.post(TRACK_URL).mock(side_effect=httpx.ConnectError("boom")) + _capture_smid() + + rt.track_llm(input_tokens=60, output_tokens=40, model="claude-sonnet-4-6") + rt._transport.flush_now() + + rows = [ + line + for line in Path(f"{wal}.v3").read_text().splitlines() + if line.strip() + ] + assert len(rows) == 1, "the undelivered consume was dropped" + + +class TestAnAnswerIsARefusal: + """SDK-1 (security audit 2026-11-06). + + The pre-fix rule asked "does this response NAME an enforcement + failure?". A bare 5xx names none — and it dropped, so the SDK + reported a spend the backend had refused as a successful track. + The rule is now "did the backend ANSWER?", asked of the wire. + """ + + @respx.mock + def test_server_5xx_raises(self, make_runtime): + from nullrun.breaker.exceptions import NullRunBackendError + rt = make_runtime() respx.post(TRACK_URL).mock( return_value=Response(500, json={"error_message": "internal"}) ) _capture_smid() - rt.track_llm(input_tokens=60, output_tokens=40, model="claude-sonnet-4-6") + with pytest.raises(NullRunBackendError): + rt.track_llm(input_tokens=60, output_tokens=40, model="claude-sonnet-4-6") rt._transport.flush_now() diff --git a/tests/test_sdk10_malformed_reservation.py b/tests/test_sdk10_malformed_reservation.py new file mode 100644 index 0000000..4868d80 --- /dev/null +++ b/tests/test_sdk10_malformed_reservation.py @@ -0,0 +1,582 @@ +"""SDK-10 — a reservation id the SDK cannot bind is a REFUSAL, not a warning. + +The audit's subject +------------------- +``_capture_server_minted_execution_id`` dropped a ``reservation_id`` +that is not a UUID with a ``logger.warning`` and returned ``None``. +The audit's own Verify clause: *"a server response with +``reservation_id = "abc"`` ⇒ the SDK must fail closed, not warn and +continue."* + +What the audit got wrong about the CONSEQUENCE +--------------------------------------------- +The audit said the chain cache loses the server-minted +``execution_id`` and later steps reuse the WRONG reservation. +Neither is what happens. The chain cache was deleted by the SDK-1/2/3 +fix (commit 87a4c2b), six hours AFTER the audit document was written, +and the capture's miss branch calls ``clear_server_minted_execution_id()``, +which wipes ALL the capture slots. So a later call finds NOTHING — it +does not find a wrong reservation. + +What IS reachable, and it is worse, is this: the drop read "the server +reserved nothing" when the truth was "the server reserved something I +cannot bind". The call then proceeded with NO reservation: + +* the ``/track`` event had no ``execution_id`` to carry, so the cost + event was DROPPED (``dropped_llm_call_no_reservation``) — the spend + was never booked, and the backend's reserve sat on the period counter + until its TTL ran out; +* the agent got the call for FREE. + +And the shape that reaches it is the LLM-span pre-flight +(``on_llm_start`` → ``check_workflow_budget``), which has no +``/execute`` behind it to recover the verdict. + +Two fixes, because one is not enough +------------------------------------ +1. The capture REFUSES a present-but-unusable id + (``NullRunMalformedGateResponseError`` — the class the SDK already + uses for "something answered, and what it said was not a verdict"). +2. That refusal must REACH the caller. The instrumentation's blanket + arms re-raise the enforcement family and swallow everything else, + so the malformed-answer class joins ``ENFORCEMENT_EXCEPTIONS``. + WITHOUT (2) the refusal is raised and then swallowed one layer up, + and the LLM call proceeds anyway — which is the hole. + +The ABSENT id is a different fact and must NOT be refused: a block +reserves nothing, so a body with no ``reservation_id`` is a fact, not +a fault. Every test below says which of the two it is. + +The counter-test is not optional. A file full of "the SDK refuses" +assertions passes if the SDK refuses EVERYTHING, including the calls +it should have allowed. The last class is that counter-test. +""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest +import respx + +from nullrun.breaker.exceptions import ( + NullRunMalformedGateResponseError, +) +from nullrun.context import ( + clear_server_minted_execution_id, + get_server_minted_execution_id, + set_server_minted_execution_id, +) +from nullrun.observability import metrics +from nullrun.runtime import _capture_server_minted_execution_id + +BASE_URL = "https://api.test.nullrun.io" +GATE_URL = f"{BASE_URL}/api/v1/gate" + +# A real server-minted id: the backend's ``mint_execution_id`` is +# ``Uuid::now_v7()``, and the ``/gate`` Allow arm threads that id into +# ``GateResponse.reservation_id`` (the Step-9 Lua reserve's id). The +# ``/execute`` arm reuses the SAME id (ADR-073). +SERVER_MINTED_UUID = "0195f0c0-0000-7000-8000-000000000001" + + +def _gate_allow(**extra) -> httpx.Response: + """A real ``/gate`` allow. + + ``decision_source``, ``explanation`` and ``policy_version`` are the + non-``Option`` fields on the backend's ``GateResponse`` + (``gate/internal.rs:637-641``); all three are present on every real + answer, and omitting any of them is what makes a body stop being a + verdict. + """ + body = { + "decision": "allow", + "decision_source": "gateway", + "explanation": "within budget", + "policy_version": 1, + } + body.update(extra) + return httpx.Response(200, json=body) + + +@pytest.fixture(autouse=True) +def _clean_capture(): + """The capture writes to contextvars; they outlive a test.""" + clear_server_minted_execution_id() + yield + clear_server_minted_execution_id() + + +@pytest.fixture(autouse=True) +def _clear_deferred(): + """The deferred-enforcement stash is thread-local and outlives a test. + + It is the SDK-4 secondary channel: a refusal that a framework + callback could not abort is stashed, and the ``@protect`` boundary + drains it. A stash left behind by one test is drained by the NEXT + test's ``@protect`` — which would make an unrelated test fail with + a refusal it never caused. + """ + from nullrun.instrumentation.langgraph import drain_deferred_enforcement + + drain_deferred_enforcement() + yield + drain_deferred_enforcement() + + +# --------------------------------------------------------------------------- +# The capture itself — the audit's Verify clause +# --------------------------------------------------------------------------- + + +class TestTheCaptureRefusesAnUnusableReservation: + def test_a_non_uuid_reservation_id_is_refused(self): + """The audit's Verify clause, verbatim. + + "a server response with ``reservation_id = "abc"`` ⇒ the SDK + must fail closed, not warn and continue". + """ + with pytest.raises(NullRunMalformedGateResponseError) as caught: + _capture_server_minted_execution_id({"reservation_id": "abc"}) + + assert "reservation_id" in str(caught.value) + + def test_a_non_string_reservation_id_is_refused(self): + with pytest.raises(NullRunMalformedGateResponseError): + _capture_server_minted_execution_id({"reservation_id": 123456}) + + def test_an_empty_reservation_id_is_refused(self): + """Empty is PRESENT and unusable. + + The old code folded the empty string into the same arm as an + absent id (``if not raw``), so an empty string was read as + "the server reserved nothing". It is the opposite: the server + answered with a reservation id, and that id names nothing. + """ + with pytest.raises(NullRunMalformedGateResponseError): + _capture_server_minted_execution_id({"reservation_id": ""}) + + def test_the_refusal_is_counted(self): + """A refusal nobody can see is a wish, not a control.""" + before = getattr( + metrics.runtime, "gate_malformed_reservation_total", 0 + ) + with pytest.raises(NullRunMalformedGateResponseError): + _capture_server_minted_execution_id({"reservation_id": "abc"}) + after = getattr( + metrics.runtime, "gate_malformed_reservation_total", 0 + ) + assert after == before + 1, ( + "the refusal must be counted — it is the only signal an " + "operator gets that the gate answered with something the " + "SDK could not bind" + ) + + +class TestAnAbsentReservationIsNotARefusal: + """The other half, and the half a block depends on. + + A BLOCK reserves nothing. A body with no ``reservation_id`` says + "the server held no money for this call" — a fact. Refusing it + would turn every block into a protocol error and hide the block. + """ + + def test_a_missing_reservation_id_is_not_a_refusal(self): + assert _capture_server_minted_execution_id({"decision": "block"}) is None + + def test_a_null_reservation_id_is_not_a_refusal(self): + """The backend sometimes sends ``reservation_id: null``.""" + assert _capture_server_minted_execution_id({"reservation_id": None}) is None + + def test_a_non_dict_response_is_not_a_refusal(self): + assert _capture_server_minted_execution_id("not a dict") is None # type: ignore[arg-type] + + def test_an_absent_id_clears_the_previous_one(self): + """The old value must not leak into the next call.""" + set_server_minted_execution_id(SERVER_MINTED_UUID) + assert _capture_server_minted_execution_id({"decision": "block"}) is None + assert get_server_minted_execution_id() is None + + def test_a_real_uuid_is_still_captured(self): + out = _capture_server_minted_execution_id( + {"reservation_id": SERVER_MINTED_UUID} + ) + assert out == SERVER_MINTED_UUID + assert get_server_minted_execution_id() == SERVER_MINTED_UUID + + +# --------------------------------------------------------------------------- +# The whole /gate path, over the wire +# --------------------------------------------------------------------------- + + +class TestTheGatePathRefuses: + def test_the_gate_answer_with_a_malformed_reservation_id_is_refused( + self, make_runtime, mock_api + ): + """End to end over the wire, with the real runtime.""" + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id="abc") + ) + rt = make_runtime() + + with pytest.raises(NullRunMalformedGateResponseError): + rt.check_workflow_budget() + + def test_a_gate_allow_with_no_reservation_id_is_not_a_refusal( + self, make_runtime, mock_api + ): + """The counter-test for the class above. + + A real backend on a path with no reserve (a soft pass, a + workflow with no budget policy) answers without a reservation + id. The SDK must accept that answer, or the gate would refuse + every unbudgeted call in the fleet. + """ + respx.post(GATE_URL).mock(return_value=_gate_allow()) + rt = make_runtime() + + assert rt.check_workflow_budget() is None + + def test_a_gate_allow_with_a_real_reservation_id_still_works( + self, make_runtime, mock_api + ): + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id=SERVER_MINTED_UUID) + ) + rt = make_runtime() + + assert rt.check_workflow_budget() == SERVER_MINTED_UUID + + +# --------------------------------------------------------------------------- +# The money — the LLM-span path +# --------------------------------------------------------------------------- + + +class _RecordingTransport(httpx.BaseTransport): + """An inner transport that records the requests it was asked to send.""" + + def __init__(self, body: bytes = b"{}", status: int = 200) -> None: + self.body = body + self.status = status + self.sent: list[httpx.Request] = [] + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.sent.append(request) + return httpx.Response( + self.status, + content=self.body, + headers={"content-type": "application/json"}, + ) + + +class TestTheLlmCallNeverRuns: + """The hole, at the level the agent sees it. + + The LLM-span pre-flight (``on_llm_start`` → ``check_workflow_budget``) + has NO ``/execute`` behind it, so nothing recovers a lost verdict. + Pre-fix the answer was "warn and continue": the provider call went + out, and its cost event was dropped — the agent got the call for + free. + """ + + def test_the_refusal_reaches_the_caller_through_a_real_callback_manager( + self, make_runtime, mock_api + ): + """A source-pin cannot prove the framework PROPAGATES what the + handler raises — that is langchain-core's contract. So this + drives the real ``CallbackManager``. + """ + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + from nullrun.instrumentation.langgraph import NullRunCallback + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id="abc") + ) + rt = make_runtime() + cb = NullRunCallback(runtime=rt) + + with pytest.raises(NullRunMalformedGateResponseError): + CallbackManager([cb]).on_llm_start({}, ["hi"]) + + def test_the_refusal_reaches_the_caller_on_the_async_path( + self, make_runtime, mock_api + ): + """The async dispatch is a DIFFERENT path in langchain-core.""" + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import AsyncCallbackManager + + from nullrun.instrumentation.langgraph import NullRunCallback + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id="abc") + ) + rt = make_runtime() + cb = NullRunCallback(runtime=rt) + + with pytest.raises(NullRunMalformedGateResponseError): + asyncio.run(AsyncCallbackManager([cb]).on_llm_start({}, ["hi"])) + + def test_the_refusal_is_recorded_for_the_protect_boundary( + self, make_runtime, mock_api + ): + """The secondary channel stays: a host framework that ignores + ``raise_error`` still gets the decision at the ``@protect`` + boundary. + """ + from nullrun.instrumentation.langgraph import ( + NullRunCallback, + drain_deferred_enforcement, + ) + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id="abc") + ) + rt = make_runtime() + cb = NullRunCallback(runtime=rt) + + with pytest.raises(NullRunMalformedGateResponseError): + cb.on_llm_start({}, ["hi"], run_id="run-1") + + deferred = drain_deferred_enforcement() + assert deferred is not None, ( + "the refusal must also be recorded — it is the only channel " + "that can abort on the @protect boundary's own thread" + ) + assert isinstance(deferred, NullRunMalformedGateResponseError) + + +class TestTheProviderRequestNeverLeaves: + """The money, on the zero-code path. + + The httpx transport reserves BEFORE the provider request goes out. + Pre-fix the reservation was dropped, so the request went out + unbilled. Post-fix the request must not go out at all. + """ + + def test_the_provider_request_never_leaves( + self, make_runtime, mock_api + ): + from nullrun.instrumentation.auto import NullRunSyncTransport + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id="abc") + ) + rt = make_runtime() + + inner = _RecordingTransport( + body=b'{"usage":{"prompt_tokens":10,"completion_tokens":5}}' + ) + client = httpx.Client(transport=NullRunSyncTransport(inner, rt)) + + with pytest.raises(NullRunMalformedGateResponseError): + client.post( + "https://api.openai.com/v1/chat/completions", + json={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert inner.sent == [], ( + "the provider request went out with no reservation: the " + "spend was never booked, and the agent got the call for free" + ) + + +class TestTheProtectBoundaryHolds: + def test_the_tool_body_does_not_run(self, make_runtime, mock_api): + """``@protect`` is what users call, so it is the level that counts.""" + from nullrun.decorators import protect + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id="abc") + ) + make_runtime() + ran: list[str] = [] + + @protect + def charge() -> str: + ran.append("body") + return "charged" + + with pytest.raises(NullRunMalformedGateResponseError): + charge() + + assert ran == [], "the decorated body ran on an unbindable answer" + + +# --------------------------------------------------------------------------- +# The fail-OPEN policy is untouched +# --------------------------------------------------------------------------- + + +class TestATransportFailureStillFailsOpen: + """ADR-008: a dead backend must not freeze the agent. + + The refusal above is an ANSWER the SDK cannot use. A dead backend + is the opposite case — no answer at all — and it keeps the + documented fail-OPEN posture. Conflating the two would take every + agent down during an outage. + """ + + def test_the_llm_span_path_still_proceeds_when_the_gate_is_unreachable( + self, make_runtime, mock_api + ): + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + from nullrun.instrumentation.langgraph import NullRunCallback + + respx.post(GATE_URL).mock(return_value=httpx.Response(503, text="down")) + rt = make_runtime() + cb = NullRunCallback(runtime=rt) + + CallbackManager([cb]).on_llm_start({}, ["hi"]) # must not raise + + def test_the_provider_request_still_leaves_when_the_gate_is_unreachable( + self, make_runtime, mock_api + ): + from nullrun.instrumentation.auto import NullRunSyncTransport + + respx.post(GATE_URL).mock(return_value=httpx.Response(503, text="down")) + rt = make_runtime() + + inner = _RecordingTransport( + body=b'{"usage":{"prompt_tokens":10,"completion_tokens":5}}' + ) + client = httpx.Client(transport=NullRunSyncTransport(inner, rt)) + + response = client.post( + "https://api.openai.com/v1/chat/completions", + json={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 200 + assert len(inner.sent) == 1, ( + "a dead backend must not stop the call — ADR-008's fail-OPEN " + "posture is the documented policy for an unreachable gate" + ) + + +class TestTheSiblingHoleIsClosed: + """The same family membership closes a second hole. + + A body that is not JSON at all (a proxy error page, a captive + portal, a TLS-interception box) ALREADY raised + ``NullRunMalformedGateResponseError`` in the runtime. But on the + LLM-span path the instrumentation's blanket arm swallowed it, so + the LLM call proceeded on a non-NULLRUN answer — the same free call, + the same hole, one class away. + """ + + def test_a_non_json_gate_body_is_refused_on_the_llm_span_path( + self, make_runtime, mock_api + ): + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + from nullrun.instrumentation.langgraph import NullRunCallback + + respx.post(GATE_URL).mock( + return_value=httpx.Response(200, text="captive portal") + ) + rt = make_runtime() + cb = NullRunCallback(runtime=rt) + + with pytest.raises(NullRunMalformedGateResponseError): + CallbackManager([cb]).on_llm_start({}, ["hi"]) + + +# --------------------------------------------------------------------------- +# The counter-test: the SDK must not refuse what it can use +# --------------------------------------------------------------------------- + + +class TestAGoodAnswerStillWorks: + def test_a_real_allow_still_captures_its_reservation( + self, make_runtime, mock_api + ): + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id=SERVER_MINTED_UUID) + ) + rt = make_runtime() + + assert rt.check_workflow_budget() == SERVER_MINTED_UUID + assert get_server_minted_execution_id() == SERVER_MINTED_UUID + + def test_a_real_allow_still_runs_the_provider_request( + self, make_runtime, mock_api + ): + from nullrun.instrumentation.auto import NullRunSyncTransport + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id=SERVER_MINTED_UUID) + ) + rt = make_runtime() + + inner = _RecordingTransport( + body=b'{"usage":{"prompt_tokens":10,"completion_tokens":5}}' + ) + client = httpx.Client(transport=NullRunSyncTransport(inner, rt)) + + response = client.post( + "https://api.openai.com/v1/chat/completions", + json={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 200 + assert len(inner.sent) == 1 + + def test_a_real_allow_still_runs_the_tool_body(self, make_runtime, mock_api): + from nullrun.decorators import protect + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id=SERVER_MINTED_UUID) + ) + make_runtime() + ran: list[str] = [] + + @protect + def charge() -> str: + ran.append("body") + return "charged" + + assert charge() == "charged" + assert ran == ["body"], ( + "the SDK refused a call the gate allowed — a suite of " + "refusal assertions passes if the SDK refuses everything" + ) + + +class TestTheFamilyHoldsTheClass: + """The refusal class must be in the enforcement family. + + Without the membership, the instrumentation's blanket arm downgrades + it to a fail-OPEN transport error — the refusal is raised and then + swallowed one layer up. + """ + + def test_the_malformed_answer_class_is_an_enforcement_exception(self): + from nullrun.breaker.exceptions import ENFORCEMENT_EXCEPTIONS + + assert issubclass( + NullRunMalformedGateResponseError, ENFORCEMENT_EXCEPTIONS + ), ( + "the class must be caught by the enforcement arm BEFORE the " + "broad transport arm, or the refusal is downgraded to a " + "fail-OPEN transport error and the call proceeds unbilled" + ) + + def test_a_transport_failure_is_still_not_in_the_family(self): + """ADR-008: the gate could not be reached → fail OPEN.""" + from nullrun.breaker.exceptions import ( + ENFORCEMENT_EXCEPTIONS, + NullRunBackendError, + ) + + assert not isinstance( + NullRunBackendError("boom", endpoint="gate", status_code=500), + ENFORCEMENT_EXCEPTIONS, + ) diff --git a/tests/test_sdk11_gate_indeterminate.py b/tests/test_sdk11_gate_indeterminate.py new file mode 100644 index 0000000..565a1e3 --- /dev/null +++ b/tests/test_sdk11_gate_indeterminate.py @@ -0,0 +1,594 @@ +"""SDK-11 — one implementation, one handler: an indeterminate gate +decision is a REFUSAL, not a permission. + +The audit's subject +------------------- +``RELIABILITY_AUDIT_2026-11-02_EVIDENCE.md`` §SDK-11, verbatim: + + *Location:* ``src/nullrun/runtime.py`` ≈L2664–2672 (the chain arm + catches narrowly) vs ≈L2713–2717 (the single-shot arm catches + ``Exception`` and **fails open**). *Mechanism:* the same check is + implemented twice with different handlers. *Failure:* a single-shot + call gets a fail-open, a chain-call gets a fail-closed — and the + fail-open path is the simpler one, so it is the one users hit + first. *Verify:* an exception inside the gate ⇒ the single-shot + path must refuse. *Fix:* one implementation, one handler. + +What the audit's line numbers actually point at +----------------------------------------------- +They match the tree at ``d71bade`` EXACTLY — and the two arms are NOT +"chain" and "single-shot". They are the **cached** branch's narrow arm +(``except (httpx.HTTPError, NullRunError)``) and the **uncached** +branch's broad arm (``except Exception``). The audit read the two +halves of ONE handler as two implementations. + +The state at HEAD is sharper than the audit's +--------------------------------------------- +SDK-1 (``87a4c2b``) deleted the cache, and with it the narrow twin. +So at HEAD there is ONE implementation and ONE handler — and it is the +BROAD one. The audit's fix ("one implementation, one handler") is +half-done: the duplication is gone, the broad catch-all remains, and +every fault inside the gate call is laundered into "allowed". + +Why "allowed" is the wrong answer +---------------------------------- +ADR-008's fail-OPEN covers ONE thing: the gate could not be REACHED, +so there is no answer to honour and a dead backend must not freeze the +agent. Everything else is a different fact: + +* the gate NAMED a refusal → honour it (block); +* the gate answered with something unusable → refuse (SDK-10); +* the SDK's own code faulted while asking → the decision is + INDETERMINATE, and an unanswered question is not a yes. + +The broad arm could not tell the third apart from the first. It +logged, counted ``gate_fail_open_total``, and returned ``None`` — +which every caller reads as "no decision, carry on". The call then ran +with NO reservation: the backend booked nothing, the cost event was +dropped, and the agent got the call for free. The same shape as the +SDK-10 hole, one class away. + +What this file proves +--------------------- +1. A non-transport fault inside the gate REFUSES, and the refusal is + typed (``NullRunGateIndeterminateError``) and counted. +2. The refusal REACHES the caller through the LangChain callback — + otherwise it is raised and then swallowed one layer up, and the LLM + call proceeds anyway (the SDK-10 hole, re-opened). +3. The provider request never leaves, and the ``@protect`` body never + runs — the money. +4. A genuine transport failure STILL fails OPEN (ADR-008 preserved), + and a refusal the gate NAMED is STILL honoured. + +The last class is the counter-test. A file full of "the SDK refuses" +assertions passes if the SDK refuses everything, including the calls +it should have allowed. +""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest +import respx + +from nullrun.breaker.exceptions import ( + ENFORCEMENT_EXCEPTIONS, + NullRunBackendError, + NullRunGateIndeterminateError, + NullRunTransportError, + RateLimitError, +) +from nullrun.observability import metrics +from nullrun.transport import TransportErrorSource + +BASE_URL = "https://api.test.nullrun.io" +GATE_URL = f"{BASE_URL}/api/v1/gate" + +# A real server-minted id: the backend's ``mint_execution_id`` is +# ``Uuid::now_v7()``, and the ``/gate`` Allow arm threads that id into +# ``GateResponse.reservation_id`` (the Step-9 Lua reserve's id). +SERVER_MINTED_UUID = "0195f0c0-0000-7000-8000-000000000001" + + +def _gate_allow(**extra) -> httpx.Response: + """A real ``/gate`` allow. + + ``decision_source``, ``explanation`` and ``policy_version`` are the + non-``Option`` fields on the backend's ``GateResponse``; all three + are present on every real answer, and omitting any of them is what + makes a body stop being a verdict. + """ + body = { + "decision": "allow", + "decision_source": "gateway", + "explanation": "within budget", + "policy_version": 1, + } + body.update(extra) + return httpx.Response(200, json=body) + + +def _break_the_gate(rt, exc: BaseException) -> None: + """Make the transport's ``check`` raise, as an SDK bug would. + + The seam is deliberate: ``Transport.check`` is the SDK's own code, + and the handler under test is exactly the ``try`` that wraps it. + A fault injected here is what an SDK-internal bug looks like to + the runtime — and it is NOT reachable from the wire today, which + is the point: this class of failure only ever arrives as a bug, + and the guard exists for the day one does. + """ + def _boom(_check_request): + raise exc + + rt._transport.check = _boom # type: ignore[method-assign] + + +@pytest.fixture(autouse=True) +def _no_global_auto_instrumentation(monkeypatch): + """This file must not install the SDK's process-global patches. + + ``@protect`` lazily calls ``auto_instrument(runtime)`` the first time + it resolves a runtime. ``auto_instrument`` patches + ``langchain_core.callbacks.BaseCallbackManager.__init__`` for the + WHOLE PROCESS, and the patch captures the runtime it was given. + Every ``CallbackManager`` built after that — including ones built + by unrelated test files — gets a ``NullRunCallback`` bound to the + captured runtime. + + This file breaks a runtime on purpose (its ``check`` raises), so + if it were the file that installed the patch, the captured runtime + would be the broken one and the NEXT test file to build a + ``CallbackManager`` would explode with this file's injected + ``RuntimeError`` instead of its own. That is a real + order-dependence: the suite's own + ``test_langchain_swallows_callback_exceptions`` fails when this + file runs first. + + Pinning the flag makes this file a bystander: it neither installs + the patch nor tears down one an earlier file installed. The flag is + restored by ``monkeypatch`` afterwards, so the process ends the file + exactly as it started it. + + Nothing in this file needs auto-instrumentation: the transport + under test is built explicitly, and the enforcement path is + exercised directly. + """ + import nullrun.decorators as _dec + + monkeypatch.setattr(_dec, "_auto_instrument_triggered", True, raising=False) + + +@pytest.fixture(autouse=True) +def _clear_deferred(): + """The deferred-enforcement stash is thread-local and outlives a test. + + It is the SDK-4 secondary channel: a refusal that a framework + callback could not abort is stashed, and the ``@protect`` boundary + drains it. A stash left behind by one test is drained by the NEXT + test's ``@protect`` — which would make an unrelated test fail with + a refusal it never caused. + """ + from nullrun.instrumentation.langgraph import drain_deferred_enforcement + + drain_deferred_enforcement() + yield + drain_deferred_enforcement() + + +# --------------------------------------------------------------------------- +# The refusal +# --------------------------------------------------------------------------- + + +class TestAnIndeterminateDecisionRefuses: + """The audit's Verify clause: an exception inside the gate ⇒ refuse.""" + + def test_a_non_transport_fault_refuses(self, make_runtime, mock_api): + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + + with pytest.raises(NullRunGateIndeterminateError): + rt.check_workflow_budget() + + def test_it_is_not_a_silent_none(self, make_runtime, mock_api): + """The pre-fix answer was ``None`` — "no decision, carry on". + + A caller that ignores exceptions would see an allowed call. + The refusal must be an exception, not a value. + """ + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + + with pytest.raises(NullRunGateIndeterminateError): + rt.check_workflow_budget() + + def test_the_refusal_carries_the_cause(self, make_runtime, mock_api): + """The message says the decision was indeterminate; the cause + says WHY. Without it the operator has a refusal and no lead.""" + boom = RuntimeError("the SDK faulted") + rt = make_runtime() + _break_the_gate(rt, boom) + + with pytest.raises(NullRunGateIndeterminateError) as caught: + rt.check_workflow_budget() + + assert caught.value.cause is boom + assert caught.value.__cause__ is boom + + def test_the_refusal_is_counted(self, make_runtime, mock_api): + """A refusal nobody can see is a wish, not a control.""" + before = getattr(metrics.runtime, "gate_indeterminate_total", 0) + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + + with pytest.raises(NullRunGateIndeterminateError): + rt.check_workflow_budget() + + after = getattr(metrics.runtime, "gate_indeterminate_total", 0) + assert after == before + 1, ( + "the refusal must be counted — it is the only signal an " + "operator gets that the SDK is refusing calls" + ) + + def test_it_is_not_counted_as_a_fail_open(self, make_runtime, mock_api): + """Two different facts, two different counters. + + ``gate_fail_open_total`` is the ADR-008 posture: enforcement + was ADVISORY and the call ran. An indeterminate decision does + the opposite — the call is stopped. Counting it as a fail-open + would send an operator to the backend dashboard when the fault + is in the SDK. + """ + before = getattr(metrics.runtime, "gate_fail_open_total", 0) + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + + with pytest.raises(NullRunGateIndeterminateError): + rt.check_workflow_budget() + + after = getattr(metrics.runtime, "gate_fail_open_total", 0) + assert after == before, ( + "an indeterminate decision is a refusal, not a fail-OPEN — " + "the two counters answer different operator questions" + ) + + +# --------------------------------------------------------------------------- +# The money — the LLM-span path +# --------------------------------------------------------------------------- + + +class _RecordingTransport(httpx.BaseTransport): + """An inner transport that records the requests it was asked to send.""" + + def __init__(self, body: bytes = b"{}", status: int = 200) -> None: + self.body = body + self.status = status + self.sent: list[httpx.Request] = [] + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.sent.append(request) + return httpx.Response( + self.status, + content=self.body, + headers={"content-type": "application/json"}, + ) + + +class TestTheProviderRequestNeverLeaves: + """The hole, at the level the agent sees it. + + The LLM-span pre-flight has NO ``/execute`` behind it, so nothing + recovers a lost verdict. Pre-fix the answer was "warn and + continue": the provider call went out, and its cost event was + dropped — the agent got the call for free. + """ + + def test_the_provider_request_never_leaves(self, make_runtime, mock_api): + from nullrun.instrumentation.auto import NullRunSyncTransport + + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + + inner = _RecordingTransport( + body=b'{"usage":{"prompt_tokens":10,"completion_tokens":5}}' + ) + client = httpx.Client(transport=NullRunSyncTransport(inner, rt)) + + with pytest.raises(NullRunGateIndeterminateError): + client.post( + "https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + }, + ) + + assert inner.sent == [], ( + "the provider request went out with no reservation: the " + "spend was never booked, and the agent got the call for free" + ) + + +class TestTheProtectBoundaryHolds: + def test_the_tool_body_does_not_run(self, make_runtime, mock_api): + """``@protect`` is what users call, so it is the level that counts.""" + from nullrun.decorators import protect + + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + ran: list[str] = [] + + @protect + def charge() -> str: + ran.append("body") + return "charged" + + with pytest.raises(NullRunGateIndeterminateError): + charge() + + assert ran == [], "the decorated body ran on an indeterminate decision" + + +# --------------------------------------------------------------------------- +# The callback must not swallow it +# --------------------------------------------------------------------------- + + +class TestTheRefusalReachesTheCaller: + """Without this, the refusal is raised and then swallowed one layer + up — and the LLM call proceeds anyway. + + ``instrumentation/langgraph.py`` re-raises only the enforcement + family and swallows everything else at debug level. A refusal class + that is not in the family is a refusal nobody sees. + """ + + def test_it_escapes_a_real_callback_manager(self, make_runtime, mock_api): + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + from nullrun.instrumentation.langgraph import NullRunCallback + + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + cb = NullRunCallback(runtime=rt) + + with pytest.raises(NullRunGateIndeterminateError): + CallbackManager([cb]).on_llm_start({}, ["hi"]) + + def test_it_escapes_the_async_dispatch(self, make_runtime, mock_api): + """The async dispatch is a DIFFERENT path in langchain-core.""" + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import AsyncCallbackManager + + from nullrun.instrumentation.langgraph import NullRunCallback + + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + cb = NullRunCallback(runtime=rt) + + with pytest.raises(NullRunGateIndeterminateError): + asyncio.run(AsyncCallbackManager([cb]).on_llm_start({}, ["hi"])) + + def test_it_is_recorded_for_the_protect_boundary(self, make_runtime, mock_api): + """The secondary channel stays: a host framework that ignores + ``raise_error`` still gets the decision at the ``@protect`` + boundary. + """ + from nullrun.instrumentation.langgraph import ( + NullRunCallback, + drain_deferred_enforcement, + ) + + rt = make_runtime() + _break_the_gate(rt, RuntimeError("the SDK faulted")) + cb = NullRunCallback(runtime=rt) + + with pytest.raises(NullRunGateIndeterminateError): + cb.on_llm_start({}, ["hi"], run_id="run-1") + + deferred = drain_deferred_enforcement() + assert deferred is not None, ( + "the refusal must also be recorded — it is the only channel " + "that can abort on the @protect boundary's own thread" + ) + assert isinstance(deferred, NullRunGateIndeterminateError) + + +# --------------------------------------------------------------------------- +# ADR-008 is preserved +# --------------------------------------------------------------------------- + + +class TestATransportFailureStillFailsOpen: + """A dead backend must not freeze the agent. + + The refusal above is a fault in the SDK's own code. A dead backend + is the opposite case — no answer at all — and it keeps the + documented fail-OPEN posture. Conflating the two would take every + agent down during an outage. + """ + + def test_a_network_failure_still_fails_open(self, make_runtime, mock_api): + rt = make_runtime() + _break_the_gate(rt, httpx.ConnectError("connection refused")) + + assert rt.check_workflow_budget() is None + + def test_a_transport_error_still_fails_open(self, make_runtime, mock_api): + rt = make_runtime() + _break_the_gate( + rt, + NullRunTransportError( + "gateway down", + source=TransportErrorSource.NETWORK_ERROR, + endpoint="check", + ), + ) + + assert rt.check_workflow_budget() is None + + def test_a_gateway_5xx_still_fails_open(self, make_runtime, mock_api): + rt = make_runtime() + _break_the_gate( + rt, + NullRunBackendError("boom", endpoint="check", status_code=502), + ) + + assert rt.check_workflow_budget() is None + + def test_the_llm_span_path_still_proceeds_when_the_gate_is_unreachable( + self, make_runtime, mock_api + ): + pytest.importorskip("langchain_core") + from langchain_core.callbacks.manager import CallbackManager + + from nullrun.instrumentation.langgraph import NullRunCallback + + rt = make_runtime() + _break_the_gate(rt, httpx.ConnectError("connection refused")) + cb = NullRunCallback(runtime=rt) + + CallbackManager([cb]).on_llm_start({}, ["hi"]) # must not raise + + def test_the_provider_request_still_leaves_when_the_gate_is_unreachable( + self, make_runtime, mock_api + ): + from nullrun.instrumentation.auto import NullRunSyncTransport + + rt = make_runtime() + _break_the_gate(rt, httpx.ConnectError("connection refused")) + + inner = _RecordingTransport( + body=b'{"usage":{"prompt_tokens":10,"completion_tokens":5}}' + ) + client = httpx.Client(transport=NullRunSyncTransport(inner, rt)) + + response = client.post( + "https://api.openai.com/v1/chat/completions", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + }, + ) + + assert response.status_code == 200 + assert len(inner.sent) == 1, ( + "a dead backend must not stop the call — ADR-008's fail-OPEN " + "posture is the documented policy for an unreachable gate" + ) + + +# --------------------------------------------------------------------------- +# A refusal the gate NAMED is still honoured +# --------------------------------------------------------------------------- + + +class TestANamedRefusalIsNotDowngraded: + """The family arm comes FIRST, and the ordering is load-bearing. + + ``RateLimitError`` is a ``NullRunTransportError``. If the transport + arm came first, a 429 would be swallowed by it and fail OPEN — the + exact hole ADR-062 and SDK-4 closed. + """ + + def test_a_rate_limit_refusal_is_not_downgraded_to_a_fail_open( + self, make_runtime, mock_api + ): + rt = make_runtime() + _break_the_gate( + rt, + RateLimitError( + "too many calls", + source=TransportErrorSource.NETWORK_ERROR, + endpoint="check", + ), + ) + + with pytest.raises(RateLimitError): + rt.check_workflow_budget() + + def test_a_budget_refusal_is_not_downgraded_to_a_fail_open( + self, make_runtime, mock_api + ): + from nullrun.breaker.exceptions import NullRunBudgetError + + rt = make_runtime() + _break_the_gate( + rt, + NullRunBudgetError(workflow_id="w1", reason="budget exhausted"), + ) + + with pytest.raises(NullRunBudgetError): + rt.check_workflow_budget() + + +# --------------------------------------------------------------------------- +# The counter-test: the SDK must not refuse what it can use +# --------------------------------------------------------------------------- + + +class TestAGoodAnswerStillWorks: + def test_a_real_allow_still_works(self, make_runtime, mock_api): + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id=SERVER_MINTED_UUID) + ) + rt = make_runtime() + + assert rt.check_workflow_budget() == SERVER_MINTED_UUID + + def test_a_real_allow_still_runs_the_tool_body(self, make_runtime, mock_api): + from nullrun.decorators import protect + + respx.post(GATE_URL).mock( + return_value=_gate_allow(reservation_id=SERVER_MINTED_UUID) + ) + make_runtime() + ran: list[str] = [] + + @protect + def charge() -> str: + ran.append("body") + return "charged" + + assert charge() == "charged" + assert ran == ["body"], ( + "the SDK refused a call the gate allowed — a suite of " + "refusal assertions passes if the SDK refuses everything" + ) + + +# --------------------------------------------------------------------------- +# The family holds the class +# --------------------------------------------------------------------------- + + +class TestTheFamilyHoldsTheClass: + """Without the membership, the instrumentation's blanket arm + downgrades the refusal to a swallowed debug line — the refusal is + raised and then thrown away one layer up. + """ + + def test_the_indeterminate_class_is_an_enforcement_exception(self): + assert issubclass(NullRunGateIndeterminateError, ENFORCEMENT_EXCEPTIONS), ( + "the class must be caught by the enforcement arm BEFORE the " + "transport arm, or the refusal is downgraded to a fail-OPEN " + "transport error and the call proceeds unbilled" + ) + + def test_it_is_not_a_transport_error(self): + """ADR-126's two-step opt-in opens TRANSPORT errors. + + An indeterminate decision must not be convertible into "the + body runs" by flipping a switch. Deriving from + ``NullRunTransportError`` would hand it to the ``@protect`` + boundary's transport arm, which that switch can open. + """ + assert not issubclass(NullRunGateIndeterminateError, NullRunTransportError), ( + "the indeterminate refusal must not be a transport error — " + "that is the one class ADR-126's opt-in can open" + ) diff --git a/tests/test_sdk12_unknown_action.py b/tests/test_sdk12_unknown_action.py new file mode 100644 index 0000000..8463f5e --- /dev/null +++ b/tests/test_sdk12_unknown_action.py @@ -0,0 +1,384 @@ +"""SDK-12 — an unknown action type is REFUSED, not silently returned. + +The audit entry, verbatim: + + **SDK-12 — an unknown action type is logged and returned.** + *Location:* ``src/nullrun/actions.py`` (unknown action type ⇒ logged + and returned; the code's own comment calls it fail-open). + *Failure:* an action the SDK does not know is **not executed and not + refused** — the caller believes it ran. ``except BaseException`` + also swallows ``KeyboardInterrupt``/``SystemExit``. + *Verify:* an unknown action type must raise. *Fix:* raise; narrow + the handler. + +What these tests pin +-------------------- +1. An unknown action type RAISES a typed error, so the caller can tell + "it ran" from "it could not run". +2. The raise is NOT a block — B14's anti-DoS property survives (an + unrecognised control-plane value must not stop the workflow). +3. The raise is NOT a transport error — ADR-126's two-step opt-in must + not be able to convert it into "the body runs". +4. ``except BaseException`` is gone: KeyboardInterrupt / SystemExit + reach the caller. +5. A handler's OWN refusal (kill / pause / block) is still swallowed + and logged at INFO; a handler FAULT is logged at ERROR. +6. At the only production call site (``/track/batch``) the raise is + reported as a control-plane integrity event, not as a "malformed + action", and the rest of the batch still runs. +""" + +from __future__ import annotations + +import logging + +import pytest +import respx +from httpx import Response + +import nullrun.actions as _act +import nullrun.decorators as _dec +from nullrun.actions import ActionHandler, ActionType +from nullrun.breaker.exceptions import ( + NullRunBlockedException, + NullRunTransportError, + NullRunUnknownActionError, +) +from nullrun.observability import metrics +from nullrun.runtime import NullRunRuntime + +BASE_URL = "https://api.test.nullrun.io" + + +# --------------------------------------------------------------------------- +# 1. The unknown action type raises +# --------------------------------------------------------------------------- + + +def test_an_unknown_action_type_raises(): + """The audit's Verify clause, verbatim: it must raise.""" + h = ActionHandler() + with pytest.raises(NullRunUnknownActionError) as info: + h.handle("totally_made_up_action", "wf-mystery", "test reason") + assert "totally_made_up_action" in str(info.value) + + +def test_the_unknown_action_is_not_a_block(): + """B14's property survives: an unknown type must not stop the workflow. + + Pre-B14 the unknown type degraded to ``ActionType.BLOCK`` and the + SDK became a DoS amplifier. The raise must therefore NOT be a + ``NullRunBlockedException`` — an infrastructure error is the right + shape: the action did not run, but the workflow is not blocked. + """ + h = ActionHandler() + with pytest.raises(NullRunUnknownActionError) as info: + h.handle("weird", "wf-1", reason="x") + assert not isinstance(info.value, NullRunBlockedException), ( + "the unknown-action refusal must not be a block — that was the " + "DoS amplifier B14 removed. An unrecognised control-plane value " + "must not stop every subsequent tool call in the workflow." + ) + + +def test_the_unknown_action_is_not_a_transport_error(): + """ADR-126's two-step opt-in opens TRANSPORT errors only. + + "The body runs" is not an acceptable answer to "the control plane + sent an instruction this SDK cannot execute". + """ + assert not issubclass(NullRunUnknownActionError, NullRunTransportError) + + +def test_the_unknown_action_invokes_no_handler(): + """No handler runs — the action is not executed.""" + h = ActionHandler() + handler_mock = pytest.importorskip("unittest.mock") + mock = handler_mock.MagicMock() + h.register_handler(ActionType.BLOCK, mock) + h.register_handler(ActionType.ALERT, mock) + with pytest.raises(NullRunUnknownActionError): + h.handle("nope", "wf-1", reason="x") + mock.assert_not_called() + + +def test_the_unknown_action_is_recorded_forensically(): + """The history entry survives — the operator can correlate the log.""" + h = ActionHandler() + with pytest.raises(NullRunUnknownActionError): + h.handle("not_a_real_action", "wf-mystery", "real reason") + history = h.get_action_history() + assert len(history) == 1 + assert "unknown_action_type:not_a_real_action" in history[0].reason + + +def test_the_unknown_action_is_counted(): + """The counter is the operator's signal that server and SDK drifted.""" + before = metrics.runtime.unknown_action_total + h = ActionHandler() + with pytest.raises(NullRunUnknownActionError): + h.handle("counted", "wf-1", reason="x") + assert metrics.runtime.unknown_action_total == before + 1 + + +def test_the_unknown_action_carries_its_context(): + """The error names the action and the workflow — it is the only handle.""" + h = ActionHandler() + with pytest.raises(NullRunUnknownActionError) as info: + h.handle("mystery", "wf-42", reason="because") + exc = info.value + assert exc.action == "mystery" + assert exc.workflow_id == "wf-42" + assert exc.reason == "because" + assert exc.error_code == "NR-P005" + + +def test_a_known_action_after_an_unknown_one_still_runs(): + """One bad action must not poison the dispatcher.""" + h = ActionHandler() + with pytest.raises(NullRunUnknownActionError): + h.handle("malformed_first", "wf-mix", "first") + h.handle(ActionType.KILL, "wf-mix", "second") + history = h.get_action_history() + assert len(history) == 2 + assert history[0].reason == "unknown_action_type:malformed_first" + assert history[1].reason == "second" + + +# --------------------------------------------------------------------------- +# 2. The handler arm is narrowed +# --------------------------------------------------------------------------- + + +def test_keyboard_interrupt_is_not_swallowed(): + """``except BaseException`` is gone — a Ctrl-C must reach the caller.""" + h = ActionHandler() + + def boom(workflow_id, reason, **details): + raise KeyboardInterrupt + + h.register_handler(ActionType.ALERT, boom) + with pytest.raises(KeyboardInterrupt): + h.handle("alert", "wf-1", reason="x") + + +def test_system_exit_is_not_swallowed(): + """Same for SystemExit.""" + h = ActionHandler() + + def boom(workflow_id, reason, **details): + raise SystemExit(2) + + h.register_handler(ActionType.ALERT, boom) + with pytest.raises(SystemExit): + h.handle("alert", "wf-1", reason="x") + + +def test_the_handlers_own_refusal_is_swallowed_and_logged_at_info(caplog): + """A kill is the action's OUTCOME, not a fault in the handler. + + Pre-SDK-12 the kill refusal was logged at ERROR, so every + server-side kill raised a false alarm on the ERROR channel. + """ + h = ActionHandler() + with caplog.at_level(logging.INFO, logger="nullrun.actions"): + h.handle("kill", "wf-1", reason="budget") # must not raise + assert any(r.levelno >= logging.ERROR for r in caplog.records) is False, ( + "a delivered kill must not be logged as an ERROR — it is the " + "action working as designed" + ) + assert any("kill" in r.getMessage() for r in caplog.records) + + +def test_a_handler_fault_is_still_logged_at_error(caplog): + """A buggy custom handler stays on the ERROR channel.""" + h = ActionHandler() + + def boom(workflow_id, reason, **details): + raise RuntimeError("oops") + + h.register_handler(ActionType.ALERT, boom) + with caplog.at_level(logging.INFO, logger="nullrun.actions"): + h.handle("alert", "wf-1", reason="x") # must not raise + assert any(r.levelno >= logging.ERROR for r in caplog.records) + + +def test_the_kill_is_still_recorded_in_history(): + """The swallow must not cost the audit entry.""" + h = ActionHandler() + h.handle("kill", "wf-1", reason="x") + assert any(e.action_type == "kill" for e in h.get_action_history()) + + +# --------------------------------------------------------------------------- +# 3. The public helper +# --------------------------------------------------------------------------- + + +def test_the_public_helper_raises_for_an_unknown_type(): + """``nullrun.handle_action`` is the public door — it must not lie.""" + import nullrun + + with pytest.raises(NullRunUnknownActionError): + nullrun.handle_action("bogus", "wf-1", "x") + + +# --------------------------------------------------------------------------- +# 4. The only production call site: /track/batch +# --------------------------------------------------------------------------- + + +@pytest.fixture +def mock_api(): + with respx.mock: + respx.post(f"{BASE_URL}/api/v1/auth/verify").mock( + return_value=Response( + 200, + json={ + "organization_id": "ws-test", + "workflow_id": "00000000-0000-0000-0000-000000000001", + "plan": "pro", + "features": [], + "limits": {"max_cost_cents": 10000}, + }, + ) + ) + yield + + +def _runtime(api_key: str = "test-key"): + NullRunRuntime._instance = None + _dec._runtime = None + _act._action_handler = None + rt = NullRunRuntime( + api_key=api_key, + api_url=BASE_URL, + debug=True, + polling=False, + ) + NullRunRuntime._instance = rt + _dec._runtime = rt + return rt + + +def _batch_response(actions): + return Response( + 200, + json={ + "processed": 1, + "actions": actions, + "messages": [], + "snapshots": [], + "accepted_event_ids": ["e1"], + "rejected_count": 0, + "rejection_details": [], + }, + ) + + +def _send_one_batch(rt): + return rt._transport._send_batch_with_retry_info( + batch=[ + { + "event_type": "llm_call", + "workflow_id": "wf-1", + "model": "gpt-4", + "tokens": 100, + "cost_cents": 1, + } + ] + ) + + +def test_an_unknown_action_in_a_batch_is_reported_as_an_integrity_event( + mock_api, caplog +): + """Not "skipped malformed action" — that mislabelling is the defect. + + The action was not executed. Filing that under "malformed payload" + hides a server/SDK version mismatch behind a WARNING. + + The assertion is on the TRANSPORT's own log line, at ANY level: the + generic arm's message must not appear for this action at all, and + the transport's integrity-event line must appear at ERROR. (An + earlier version of this test asserted only "some ERROR mentions the + action type" — the dispatcher's own ERROR log satisfies that, so + the test passed even with the transport fix reverted. That was a + vacuous pass; it is pinned by the mutation in the SDK-12 ADR.) + """ + rt = _runtime() + route = respx.post(f"{BASE_URL}/api/v1/track/batch").mock( + return_value=_batch_response( + [{"type": "throttle", "workflow_id": "wf-1", "reason": "x"}] + ) + ) + with caplog.at_level(logging.INFO): + _send_one_batch(rt) + + assert route.called + messages = [r.getMessage() for r in caplog.records] + assert not any("Skipping malformed action" in m for m in messages), ( + "a control-plane integrity event must not be filed as a " + f"malformed payload. Log lines: {messages}" + ) + transport_errors = [ + r.getMessage() + for r in caplog.records + if r.levelno >= logging.ERROR + and "Control plane sent an action type" in r.getMessage() + ] + assert transport_errors, ( + "the /track/batch call site must report the unknown action as an " + f"integrity event at ERROR. Log lines: {messages}" + ) + assert any("throttle" in m for m in transport_errors) + + +def test_one_unknown_action_does_not_abort_the_rest_of_the_batch(mock_api): + """The loop continues: the next action is still dispatched.""" + rt = _runtime() + seen: list[tuple[str, str]] = [] + + def alert_handler(workflow_id, reason, **details): + seen.append((workflow_id, reason)) + + _act.get_action_handler().register_handler(ActionType.ALERT, alert_handler) + + respx.post(f"{BASE_URL}/api/v1/track/batch").mock( + return_value=_batch_response( + [ + {"type": "throttle", "workflow_id": "wf-1", "reason": "x"}, + {"type": "alert", "workflow_id": "wf-2", "reason": "y"}, + ] + ) + ) + _send_one_batch(rt) + + assert seen == [("wf-2", "y")], ( + "the known action after the unknown one must still be dispatched" + ) + + +def test_a_malformed_action_still_logs_the_warning(mock_api, caplog): + """The generic arm keeps its own label — only the integrity event moved.""" + rt = _runtime() + respx.post(f"{BASE_URL}/api/v1/track/batch").mock( + return_value=_batch_response(["not-a-dict"]) + ) + with caplog.at_level(logging.INFO): + _send_one_batch(rt) + assert any( + "non-dict action" in r.getMessage() for r in caplog.records + ), "the malformed-payload arm must keep its own signal" + + +def test_the_batch_still_succeeds_with_an_unknown_action(mock_api): + """The cost events were accepted — the batch must not be reported as failed.""" + rt = _runtime() + respx.post(f"{BASE_URL}/api/v1/track/batch").mock( + return_value=_batch_response( + [{"type": "throttle", "workflow_id": "wf-1", "reason": "x"}] + ) + ) + result = _send_one_batch(rt) + assert result.accepted_event_ids == ["e1"] diff --git a/tests/test_sdk13_decorator_policy_gate_indeterminate.py b/tests/test_sdk13_decorator_policy_gate_indeterminate.py new file mode 100644 index 0000000..1c7e70b --- /dev/null +++ b/tests/test_sdk13_decorator_policy_gate_indeterminate.py @@ -0,0 +1,481 @@ +"""SDK-13 — an SDK-internal fault in the @protect policy gate is a REFUSAL. + +The audit entry, verbatim: + + **SDK-13 — the decorator reports "policy engine unavailable" and + proceeds.** *Location:* ``src/nullrun/decorators.py`` ≈L1082–1108 + (broad ``except Exception`` ⇒ ``NR-B001`` / "policy engine + unavailable"). *Failure:* an engine error is converted into a + pass. *Verify:* an exception inside the policy engine must refuse. + *Fix:* re-raise, or make the error surface as a distinct refusal + code. + +What the audit got right and what it got wrong +---------------------------------------------- + +**Right:** the site is a broad ``except Exception``, and the refusal it +produced was not a *distinct* refusal — it wore the network code +``NR-B001`` and the policy-block type. + +**Wrong:** "an engine error is converted into a pass" was FALSE for the +default path. The arm already raised ``NullRunBlockedException``, so the +decorated body did not run. The audit read the ``if fail_open: return`` +arm — the documented dev/test opt-out — as the default. + +**What was actually true, and is worse:** the arm's fail-OPEN branch is +reachable ONLY for SDK-internal faults. The ``/execute`` path converts +every transport failure into ``NullRunTransportError`` (a class the arm +above owns), so what reaches the catch-all is a fault in the SDK's OWN +code. The two-step opt-out is documented as covering "the policy engine +is UNREACHABLE"; under it, an SDK bug ran the body — which is exactly +the "converted into a pass" the audit describes, reached through the +only path where it can happen. + +⇒ The defect was the LABEL, not the absence of a refusal: an SDK fault +was reported as a *network* error (``NR-B001``) and typed as a *policy +decision* (``NullRunBlockedException`` ⊂ ``NullRunDecision``), so the +FastAPI integration rendered it **HTTP 429 / category "decision"** — the +operator was told the policy refused (or rate-limited) when the truth is +that the SDK has a bug. + +This is the /execute twin of ADR-129's row 29 on the /gate side: same +fact, same class (``NullRunGateIndeterminateError`` / ``NR-P004``), same +counter discipline. + +What these tests pin +-------------------- + +1. The audit's Verify clause, literally: an exception inside the policy + engine refuses, and the body does not run. +2. The refusal is TYPED as an indeterminate decision (``NR-P004``), not + as a network error and not as a policy decision. +3. The two-step opt-out does NOT open an SDK-internal fault — and it + still DOES open an unreachable engine (the documented bypass + survives; this is the control that stops the fix from over-reaching). +4. The refusal is counted, logged at ERROR, and carries the original + exception as its cause. +5. On the wire (FastAPI) it is a 503 / "infrastructure", not a 429 / + "decision". +6. A decision the gateway NAMED is still honoured as a decision. +""" + +from __future__ import annotations + +import dataclasses +import logging + +import httpx +import pytest +import respx + +from nullrun import messages +from nullrun.breaker.exceptions import ( + NullRunBlockedException, + NullRunDecision, + NullRunGateIndeterminateError, + NullRunInfrastructureError, + NullRunTransportError, +) +from nullrun.observability import metrics + +BASE_URL = "https://api.test.nullrun.io" +EXECUTE_URL = f"{BASE_URL}/api/v1/execute" + +_FLAG = "NULLRUN_SENSITIVE_FAIL_OPEN" +_ACK = "NULLRUN_ALLOW_SENSITIVE_FAIL_OPEN" + + +@pytest.fixture(autouse=True) +def _clean_flag_env(monkeypatch): + """Both vars start unset, so a developer's shell cannot decide + whether these tests pass.""" + monkeypatch.delenv(_FLAG, raising=False) + monkeypatch.delenv(_ACK, raising=False) + monkeypatch.delenv("NULLRUN_ENV", raising=False) + monkeypatch.delenv("NULLRUN_API_URL", raising=False) + + +def _boom(*_args, **_kwargs): + """An SDK-internal fault: not a transport error, not a decision. + + This is the shape the catch-all is for — the SDK's own code faulted + while asking the policy engine a question. A ``TypeError`` is used + because it is the shape a real SDK bug has: nothing in the SDK + raises it on purpose. + """ + raise TypeError("the request body could not be serialised") + + +def _install_fault(rt, monkeypatch): + """Put the fault at the seam the decorator's ``try`` wraps. + + ``runtime.execute`` only catches ``NullRunBlockedException`` around + the transport call, so the fault reaches the decorator exactly as + an in-SDK fault would. + """ + monkeypatch.setattr(rt._transport, "execute", _boom) + + +def _protected(): + """A protected function plus the list that records whether it ran.""" + from nullrun.decorators import protect + + ran: list[int] = [] + + @protect + def charge_card(amount: int) -> str: + ran.append(amount) + return "charged" + + return charge_card, ran + + +# --------------------------------------------------------------------------- +# 1. The audit's Verify clause +# --------------------------------------------------------------------------- + + +def test_an_exception_inside_the_policy_engine_refuses(make_runtime, monkeypatch): + """The audit's clause, verbatim: it must refuse.""" + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, ran = _protected() + + with pytest.raises(Exception) as info: + charge_card(100) + + assert ran == [], "the body ran — the engine error became a pass" + assert info.value is not None + + +def test_the_refusal_carries_the_original_exception_as_its_cause( + make_runtime, monkeypatch +): + """The cause is what makes the refusal actionable.""" + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError) as info: + charge_card(100) + + assert isinstance(info.value.cause, TypeError) + assert "could not be serialised" in str(info.value.cause) + assert info.value.__cause__ is info.value.cause + + +# --------------------------------------------------------------------------- +# 2. The refusal is a DISTINCT refusal +# --------------------------------------------------------------------------- + + +def test_the_refusal_is_the_indeterminate_class(make_runtime, monkeypatch): + """``NR-P004`` — the same class the /gate side uses (ADR-129).""" + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError) as info: + charge_card(100) + + assert info.value.error_code == "NR-P004" + + +def test_the_refusal_is_not_reported_as_a_network_error(make_runtime, monkeypatch): + """Pre-fix it wore ``NR-B001`` — a NETWORK code. + + Every network failure is classified by the arm above (network / + gateway / auth / breaker-open), so a fault that reaches the + catch-all is by construction NOT a network failure. Reporting it + as one sends the operator to the network. + """ + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError) as info: + charge_card(100) + + assert info.value.error_code != "NR-B001" + + +def test_the_refusal_is_not_a_policy_decision(make_runtime, monkeypatch): + """A decision reads as "the policy engine refused this". + + The SDK could not obtain a decision at all — the workflow is not + blocked BY POLICY, and a cookbook branch on + ``except NullRunDecision`` must not treat an SDK bug as a verdict. + """ + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError) as info: + charge_card(100) + + assert not isinstance(info.value, NullRunDecision) + assert not isinstance(info.value, NullRunBlockedException) + + +def test_the_refusal_is_not_a_transport_error(make_runtime, monkeypatch): + """ADR-126's two-step opt-out opens TRANSPORT errors only. + + "The body runs" is not an acceptable answer to "the SDK could not + determine the answer". + """ + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError) as info: + charge_card(100) + + assert not isinstance(info.value, NullRunTransportError) + assert isinstance(info.value, NullRunInfrastructureError) + + +# --------------------------------------------------------------------------- +# 3. The opt-out does not open an SDK bug — but still opens an outage +# --------------------------------------------------------------------------- + + +def test_the_opt_out_does_not_open_an_sdk_internal_fault(make_runtime, monkeypatch): + """The audit's "converted into a pass", reached the only way it can. + + The two-step opt-out is documented as covering "the policy engine is + UNREACHABLE". An SDK bug is a different fact: nothing downstream + corrects it, and ``/track`` does not gate on the policy answer. + Pre-fix this arm returned and the body ran. + """ + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + monkeypatch.setenv(_FLAG, "1") + _install_fault(rt, monkeypatch) + charge_card, ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError): + charge_card(100) + + assert ran == [], ( + "the opt-out opened an SDK-internal fault — the body ran with no " + "policy decision" + ) + + +def test_the_opt_out_still_opens_an_unreachable_engine(make_runtime, monkeypatch): + """The control: the documented dev/test bypass must survive. + + A transport failure is the case the opt-out exists for: the engine + could not be REACHED, so there is no answer to honour. This test + is what stops the fix above from over-reaching into the bypass. + """ + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + monkeypatch.setenv(_FLAG, "1") + + respx.post(EXECUTE_URL).mock(side_effect=httpx.ConnectError("connection refused")) + charge_card, ran = _protected() + + charge_card(100) + + assert ran == [100], ( + "the documented dev/test bypass stopped working — the fix must " + "not touch the TRANSPORT failure path" + ) + + +def test_an_unreachable_engine_without_the_opt_out_still_refuses( + make_runtime, monkeypatch +): + """The control for the other side: no opt-out, no body.""" + make_runtime(api_url=BASE_URL) + + respx.post(EXECUTE_URL).mock(side_effect=httpx.ConnectError("connection refused")) + charge_card, ran = _protected() + + with pytest.raises(NullRunBlockedException): + charge_card(100) + + assert ran == [] + + +# --------------------------------------------------------------------------- +# 4. The refusal is visible +# --------------------------------------------------------------------------- + + +def test_the_refusal_is_counted(make_runtime, monkeypatch): + """A refusal that is not counted is invisible to the operator.""" + before = metrics.runtime.policy_gate_indeterminate_total + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError): + charge_card(100) + + assert metrics.runtime.policy_gate_indeterminate_total == before + 1 + + +def test_the_refusal_is_logged_at_error(make_runtime, monkeypatch, caplog): + """An SDK fault is a fault: it belongs on the ERROR channel.""" + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with caplog.at_level(logging.INFO, logger="nullrun.decorators"): + with pytest.raises(NullRunGateIndeterminateError): + charge_card(100) + + errors = [ + r.getMessage() + for r in caplog.records + if r.levelno >= logging.ERROR and "charge_card" in r.getMessage() + ] + assert errors, ( + "the refusal must be logged at ERROR with the tool name — " + f"records: {[r.getMessage() for r in caplog.records]}" + ) + + +def test_the_refusal_fires_the_error_hook(make_runtime, monkeypatch): + """The hook is how an operator's agent learns the call was stopped.""" + import nullrun + + seen: list[tuple[object, object]] = [] + unregister = nullrun.on_error(lambda err, ctx: seen.append((err, ctx))) + + try: + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + with pytest.raises(NullRunGateIndeterminateError): + charge_card(100) + finally: + unregister() + + assert seen, "the error hook did not fire for the refusal" + err, ctx = seen[-1] + assert isinstance(err, NullRunGateIndeterminateError) + assert getattr(ctx, "tool_name", None) == "charge_card", ( + "the hook context must name the tool — the refusal class carries " + "no tool_name field of its own" + ) + + +# --------------------------------------------------------------------------- +# 5. On the wire +# --------------------------------------------------------------------------- + + +def test_the_refusal_is_a_503_not_a_429(make_runtime, monkeypatch): + """The mislabelling, made visible. + + Pre-fix the refusal was a ``NullRunBlockedException`` with + ``NR-B001``: the FastAPI integration mapped it through + ``_DECISION_STATUS`` and, finding no entry, fell back to the + generic decision status — **429 Too Many Requests**, category + "decision". The operator was told the request was rate-limited. + """ + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from nullrun.integrations import fastapi as nr_fastapi + + rt = make_runtime(api_url=BASE_URL) + _install_fault(rt, monkeypatch) + charge_card, _ran = _protected() + + app = FastAPI() + nr_fastapi.install(app) + + @app.get("/charge") + def charge(): + return charge_card(100) + + client = TestClient(app, raise_server_exceptions=False) + resp = client.get("/charge") + + assert resp.status_code == 503, ( + "an indeterminate decision is an infrastructure failure, not a " + f"rate limit — got {resp.status_code}" + ) + body = resp.json() + assert body["error_code"] == "NR-P004" + assert body["category"] == "infrastructure" + assert body["retryable"] is True + + +def test_the_refusal_has_an_operator_message(): + """Not the generic fallback. + + ``NR-P004`` had NO entry in ``DEFAULT_MESSAGES``, so the operator + saw "Something went wrong. Please try again." for the one failure + that STOPS the call. + """ + err = NullRunGateIndeterminateError("no decision") + out = messages.format_user_message(err) + + assert out == messages.DEFAULT_MESSAGES["NR-P004"] + assert out != messages.FALLBACK_MESSAGE + + +# --------------------------------------------------------------------------- +# 6. The decision path is untouched +# --------------------------------------------------------------------------- + + +def test_a_gateway_block_is_still_a_decision(make_runtime): + """The control: a decision the gateway NAMED is still a decision.""" + make_runtime(api_url=BASE_URL) + + respx.post(EXECUTE_URL).mock( + return_value=httpx.Response( + 402, + json={ + "error_code": "BUDGET_HARD_BLOCKED", + "error_message": "the period budget is exhausted", + "details": {"error_code": "BUDGET_HARD_BLOCKED"}, + }, + ) + ) + charge_card, ran = _protected() + + with pytest.raises(NullRunBlockedException) as info: + charge_card(100) + + assert ran == [] + assert not isinstance(info.value, NullRunGateIndeterminateError) + + +def test_a_good_answer_still_runs_the_body(make_runtime): + """The control: the fix must not over-refuse.""" + make_runtime(api_url=BASE_URL) + charge_card, ran = _protected() + + charge_card(100) + + assert ran == [100] + + +# --------------------------------------------------------------------------- +# 7. The metrics export is complete +# --------------------------------------------------------------------------- + + +def test_every_counter_is_exported(): + """A counter in the dataclass but not in ``to_dict()`` is invisible. + + Pre-fix three counters were in that state — two runtime counters + and one transport counter — so the operator could not see them even + though the SDK was counting them. + """ + exported = metrics.to_dict() + for half in ("runtime", "transport"): + fields = {f.name for f in dataclasses.fields(getattr(metrics, half))} + missing = fields - set(exported[half]) + assert not missing, ( + f"the {half} counters {sorted(missing)} are counted but never " + "exported — they are invisible to the operator" + ) diff --git a/tests/test_sdk16_callback_matches_protect.py b/tests/test_sdk16_callback_matches_protect.py new file mode 100644 index 0000000..fec0372 --- /dev/null +++ b/tests/test_sdk16_callback_matches_protect.py @@ -0,0 +1,204 @@ +"""SDK-16 — the callback and the `@protect` boundary must AGREE. + +The finding +----------- +The SDK has one authoritative fail-OPEN/CLOSED table +(`nullrun/runtime.py`'s module docstring, ADR-063 §2.4). Its row for +`check_workflow_budget` says the gate fails OPEN for TRANSPORT errors +ONLY and CLOSED for everything else. `runtime.check_workflow_budget` +honours that: it RAISES on a 401, on an unreadable answer, on an +unclassifiable refusal and on a rate-limit backend that is down. + +The `@protect` boundary honoured it too — it re-raises every +`NullRunInfrastructureError`. + +The LangChain callback did NOT. Its broad arm swallowed exactly the +classes the runtime raises on purpose, so the SAME refusal killed the +call through one path and vanished through the other. On the callback +path the LLM call then ran with no reservation and its cost event was +dropped: the agent got the call for free. + +Why this file is separate from `test_langchain_enforcement.py` +--------------------------------------------------------------- +That file drives the callback with a DOUBLE — it proves the callback's +own classification. This file drives the REAL runtime against a mocked +backend, and asks the question the ledger row actually asked: does the +same refusal produce the same OUTCOME on both paths? + +Both halves matter. A double proves the callback's arms; a real +runtime proves that the classes the callback now re-raises are the +classes the runtime actually produces — that the 401 in the table is +the 401 the wire produces, and that the unclassifiable refusal is the +one `resolve_refusal_category` raises. + +The counter-test is not optional: `test_a_transport_failure_still_` +lets_the_agent_through` proves the widening did not turn "the backend +is down" into a refusal. +""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from nullrun.breaker.categories import NullRunUnclassifiedRefusalError +from nullrun.breaker.exceptions import ( + NullRunAuthError, + NullRunBackendError, + NullRunMalformedGateResponseError, +) +from nullrun.decorators import protect +from nullrun.instrumentation.langgraph import ( + NullRunCallback, + drain_deferred_enforcement, +) + +BASE_URL = "https://api.test.nullrun.io" +GATE_URL = f"{BASE_URL}/api/v1/gate" + + +def _gate_unauthorized() -> httpx.Response: + """A 401 from the gate: a CREDENTIAL failure, not a transport one.""" + return httpx.Response( + 401, + json={"error_code": "UNAUTHORIZED", "message": "invalid api key"}, + ) + + +def _gate_unclassified_refusal() -> httpx.Response: + """A refusal with NO category. + + ADR-062 §2.2: the gate does not send a bare "no", it says why. A + refusal whose category the SDK does not trust must RAISE, never be + guessed. + """ + return httpx.Response( + 403, + json={ + "decision": "block", + "decision_source": "gateway", + "explanation": "refused", + "error_code": "NR-UNKNOWN", + }, + ) + + +def _gate_malformed() -> httpx.Response: + """A body that is not a verdict: no `decision_source`, no provenance.""" + return httpx.Response(200, json={"decision": "allow"}) + + +# The gate answer, and the class the SDK's OWN table says it produces. +_CASES = { + "unauthorized": (_gate_unauthorized, NullRunAuthError), + "unclassified_refusal": (_gate_unclassified_refusal, NullRunUnclassifiedRefusalError), + "malformed_body": (_gate_malformed, NullRunMalformedGateResponseError), +} + + +@pytest.fixture +def ran() -> list[str]: + """Records whether the decorated body executed.""" + return [] + + +@pytest.fixture(autouse=True) +def _no_deferred_residue(): + """Leave no residue for the next test on this thread. + + The callback stashes a refusal for the `@protect` boundary + (`record_deferred_enforcement`) AND re-raises it. The stash is a + thread-local, so a test that drives the callback directly leaves + the refusal behind. The next `@protect` call on the same thread + would then drain it and refuse a call the stash has nothing to do + with. That is the designed fallback, but it is the CALLER's job to + consume it — so this file consumes it. + """ + yield + while drain_deferred_enforcement() is not None: + pass + + +def _charge(ran: list[str]) -> str: + @protect + def charge_card() -> str: + ran.append("body") + return "charged" + + return charge_card() + + +def _mock_gate(builder) -> None: + respx.post(GATE_URL).mock(return_value=builder()) + + +class TestTheSameRefusalOneOutcome: + """One refusal, two paths, one outcome.""" + + @pytest.mark.parametrize("case", list(_CASES), ids=list(_CASES)) + def test_the_refusal_stops_the_protect_body(self, case, make_runtime, mock_api, ran): + """The `@protect` half — the boundary that already refused.""" + _mock_gate(_CASES[case][0]) + make_runtime() + + with pytest.raises(_CASES[case][1]): + _charge(ran) + + assert ran == [], ( + "the decorated body ran on a refused call — the @protect " + "boundary is the reference behaviour the callback must match" + ) + + @pytest.mark.parametrize("case", list(_CASES), ids=list(_CASES)) + def test_the_refusal_reaches_the_callback_caller(self, case, make_runtime, mock_api): + """The callback half — the path that used to swallow it. + + Pre-fix, the refusal was logged at debug and the LLM call + proceeded with no reservation: the cost event was dropped and + the agent got the call for free. + """ + _mock_gate(_CASES[case][0]) + rt = make_runtime() + cb = NullRunCallback(runtime=rt) + + with pytest.raises(_CASES[case][1]): + cb.on_llm_start({}, ["hi"], run_id="run-sdk16") + + # The refusal is ALSO stashed for the `@protect` boundary, which + # is the fallback for a host framework that ignores `raise_error`. + # A refused call must not lose its refusal on the way out. + stashed = drain_deferred_enforcement() + assert isinstance(stashed, _CASES[case][1]), ( + "the refusal was raised but not stashed — a host framework " + "that swallows callback exceptions would lose it" + ) + + +class TestTheCounterTest: + """The widening must not freeze the agent when the backend is down.""" + + def test_a_transport_failure_still_lets_the_agent_through( + self, make_runtime, mock_api, monkeypatch + ): + """ADR-008: a dead backend must not freeze the agent. + + A transport failure is an infrastructure error TOO — the + callback's new infrastructure arm would catch it and refuse — + unless the transport arm comes first. The order is pinned in + `test_langchain_enforcement.py`; this test pins the BEHAVIOUR + it exists to protect. + """ + make_runtime() + + def _boom(*_a, **_k): + raise NullRunBackendError( + "the backend is down", endpoint="gate", status_code=503 + ) + + monkeypatch.setattr( + "nullrun.runtime.NullRunRuntime.check_workflow_budget", _boom + ) + + cb = NullRunCallback(runtime=make_runtime()) + cb.on_llm_start({}, ["hi"], run_id="run-sdk16") # must not raise diff --git a/tests/test_sdk9_shared_state.py b/tests/test_sdk9_shared_state.py new file mode 100644 index 0000000..873329f --- /dev/null +++ b/tests/test_sdk9_shared_state.py @@ -0,0 +1,486 @@ +"""SDK-9 — the SDK's shared state is read and written under a lock. + +The audit named a subject that no longer exists. ``_GATE_CACHE`` and +``_invalidate_gate_cache_for_chain`` were deleted by the SDK-1/2/3 fix +(commit 87a4c2b) six hours AFTER the audit document was written, so the +audit's line numbers describe a tree that is gone. Its *Fix* line also +cites a lock "the SDK already uses for the cache elsewhere" — no such +lock ever existed for that cache. + +The subject survives elsewhere, and this file pins it where it does. + +The money consequence, stated once and MEASURED: a crash inside the +enforcement path is caught by the gate's fail-OPEN arm, which returns +``None`` — and ``None`` means "nothing was decided". So the defect is +not a crash. It is a verdict that is LOST. + +The direction of the loss is the opposite of the audit's. The audit +said "stale allows in the cache". The measured loss is a **lost +REFUSAL**: the box refuses (lease dead, budget exhausted), the crash +swallows the verdict, the fail-OPEN arm answers "nothing was decided", +and the caller PROCEEDS on a call the box said no to. + +Which path loses it depends on whether a SECOND gate stands behind +the pre-flight: + +* the ``@protect`` tool path (pre-flight → ``/execute`` → body) does + NOT lose it — the crash happens AFTER the answer is stored, so the + ``/execute`` half reads the box's refusal back out of the same + cache and blocks. Measured: 576/576 blocked, before and after; +* the LLM-span path (``on_llm_start`` → ``check_workflow_budget``) has + NO ``/execute`` behind it, so nothing recovers the verdict. Measured + on the real callback with a refusing box: **35–43 of 576 LLM calls + proceeded on a refused lease** before the fix, 0 after. + +Every test here says which direction it fails in. +""" + +from __future__ import annotations + +import sys +import threading +import uuid + +import pytest + +from nullrun.edge import EDGE_LEASE, EdgeTransport + +# --------------------------------------------------------------------------- +# The race is a two-bytecode race, so the test has to make it land +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fast_gil(): + """Make the GIL yield often enough for a two-bytecode race to land. + + CPython only preempts a thread at the switch interval (5 ms by + default), and the window between ``iter(d)`` and ``next()`` is two + bytecodes. At the default interval the race is a lottery ticket — + measured: 160,000 records across 8 threads produced ZERO hits. At + 10 microseconds the same code reproduces in well under a second. + + The switch interval changes only WHEN a thread yields; it changes + no semantics, so the code under test is the code that ships. + """ + old = sys.getswitchinterval() + sys.setswitchinterval(1e-5) + try: + yield + finally: + sys.setswitchinterval(old) + + +# --------------------------------------------------------------------------- +# Doubles +# --------------------------------------------------------------------------- + + +class _FakeResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +def _allowing_box(body): + """A box that says yes, and charges for the answer.""" + return _FakeResponse( + { + "allowed": True, + "lease_id": body["lease_id"], + "granted_millicents": 500_000, + "spent_millicents": 10, + "remaining_millicents": 499_990, + } + ) + + +def _refusing_box(body): + """A box that says no — the lease is dead, or the budget is gone.""" + return _FakeResponse({"allowed": False, "error_code": "EDGE_LEASE_EXHAUSTED"}) + + +def _edge_transport(responder=None, *, answer_cap=None): + t = EdgeTransport( + base_url="http://127.0.0.1:1", + lease_id="0195f0c0-0000-7000-8000-000000000001", + box_token="box-token", + api_key="nr_org_key", + timeout=0.05, + max_retries=0, + ) + t._post = responder if responder is not None else _allowing_box # type: ignore[method-assign] + if answer_cap is not None: + t._answer_cap = answer_cap + return t + + +def _run_threads(workers): + """Run `workers` concurrently; return (results, errors). + + A worker that raises records the exception and returns — one + failure must not hide the others, and the assertion below needs to + see the FIRST failure, not the last. + """ + errors: list[BaseException] = [] + results: list[object] = [] + barrier = threading.Barrier(len(workers)) + + def wrapped(fn): + def _inner(): + try: + barrier.wait(timeout=10) + fn() + except BaseException as exc: # noqa: BLE001 — the failure IS the test + errors.append(exc) + + return _inner + + threads = [threading.Thread(target=wrapped(w)) for w in workers] + for th in threads: + th.start() + for th in threads: + th.join(timeout=120) + return results, errors + + +# --------------------------------------------------------------------------- +# The box's answer cache — the live instance of the audit's shape +# --------------------------------------------------------------------------- + + +def test_recording_the_box_s_answer_under_concurrency_never_raises(fast_gil): + """The reproduction. The eviction walks the LIVE dict. + + Pre-fix this raises ``RuntimeError: dictionary changed size during + iteration`` — CPython's own guard against mutating a dict while + iterating it — and, when another thread pops the same key first, + ``KeyError`` from the eviction's ``pop``. + + The cap is lowered so the eviction runs on nearly every record: + the defect is in the eviction, not in the size of the cache, and a + test that only fails at 64 entries is a test that fails rarely. + """ + t = _edge_transport(answer_cap=4) + + def worker(n): + for i in range(2000): + t._record(f"evt-{n}-{i}", {"decision": "allow"}) + + _, errors = _run_threads([lambda n=n: worker(n) for n in range(8)]) + + assert errors == [], ( + "the shared dict was mutated while it was iterated: " + f"{[type(e).__name__ for e in errors[:3]]}" + ) + + +def test_the_answer_cache_stays_bounded_under_concurrency(fast_gil): + """The cap is a promise, and it is an EXACT promise. + + Too many entries is an unbounded leak. Too FEW is an answer that + was evicted before the /execute half consumed it — and a missing + answer is a refusal (``EDGE_NOT_ENFORCED``), so an over-eager + eviction turns a paid-for call into a spurious block. + + The exact number is what makes the lock load-bearing: the eviction + is a read-modify-write ("is it over the cap? then drop one"), and + only the lock makes that one step. Two threads that both decide to + evict drop two entries for one insert. + """ + t = _edge_transport(answer_cap=4) + + def worker(n): + for i in range(2000): + t._record(f"evt-{n}-{i}", {"decision": "allow"}) + + _, errors = _run_threads([lambda n=n: worker(n) for n in range(8)]) + + assert errors == [] + assert len(t._answers) == t._answer_cap, ( + f"the cache holds {len(t._answers)} entries, not {t._answer_cap}: " + "the eviction ran more than once for one insert" + ) + + +def test_a_concurrent_enforcement_call_never_loses_the_box_s_answer(fast_gil): + """The same race, one level up: the call the agent makes. + + Every call must come back with the box's answer. Pre-fix the + crash escapes ``enforce`` — which is where the fail-OPEN arm of + the gate is waiting for it. + """ + t = _edge_transport(answer_cap=4) + answers: list[dict] = [] + + def worker(n): + for i in range(500): + answers.append( + t.enforce("gpt-4o", 0, 10, event_id=f"evt-{n}-{i}") + ) + + _, errors = _run_threads([lambda n=n: worker(n) for n in range(8)]) + + assert errors == [], ( + "the enforcement call raised instead of answering: " + f"{[type(e).__name__ for e in errors[:3]]}" + ) + assert len(answers) == 8 * 500 + assert all(a["decision"] == "allow" for a in answers) + assert all(a["decision_source"] == EDGE_LEASE for a in answers) + + +def test_a_concurrent_preflight_and_consume_never_trip(fast_gil): + """The realistic shape of the edge path: pre-flight records, the + /execute half consumes. + + Two threads doing both at once must never see an exception and + never see a half-written answer. An answer that was consumed is + gone — that is the point of the pop — and an answer that was + evicted before the consume is a refusal, which is safe. What must + NOT happen is a crash on the way, because the crash is what the + gate's fail-OPEN arm is waiting for. + """ + t = _edge_transport(answer_cap=4) + consumed: list[dict | None] = [] + + def worker(n): + for i in range(300): + event_id = f"evt-{n}-{i}" + t.enforce("gpt-4o", 0, 10, event_id=event_id) + consumed.append(t.take_answer(event_id)) + + _, errors = _run_threads([lambda n=n: worker(n) for n in range(8)]) + + assert errors == [], ( + "the pre-flight/consume pair raised: " + f"{[type(e).__name__ for e in errors[:3]]}" + ) + assert all(c is None or c.get("decision") == "allow" for c in consumed) + + +def test_the_eviction_never_drops_a_live_answer_first(fast_gil): + """The eviction's ORDER is money, and nothing else pins it. + + The cache holds two kinds of answer: a LIVE one (recorded by a + pre-flight, consumed by the ``/execute`` half moments later) and an + ORPHANED one (recorded by an LLM span, which has no ``/execute`` + behind it and is never consumed). + + The FIFO eviction drops the OLDEST. That is the right order for the + shape that occurs in production: the cache is already full of + orphans, the pre-flight records the LIVE answer, and the eviction + fires on that very record. Evicting the OLDEST drops an orphan; + evicting the NEWEST drops the answer that was just recorded — the + one the in-flight call is about to consume. + + Measured: evicting the newest instead of the oldest passes every + other test in this file — the crash is gone either way, so the + order was unpinned. This test pins it. + + No thread is needed: the order is pure insertion order, so the + failure is deterministic rather than a race. + """ + t = _edge_transport(answer_cap=4) + + # The cache is full of orphans — the LLM-span shape. + for i in range(t._answer_cap): + t._record(f"orphan-{i}", {"decision": "allow"}) + + assert len(t._answers) == t._answer_cap + + # The pre-flight now records the answer the in-flight call will + # consume. This record tips the cache over the cap, so the + # eviction fires ON THE LIVE ANSWER. + live = "LIVE-answer" + t._record(live, {"decision": "allow"}) + + assert t.take_answer(live) is not None, ( + "the eviction dropped the LIVE answer it had just recorded: the " + "FIFO must evict the oldest entry, or a call the box allowed and " + "charged is refused with EDGE_NOT_ENFORCED" + ) + + +def test_a_concurrent_gate_call_never_falls_into_the_fail_open_arm(monkeypatch, fast_gil): + """The consequence, at the level the agent sees it. + + ``check_workflow_budget`` returns ``None`` for two things only: a + fail-OPEN (the gate was unreachable) and a soft_pass. Neither can + happen here — the box answers "allow" every time — so a ``None`` + in the result set means the enforcement path CRASHED and the gate + swallowed the crash. + + This is the test that names the money: pre-fix, a busy agent gets + tool calls that ran with no policy decision, while the box has + already charged the lease. + """ + from nullrun.context import workflow + from nullrun.runtime import NullRunRuntime + + monkeypatch.setenv("NULLRUN_EDGE_URL", "http://127.0.0.1:18090") + monkeypatch.setenv("NULLRUN_EDGE_LEASE_ID", "0195f0c0-0000-7000-8000-000000000001") + monkeypatch.setenv("NULLRUN_EDGE_TOKEN", "box-token") + monkeypatch.setenv("NULLRUN_API_KEY", "nr_org_key") + + rt = NullRunRuntime( + api_key="nr_org_key", + api_url="https://api.nullrun.io", + _test_mode=True, + polling=False, + ) + rt._transport.edge._post = _allowing_box # type: ignore[method-assign] + rt._transport.edge._answer_cap = 4 + + outcomes: list[object] = [] + + def worker(n): + for _ in range(60): + with workflow("wf-edge"): + outcomes.append(rt.check_workflow_budget()) + + _, errors = _run_threads([lambda n=n: worker(n) for n in range(8)]) + + assert errors == [], ( + "the gate raised instead of answering: " + f"{[type(e).__name__ for e in errors[:3]]}" + ) + assert outcomes, "no gate call completed" + assert None not in outcomes, ( + "a gate call returned None: the enforcement path crashed and the " + "fail-OPEN arm swallowed it, so the tool would run with no policy " + "decision at all" + ) + + +def test_a_refused_box_never_becomes_a_failed_open(monkeypatch, fast_gil): + """The MONEY direction: a refusal must never read as "no decision". + + This is the direction the audit got backwards. The audit said the + crash leaves "stale allows in the cache". The measured loss is the + opposite: the box REFUSES, the crash swallows the verdict, the + fail-OPEN arm returns ``None`` — "nothing was decided" — and the + caller PROCEEDS on a call the box said no to. + + Measured on the real LLM-span path with a refusing box (96 threads + x 6 calls): **35–43 of 576 LLM calls proceeded on a refused lease** + before the fix. After the fix: 0. + + So the invariant this test pins is not "no crash" — it is "a + refusal is never converted into a fail-OPEN". + + ``check_workflow_budget`` returns ``None`` for exactly two things: + a fail-OPEN, and a soft_pass. Neither can happen here — the box + REFUSES every time — so a ``None`` here means the verdict was + destroyed, not that the gate was unreachable. + """ + from nullrun.breaker.exceptions import NullRunBlockedException + from nullrun.context import workflow + from nullrun.runtime import NullRunRuntime + + monkeypatch.setenv("NULLRUN_EDGE_URL", "http://127.0.0.1:18090") + monkeypatch.setenv("NULLRUN_EDGE_LEASE_ID", "0195f0c0-0000-7000-8000-000000000001") + monkeypatch.setenv("NULLRUN_EDGE_TOKEN", "box-token") + monkeypatch.setenv("NULLRUN_API_KEY", "nr_org_key") + + rt = NullRunRuntime( + api_key="nr_org_key", + api_url="https://api.nullrun.io", + _test_mode=True, + polling=False, + ) + rt._transport.edge._post = _refusing_box # type: ignore[method-assign] + rt._transport.edge._answer_cap = 4 + + verdicts: list[object] = [] + + def worker(n): + for _ in range(40): + with workflow("wf-edge"): + try: + verdicts.append(rt.check_workflow_budget()) + except NullRunBlockedException: + # The refusal reached the caller: correct. + verdicts.append("refused") + + _, errors = _run_threads([lambda n=n: worker(n) for n in range(8)]) + + assert errors == [], ( + "the gate raised something that is not a refusal: " + f"{[type(e).__name__ for e in errors[:3]]}" + ) + assert verdicts, "no gate call completed" + assert None not in verdicts, ( + "a REFUSED box was converted into a failed-open: the enforcement " + "path crashed, the fail-OPEN arm swallowed the refusal, and the " + "caller proceeded on a call the box said no to" + ) + + +# --------------------------------------------------------------------------- +# The audit's own Verify clause +# --------------------------------------------------------------------------- + + +def test_concurrent_track_refusals_all_propagate(monkeypatch, fast_gil): + """The audit's Verify clause, verbatim. + + "hammer the track path with concurrent refusals => no RuntimeError". + + At HEAD the track path's refusal arm touches no shared state at + all — the gate cache it used to invalidate is gone — so this test + is GREEN before the fix as well as after it. It is here because the + clause asks for a check that EXISTS, and a check that never ran is + worse than a red one. + + The refusals themselves must still reach the caller: a refusal that + is swallowed is SDK-6, and this test would happily pass if the + arm started dropping them. + """ + from nullrun.breaker.exceptions import NullRunDecision + from nullrun.runtime import NullRunRuntime + + rt = NullRunRuntime( + api_key="nr_org_key", + api_url="https://api.nullrun.io", + _test_mode=True, + polling=False, + ) + + def refusing_track_single(payload): + raise NullRunDecision("the server refused the consume") + + monkeypatch.setattr(rt._transport, "track_single", refusing_track_single) + + refused: list[BaseException] = [] + other: list[BaseException] = [] + + def worker(n): + for _ in range(40): + try: + rt.track( + { + "type": "llm_call", + "workflow_id": "wf-track", + "tokens": 100, + }, + reservation_id=str(uuid.uuid4()), + ) + except NullRunDecision as exc: + refused.append(exc) + except BaseException as exc: # noqa: BLE001 — the failure IS the test + other.append(exc) + return + + _, errors = _run_threads([lambda n=n: worker(n) for n in range(8)]) + assert errors == [] + assert other == [], ( + "the track path raised something that is not a refusal: " + f"{[type(e).__name__ for e in other[:3]]}" + ) + assert len(refused) == 8 * 40, ( + f"only {len(refused)} of {8 * 40} refusals reached the caller" + ) diff --git a/tests/test_sensitive_fail_open_guard.py b/tests/test_sensitive_fail_open_guard.py index f3d0877..31cc0fe 100644 --- a/tests/test_sensitive_fail_open_guard.py +++ b/tests/test_sensitive_fail_open_guard.py @@ -9,7 +9,7 @@ since it was caught doing the same thing. The asymmetry was an oversight, and it is the more dangerous half: the budget opt-out skips a *pre-flight*, while this one lets ``charge_card`` run with no -policy evaluation at all. ADR-008 calls that a security regression +policy evaluation at all. ADR-063 §2.4 calls that a security regression rather than an availability trade-off. The guard refuses the bypass rather than raising: enforcement falls @@ -18,6 +18,16 @@ attempt is logged at ERROR with a metric rather than passing unnoticed. +**SDK-8 (2026-11-05) — the switch is now bounded per tool.** The guard +above answered "may this switch be used against production?", but the +switch itself was a single global boolean: turning it on opened EVERY +protected tool, so enabling it for one tool disabled enforcement for +all of them. ADR-126 bounds the switch with an explicit per-tool +classification (``runtime.register_fail_open_tools([...])``): the +switch may open a tool ONLY if that tool was classified, and the +classification is empty by default. So the switch alone opens nothing +— which is what the two tests below pin. + Every test below fails against the pre-fix code. """ @@ -127,12 +137,20 @@ def charge_card(amount: int) -> str: ) def test_sensitive_body_runs_in_dev_with_flag(self, make_runtime, mock_api, monkeypatch): - """The documented bypass still works off production.""" + """The documented bypass still works off production — but it is + a TWO-STEP opt-out now (SDK-8 / ADR-126): the switch AND the + per-tool classification. + + The assertion is unchanged: the body ran. What changed is the + setup, because "the switch alone opens every tool" was the + defect. + """ from nullrun.breaker.exceptions import NullRunBlockedException from nullrun.decorators import protect # Not bound, for the same reason as the prod case above. - make_runtime(api_url=BASE_URL) + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) monkeypatch.setenv(_FLAG, "1") respx.post(EXECUTE_URL).mock( @@ -159,3 +177,132 @@ def charge_card(amount: int) -> str: "NULLRUN_SENSITIVE_FAIL_OPEN=1 against a non-prod api_url " "must let the body run" ) + + +class TestTheSwitchAloneOpensNothing: + """SDK-8 / ADR-126 — the switch is bounded per tool. + + The audit's verify clause, verbatim: *"enable it and check a + non-sensitive tool — it must still fail closed."* Here the bound + is stronger than the clause asks for: with the switch ON and NO + classification, EVERY tool fails closed — sensitive or not. + """ + + def test_the_switch_alone_does_not_run_the_body(self, make_runtime, mock_api, monkeypatch): + """The switch ON, the tool NOT classified: the body must not + run. This is the defect the audit found, restated as a test.""" + from nullrun.breaker.exceptions import NullRunBlockedException + from nullrun.decorators import protect + + make_runtime(api_url=BASE_URL) + monkeypatch.setenv(_FLAG, "1") + + respx.post(EXECUTE_URL).mock( + side_effect=httpx.ConnectError("connection refused") + ) + + ran: list[int] = [] + + @protect + def charge_card(amount: int) -> str: + ran.append(amount) + return "charged" + + with pytest.raises(NullRunBlockedException): + charge_card(100) + + assert ran == [], ( + "the switch alone opened the tool -- the switch must be " + "bounded by the per-tool classification " + "(runtime.register_fail_open_tools)" + ) + + def test_a_classified_tool_does_not_open_another_tool( + self, make_runtime, mock_api, monkeypatch + ): + """Classifying tool A must not open tool B. The old switch had + no notion of "which tool" at all.""" + from nullrun.breaker.exceptions import NullRunBlockedException + from nullrun.decorators import protect + + rt = make_runtime(api_url=BASE_URL) + rt.register_fail_open_tools(["charge_card"]) + monkeypatch.setenv(_FLAG, "1") + + respx.post(EXECUTE_URL).mock( + side_effect=httpx.ConnectError("connection refused") + ) + + ran: list[int] = [] + + @protect + def read_balance(account: str) -> str: + ran.append(account) + return "42" + + with pytest.raises(NullRunBlockedException): + read_balance("acc-1") + + assert ran == [], ( + "classifying one tool opened a DIFFERENT tool -- the " + "classification is per tool, not a global permission" + ) + + def test_the_fail_closed_message_names_both_steps(self, make_runtime, mock_api, monkeypatch): + """The operator who wants the bypass must be told BOTH steps, or + they will set the env var, see no effect, and file a bug.""" + from nullrun.breaker.exceptions import NullRunBlockedException + from nullrun.decorators import protect + + make_runtime(api_url=BASE_URL) + monkeypatch.setenv(_FLAG, "1") + + respx.post(EXECUTE_URL).mock( + side_effect=httpx.ConnectError("connection refused") + ) + + @protect + def charge_card(amount: int) -> str: + return "charged" + + with pytest.raises(NullRunBlockedException) as exc: + charge_card(100) + + message = exc.value.user_action + assert "NULLRUN_SENSITIVE_FAIL_OPEN" in message + assert "register_fail_open_tools" in message, ( + "the refusal must name the per-tool classification -- " + "the switch alone opens nothing" + ) + + def test_prod_opens_only_the_classified_tool(self, make_runtime, mock_prod_api, monkeypatch): + """In production the switch needs the ack as well (row 9 of + ADR-063 §2.4), and even then it opens ONLY the classified + tool. So the prod reach is strictly narrower than it was + before SDK-8: the ack used to unlock EVERY tool.""" + from nullrun.breaker.exceptions import NullRunBlockedException + from nullrun.decorators import protect + + rt = make_runtime(api_url=PROD_URL) + rt.register_fail_open_tools(["charge_card"]) + monkeypatch.setenv(_FLAG, "1") + monkeypatch.setenv(_ACK, "1") + + respx.post(f"{PROD_URL}/api/v1/execute").mock( + side_effect=httpx.ConnectError("connection refused") + ) + + ran: list[int] = [] + + @protect + def read_balance(account: str) -> str: + ran.append(account) + return "42" + + with pytest.raises(NullRunBlockedException): + read_balance("acc-1") + + assert ran == [], ( + "in production, the double-ack opened a tool that was NOT " + "classified -- the reach is bounded per tool there too" + ) diff --git a/tests/test_status.py b/tests/test_status.py index 6a3c27a..64dab27 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -70,7 +70,8 @@ def test_minimal_runtime_yields_ok_state(self): rt = _make_runtime() s = rt.status() assert s.state == "ok" - assert s.api_key_prefix == "nr_live_te" + # SDK-5: the value is a fingerprint, not a prefix of the key. + assert s.api_key_prefix == "653ad4a8" assert s.is_healthy() is True def test_snapshot_is_frozen(self): @@ -85,13 +86,23 @@ def test_snapshot_supports_equality(self): s2 = rt.status() assert s1 == s2 - def test_api_key_prefix_truncated_to_10_chars(self): - _make_runtime(api_key="nr_live_SsBF9OMYcVCgRCNcCVcJ4khTOPKx79JG") + def test_api_key_prefix_is_a_fingerprint_not_a_prefix(self): + """SDK-5: the snapshot carries NO character of the key. + + Before the fix this field was ``api_key[:10]`` — ten + characters of a credential in a user-visible string. The + assertion below must FAIL if the raw slice comes back: the + fake key starts with ``nr_live_`` and the fingerprint does not + contain it. + """ s = _make_runtime( api_key="nr_live_SsBF9OMYcVCgRCNcCVcJ4khTOPKx79JG" ).status() - assert s.api_key_prefix == "nr_live_Ss" - assert len(s.api_key_prefix) == 10 + assert s.api_key_prefix == "c9ef9e82" + assert len(s.api_key_prefix) == 8 + # No character of the key, not even its prefix. + assert "nr_live_" not in str(s) + assert "SsBF9OMY" not in str(s) # Full key MUST NOT leak into the snapshot. assert "TOPKx79JG" not in str(s) @@ -215,7 +226,9 @@ class TestSummary: def test_ok_summary(self): out = _make_runtime().status().summary() assert "ok" in out - assert "nr_live_te" in out + # SDK-5: the summary carries a fingerprint, not a key prefix. + assert "api_key_fp=653ad4a8" in out + assert "nr_live_te" not in out def test_summary_with_organization_and_workflow(self): # Covers the ``if self.organization_id`` and diff --git a/tests/test_status_backend_reachable_has_a_writer.py b/tests/test_status_backend_reachable_has_a_writer.py index 7fbc332..03ff858 100644 --- a/tests/test_status_backend_reachable_has_a_writer.py +++ b/tests/test_status_backend_reachable_has_a_writer.py @@ -46,6 +46,8 @@ import threading +import pytest + from nullrun.observability.status import _RecentErrorRing from nullrun.runtime import NullRunRuntime @@ -160,8 +162,21 @@ class TestWriterIsActuallyWired: A helper with no call sites is exactly how the field became permanently ``None`` in the first place: the reader and the writer - were written together and the writer was then never invoked. These - pin the call sites in production source. + were written together and the writer was then never invoked. + + These drive the real code paths and read the field the writer + writes. They used to count occurrences of the call in the source, + and that count could not tell "the guard was removed" from "a + duplicated arm was removed": the /gate check had two branches (a + chain-mode cache branch and the direct branch) with the SAME + fail-open arm, so the count of ``_note_backend_attempt(False)`` was + 3 for a property that only ever had 2 distinct sites. When the + cache branch went (SDK-1) the count fell to 2 and the pin went red + on correct code. + + The property is what is pinned here: every path that learns the + backend answered records True, every path that learns it did not + record False. """ @staticmethod @@ -174,17 +189,112 @@ def _runtime_source() -> str: encoding="utf-8" ) - def test_gate_paths_call_the_writer(self): - source = self._runtime_source() - assert source.count("_note_backend_attempt(True)") >= 3, ( - "both /gate paths in check_workflow_budget and the /auth/verify " - "success path must record that the backend answered" - ) - assert source.count("_note_backend_attempt(False)") >= 3, ( - "both /gate fail-open paths and the /auth/verify transport-error " - "path must record a transport failure" + @staticmethod + def _live_runtime() -> NullRunRuntime: + """A runtime with a transport but no network.""" + return NullRunRuntime( + api_key="nr_live_testkey", + api_url="https://api.example.invalid", + _test_mode=True, + polling=False, ) + def test_the_gate_answer_path_records_reachability(self): + """The /gate success path: the backend answered.""" + from nullrun.context import workflow + + rt = self._live_runtime() + try: + rt._transport.check = lambda req: { # type: ignore[method-assign] + "decision": "allow", + "decision_source": "gateway", + } + with workflow("wf-writer"): + rt.check_workflow_budget() + assert rt.status().backend_reachable is True + finally: + rt.shutdown() + + def test_the_gate_fail_open_path_records_a_transport_failure(self): + """The /gate fail-open path: the backend did not answer. + + This is the path that used to be duplicated — once per branch of + the (now removed) chain-mode cache. It is one path now and it + still records the failure. + """ + import httpx + + from nullrun.context import workflow + + rt = self._live_runtime() + + def boom(req): + raise httpx.ConnectError("connection refused") + + try: + rt._transport.check = boom # type: ignore[method-assign] + with workflow("wf-writer"): + assert rt.check_workflow_budget() is None + assert rt.status().backend_reachable is False + finally: + rt.shutdown() + + def test_the_auth_success_path_records_reachability(self): + """The /auth/verify 200 path: the strongest liveness signal.""" + import httpx + + rt = self._live_runtime() + try: + rt._post_auth_with_retry = lambda *a, **kw: httpx.Response( # type: ignore[method-assign] + 200, + json={"organization_id": "org-test", "workflow_id": "wf-1"}, + ) + rt._authenticate() + assert rt.status().backend_reachable is True + finally: + rt.shutdown() + + def test_a_refusal_from_auth_still_counts_as_the_backend_answering(self): + """A 401 is a reachable backend with a bad key.""" + import httpx + + from nullrun.breaker.exceptions import NullRunAuthenticationError + + rt = self._live_runtime() + try: + rt._post_auth_with_retry = lambda *a, **kw: httpx.Response( # type: ignore[method-assign] + 401, json={} + ) + with pytest.raises(NullRunAuthenticationError): + rt._authenticate() + assert rt.status().backend_reachable is True + finally: + rt.shutdown() + + def test_the_auth_transport_error_path_records_a_transport_failure(self): + """The /auth/verify transport error: the backend did not answer. + + This is the FIRST backend attempt in a normal run, so it is + what makes `backend_reachable` observable at all when the + backend is down from the start. + """ + import httpx + + from nullrun.breaker.exceptions import NullRunTransportError + + rt = self._live_runtime() + + def boom(*a, **kw): + raise httpx.ConnectError("connection refused") + + try: + rt._post_auth_with_retry = boom # type: ignore[method-assign] + with pytest.raises(NullRunTransportError): + rt._authenticate() + assert rt.status().backend_reachable is False + finally: + rt.shutdown() + def test_writer_has_no_none_assignment(self): """The invariant that keeps the field truthful.""" source = self._runtime_source() diff --git a/tests/test_supply_chain_lint.py b/tests/test_supply_chain_lint.py new file mode 100644 index 0000000..b72c8f2 --- /dev/null +++ b/tests/test_supply_chain_lint.py @@ -0,0 +1,579 @@ +"""The supply-chain lint must fail on a bad workflow and pass on a good one. + +SDK-3 (security audit 2026-11-06) closed the hole by pinning every action +to a commit SHA and installing from ``uv.lock``. A lint that nobody runs is +the same as no lint, so these tests exist to prove the lint is capable of +going red. A lint that only ever says OK proves nothing. + +The same file also guards the pins that live outside a workflow: +``[build-system].requires`` and the ``publish`` dependency group. A build +backend is not part of ``uv.lock``, so ``uv lock --check`` cannot see it — +nothing but an explicit test keeps that pin honest. + +The fixtures live in a temporary directory. The repository's own workflow +files are never modified to make a test pass. +""" + +from __future__ import annotations + +import importlib.util +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "check_supply_chain.py" +PYPROJECT = REPO_ROOT / "pyproject.toml" + + +def load_lint_module(): + """Import ``scripts/check_supply_chain.py`` by path.""" + spec = importlib.util.spec_from_file_location("check_supply_chain", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + # The module registers itself before it runs. Without this, the + # ``@dataclass`` decorator cannot find the module that owns the class + # and raises AttributeError. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lint = load_lint_module() + + +GOOD_WORKFLOW = """\ +name: good +on: + push: + branches: [master] +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + enable-cache: true + - name: Install from the lock + run: uv sync --frozen --extra dev + - name: Run the tests + run: uv run --frozen --extra dev pytest -n auto +""" + + +def write_workflow(directory: Path, body: str, name: str = "workflow.yml") -> Path: + directory.mkdir(parents=True, exist_ok=True) + path = directory / name + path.write_text(body, encoding="utf-8") + return path + + +# The shape of the publish job: it holds the PyPI Trusted Publishing token. +TRUSTED_WORKFLOW = """\ +name: trusted +on: + push: + tags: ["v*"] +jobs: + publish: + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.11.28" + enable-cache: true + - name: Build + run: uv run --frozen --only-group publish python -m build +""" + + +def rules_for(path: Path, use_yaml: bool | None = None) -> list[str]: + violations = lint.find_violations([path], use_yaml=use_yaml) + return sorted(violation.rule for violation in violations) + + +# ── The good case ─────────────────────────────────────────────────────────── + + +def test_a_fully_pinned_workflow_passes(tmp_path: Path) -> None: + path = write_workflow(tmp_path, GOOD_WORKFLOW) + assert rules_for(path) == [] + + +def test_the_repositories_own_workflows_pass() -> None: + """The lint must be green on the repository it guards. + + This is the check that fails when a workflow is edited without a pin. + """ + workflow_dir = REPO_ROOT / ".github" / "workflows" + assert workflow_dir.is_dir() + violations = lint.find_violations([workflow_dir]) + assert violations == [], "\n".join(violation.render() for violation in violations) + + +# ── The lint must be CALLED ─────────────────────────────────────────────────── + + +def workflow_text(name: str) -> str: + return (REPO_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8") + + +def workflow_document(name: str) -> dict: + yaml = pytest.importorskip("yaml") + return yaml.safe_load(workflow_text(name)) + + +def test_the_lint_is_called_by_every_workflow() -> None: + """A lint that exists but is never called catches nothing. + + Deleting the invocation from a workflow would leave the repository + with a lint file that no runner executes. This test is what makes + that deletion visible. + """ + for name in ("ci.yml", "publish.yml", "publish-test.yml"): + assert "scripts/check_supply_chain.py" in workflow_text(name), name + + +def test_the_lint_runs_in_the_publish_job_before_the_upload() -> None: + """The publish job holds the PyPI Trusted Publishing token. + + The lint must run inside THAT job, and before the upload step. A + lint in a different job only helps if that job gates the publish + job; a lint inside the publish job helps even if it does not. + """ + for name in ("publish.yml", "publish-test.yml"): + job = workflow_document(name)["jobs"]["publish"] + + # The trust grant must survive the pin. + assert job["permissions"].get("id-token") == "write", name + + lint_index = None + upload_index = None + for index, step in enumerate(job["steps"]): + if "scripts/check_supply_chain.py" in step.get("run", ""): + lint_index = index + if str(step.get("uses", "")).startswith("pypa/gh-action-pypi-publish"): + upload_index = index + + assert lint_index is not None, f"{name}: the publish job does not run the lint" + assert upload_index is not None, f"{name}: the publish job does not upload" + assert lint_index < upload_index, f"{name}: the lint runs after the upload" + + +# ── The bad cases ─────────────────────────────────────────────────────────── + + +def test_a_tag_ref_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4", + "actions/checkout@v4", + ) + path = write_workflow(tmp_path, body) + assert "action-ref-not-sha-pinned" in rules_for(path) + + +def test_a_branch_ref_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4", + "actions/checkout@main", + ) + path = write_workflow(tmp_path, body) + assert "action-ref-not-sha-pinned" in rules_for(path) + + +def test_a_short_sha_is_rejected(tmp_path: Path) -> None: + """A truncated SHA is not a SHA. It must not pass.""" + body = GOOD_WORKFLOW.replace( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4", + "actions/checkout@11d5960 # v4", + ) + path = write_workflow(tmp_path, body) + assert "action-ref-not-sha-pinned" in rules_for(path) + + +def test_an_unpinned_setup_uv_version_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace('version: "0.11.28"', 'version: "latest"') + path = write_workflow(tmp_path, body) + assert "action-version-not-pinned" in rules_for(path) + + +def test_a_ranged_pip_install_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + 'run: pip install "pytest-xdist>=3.6"', + ) + path = write_workflow(tmp_path, body) + assert "dependency-not-exactly-pinned" in rules_for(path) + + +def test_a_bare_pip_install_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + "run: pip install --upgrade build", + ) + path = write_workflow(tmp_path, body) + assert "dependency-not-exactly-pinned" in rules_for(path) + + +def test_an_editable_install_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + 'run: pip install -e ".[dev]"', + ) + path = write_workflow(tmp_path, body) + assert "dependency-not-exactly-pinned" in rules_for(path) + + +def test_an_uv_sync_without_frozen_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + "run: uv sync --extra dev", + ) + path = write_workflow(tmp_path, body) + assert "uv-sync-not-frozen" in rules_for(path) + + +def test_an_uv_sync_with_upgrade_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + "run: uv sync --frozen --upgrade", + ) + path = write_workflow(tmp_path, body) + assert "uv-sync-not-frozen" in rules_for(path) + + +def test_a_uv_pip_install_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + "run: uv pip install twine==7.0.0", + ) + path = write_workflow(tmp_path, body) + assert "uv-install-bypasses-lock" in rules_for(path) + + +def test_a_uv_run_without_frozen_is_rejected(tmp_path: Path) -> None: + """``uv run`` syncs before it runs. Without ``--frozen`` it may + re-resolve and rewrite the lock, so the reviewed lock stops being + the contract.""" + body = GOOD_WORKFLOW.replace( + "run: uv run --frozen --extra dev pytest -n auto", + "run: uv run --extra dev pytest -n auto", + ) + path = write_workflow(tmp_path, body) + assert "uv-run-not-frozen" in rules_for(path) + + +def test_a_uv_run_with_locked_is_allowed(tmp_path: Path) -> None: + """``--locked`` is the other accepted spelling. It fails when the + lock is stale instead of rewriting it, which is the same promise + ``--frozen`` makes.""" + body = GOOD_WORKFLOW.replace( + "run: uv run --frozen --extra dev pytest -n auto", + "run: uv run --locked --extra dev pytest -n auto", + ) + path = write_workflow(tmp_path, body) + assert rules_for(path) == [] + + +def test_a_requirements_file_without_hashes_is_rejected(tmp_path: Path) -> None: + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + "run: pip install -r requirements.txt", + ) + path = write_workflow(tmp_path, body) + assert "requirements-file-without-hashes" in rules_for(path) + + +def test_an_exact_pip_pin_is_allowed(tmp_path: Path) -> None: + """An exact ``==`` pin is not a ranged install, so it must pass. + + The audit asks for an exact pin OR a lock-based install. This rule does + not force every workflow onto uv; it forbids the unpinned form. + """ + body = GOOD_WORKFLOW.replace( + "run: uv sync --frozen --extra dev", + "run: pip install twine==7.0.0", + ) + path = write_workflow(tmp_path, body) + assert rules_for(path) == [] + + +def test_a_lock_only_publish_job_passes(tmp_path: Path) -> None: + """The publish job installs from the lock. It must be clean.""" + path = write_workflow(tmp_path, TRUSTED_WORKFLOW) + assert rules_for(path) == [] + + +def test_a_pip_install_in_a_trusted_job_is_rejected(tmp_path: Path) -> None: + """Inside a job that holds the PyPI OIDC token, pip installs nothing. + + An exact ``==`` pin is not enough there: the job resolves outside + uv.lock, which is the hole the audit found. The lock-based install + is the only allowed form. + """ + body = TRUSTED_WORKFLOW.replace( + "run: uv run --frozen --only-group publish python -m build", + "run: pip install twine==7.0.0", + ) + path = write_workflow(tmp_path, body) + assert "pip-install-in-trusted-job" in rules_for(path) + + +def test_the_trusted_job_rule_holds_without_pyyaml(tmp_path: Path) -> None: + """The rule must survive the text scanner. + + A CI runner without PyYAML must not enforce less than a runner with + it, or the rule is a rule that only some runs apply. + """ + body = TRUSTED_WORKFLOW.replace( + "run: uv run --frozen --only-group publish python -m build", + "run: pip install twine==7.0.0", + ) + path = write_workflow(tmp_path, body) + assert "pip-install-in-trusted-job" in rules_for(path, use_yaml=False) + + +# ── The text fallback must agree with the YAML path ───────────────────────── + + +def test_the_text_scanner_agrees_with_the_yaml_scanner(tmp_path: Path) -> None: + """The fallback must find the same problems, or it is a hole. + + A CI runner without PyYAML must not silently check less than a runner + with it. + """ + body = GOOD_WORKFLOW.replace( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4", + "actions/checkout@v4", + ).replace( + "run: uv sync --frozen --extra dev", + "run: pip install --upgrade build", + ) + path = write_workflow(tmp_path, body) + with_yaml = rules_for(path, use_yaml=True) + without_yaml = rules_for(path, use_yaml=False) + assert with_yaml == without_yaml + assert "action-ref-not-sha-pinned" in without_yaml + assert "dependency-not-exactly-pinned" in without_yaml + + +# ── The pins that live outside a workflow ─────────────────────────────────── +# +# A build backend is NOT part of uv.lock's resolution: uv reads +# ``[build-system].requires`` when it builds, and the result never enters +# the lock file. So ``uv lock --check`` cannot see it, and the workflow +# lint cannot see it either. One line of pyproject.toml is all it takes to +# bring the hole back: +# +# requires = ["hatchling"] +# +# These tests are the guard for that line. + + +EXACT_PIN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*==[^\s,;]+$") + +# The shape the repository had before SDK-3. A fixture, not a copy of the +# real file: the check must fail on the FORM, not on a stored copy of the +# answer. +PRE_FIX_PYPROJECT = """\ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +publish = [ + "build", + "twine==7.0.0", +] +""" + + +def load_toml(path: Path) -> dict: + """Read a TOML file with the standard library. + + ``tomllib`` is stdlib from 3.11. On 3.10 the test falls back to + ``tomli``, which the dev extra already declares for exactly this. + """ + try: + import tomllib + except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib # type: ignore[no-redef] + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def pin_problems(pyproject: dict) -> list[str]: + """Return every pin problem in ``pyproject``. Empty means clean.""" + problems: list[str] = [] + + build_system = pyproject.get("build-system", {}) + requires = build_system.get("requires", []) + if not requires: + problems.append("build-system.requires is empty: no build backend is declared") + + names = [] + for entry in requires: + if not EXACT_PIN.match(entry): + problems.append(f"build-system.requires entry is not an exact == pin: {entry}") + names.append(entry.split("==", 1)[0]) + + backend = build_system.get("build-backend", "") + if backend.startswith("hatchling") and "hatchling" not in names: + problems.append( + f"build-system.requires does not name hatchling while the build backend is {backend}" + ) + + groups = pyproject.get("dependency-groups", {}) + if "publish" not in groups: + problems.append("dependency-groups.publish is missing") + for entry in groups.get("publish", []): + if not EXACT_PIN.match(entry): + problems.append(f"dependency-groups.publish entry is not an exact == pin: {entry}") + + return problems + + +def test_the_build_backend_is_exactly_pinned() -> None: + """``[build-system].requires`` must carry exact pins. + + This is the pin the publish job resolves inside the job that holds the + PyPI Trusted Publishing token. + """ + problems = pin_problems(load_toml(PYPROJECT)) + assert problems == [], "\n".join(problems) + + +def test_the_publish_group_is_exactly_pinned() -> None: + """The publish tooling must come from the lock, not from a range.""" + groups = load_toml(PYPROJECT).get("dependency-groups", {}) + assert "publish" in groups, "the publish dependency group is missing" + for entry in groups["publish"]: + assert EXACT_PIN.match(entry), f"not an exact == pin: {entry}" + + +def test_the_pre_fix_pyproject_fails_the_pin_check(tmp_path: Path) -> None: + """The RED proof. + + The form the repository had before SDK-3 — an unpinned build backend + and an unpinned publish entry — must fail. Without this, the check + above might be a check that passes for any input. + """ + path = tmp_path / "pyproject.toml" + path.write_text(PRE_FIX_PYPROJECT, encoding="utf-8") + + problems = pin_problems(load_toml(path)) + assert problems, "the pre-fix pyproject must not pass" + assert any("hatchling" in problem for problem in problems) + assert any("build" in problem for problem in problems) + + +# ── The script as a process ───────────────────────────────────────────────── + + +def run_script(*arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), *arguments], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=False, + ) + + +def test_the_script_exits_zero_on_the_repository(tmp_path: Path) -> None: + result = run_script() + assert result.returncode == 0, result.stdout + result.stderr + assert "supply-chain: OK" in result.stdout + + +def test_the_script_exits_nonzero_on_a_broken_workflow(tmp_path: Path) -> None: + """The proof that the lint can go red. + + The broken file lives in a temporary directory. The repository's own + workflows are never touched. + """ + body = GOOD_WORKFLOW.replace( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4", + "actions/checkout@v4", + ) + workflow = write_workflow(tmp_path / "workflows", body) + assert workflow.exists() + + result = run_script(str(workflow)) + assert result.returncode == 1 + assert "action-ref-not-sha-pinned" in result.stdout + assert "actions/checkout@v4" in result.stdout + + +def test_the_lint_catches_a_mutable_ref_and_a_ranged_install(tmp_path: Path) -> None: + """Both holes of SDK-3 in one file: a tag ref and a ranged install. + + This is the shape the audit described — a workflow that pins nothing + and resolves its dependencies at run time. The lint must name both. + """ + body = GOOD_WORKFLOW.replace( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4", + "actions/checkout@v4", + ).replace( + "run: uv sync --frozen --extra dev", + 'run: pip install -e ".[dev]" "pytest-xdist>=3.6"', + ) + workflow = write_workflow(tmp_path / "workflows", body) + + result = run_script(str(workflow)) + assert result.returncode == 1 + assert "action-ref-not-sha-pinned" in result.stdout + assert "dependency-not-exactly-pinned" in result.stdout + + +def test_the_script_self_test_passes() -> None: + """``--self-test`` plants a broken workflow and asserts the lint is + red on it. If the lint could not go red, the self-test fails.""" + result = run_script("--self-test") + assert result.returncode == 0, result.stdout + result.stderr + assert "self-test OK" in result.stdout + + +def test_the_script_resolves_paths_relative_to_itself(tmp_path: Path) -> None: + """Running from another directory must check the same files. + + The script resolves the repository from its own location, never from + the current directory. A CI job can ``cd`` anywhere. + """ + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + result = subprocess.run( + [sys.executable, str(SCRIPT)], + capture_output=True, + text=True, + cwd=elsewhere, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "supply-chain: OK" in result.stdout + + +def test_the_script_reports_a_missing_path(tmp_path: Path) -> None: + result = run_script(str(tmp_path / "nope.yml")) + assert result.returncode == 2 + assert "no such path" in result.stderr + + +@pytest.mark.parametrize("body", [GOOD_WORKFLOW]) +def test_the_fixture_is_a_valid_workflow(body: str) -> None: + """The good fixture must parse. A fixture that does not parse would make + every test above pass for the wrong reason.""" + yaml = pytest.importorskip("yaml") + document = yaml.safe_load(body) + assert document["jobs"]["test"]["steps"] diff --git a/tests/test_tool_ordering_real_langchain.py b/tests/test_tool_ordering_real_langchain.py index dd5ac24..2c7e7af 100644 --- a/tests/test_tool_ordering_real_langchain.py +++ b/tests/test_tool_ordering_real_langchain.py @@ -74,6 +74,11 @@ def _emit_sdk_error(self, *a, **k): def sensitive_fail_open_enabled(self, *a, **k): return False + def is_fail_open_allowed(self, *a, **k): + """The per-tool bound on the switch (ADR-126) — nothing is + classified in this stub.""" + return False + class _Allow(_Deny): def check_workflow_budget(self, *a, **k): diff --git a/tests/test_track_batch_retry.py b/tests/test_track_batch_retry.py index 6552207..f1a6645 100644 --- a/tests/test_track_batch_retry.py +++ b/tests/test_track_batch_retry.py @@ -23,6 +23,7 @@ from __future__ import annotations import threading +import time import httpx import pytest @@ -131,7 +132,7 @@ def test_a_429_is_not_retried_before_the_server_said_it_may_be(monkeypatch): """ delays: list[float] = [] monkeypatch.setattr( - "nullrun.transport.time.sleep", lambda s: delays.append(s) + "nullrun.transport._sleep", lambda s: delays.append(s) ) calls = {"n": 0} @@ -168,7 +169,7 @@ def test_retry_after_jitter_never_reduces_the_wait_but_does_spread_it(monkeypatc """ samples: list[float] = [] monkeypatch.setattr( - "nullrun.transport.time.sleep", lambda s: samples.append(s) + "nullrun.transport._sleep", lambda s: samples.append(s) ) for _ in range(12): @@ -193,6 +194,50 @@ def _always_429() -> httpx.Response: assert len(set(samples)) > 1, f"every client retried at the same instant: {samples}" +def test_another_threads_sleep_does_not_land_in_the_retry_delays(monkeypatch): + """The retry-delay list must hold the transport's waits and nothing else. + + ``time.sleep`` is ONE attribute on ONE module object, so a test that + patches it there captures every thread's sleep in the process. The + webhook-delivery singleton idles on ``time.sleep(0.5)``; when that + value lands in this list, the floor assertion below fails for a + reason that has nothing to do with the retry path — the same flake + the webhook tests documented, one file over. + """ + delays: list[float] = [] + monkeypatch.setattr( + "nullrun.transport._sleep", lambda s: delays.append(s) + ) + + # Another thread in this process sleeps while the patch is live. + foreign = threading.Thread(target=lambda: time.sleep(0.5)) + foreign.start() + foreign.join() + + calls = {"n": 0} + + def _always_429() -> httpx.Response: + calls["n"] += 1 + resp = httpx.Response( + 200 if calls["n"] >= 2 else 429, + **({} if calls["n"] >= 2 else {"headers": {"Retry-After": "5"}}), + json={}, + ) + resp.request = httpx.Request( + "POST", "https://api.test.nullrun.io/api/v1/track/batch" + ) + resp.raise_for_status() + return resp + + _retry_with_backoff(_always_429, max_retries=1, base_delay=0.5, max_delay=10.0) + + assert delays, "the retry happened without sleeping" + assert 0.5 not in delays, ( + f"another thread's sleep landed in the retry delays: {delays}" + ) + assert min(delays) >= 5.0, f"a retry went out before the server's floor: {delays}" + + def test_a_429_without_the_header_falls_back_to_exponential_backoff(monkeypatch): """No header is not an instruction to wait forever. @@ -203,7 +248,7 @@ def test_a_429_without_the_header_falls_back_to_exponential_backoff(monkeypatch) """ delays: list[float] = [] monkeypatch.setattr( - "nullrun.transport.time.sleep", lambda s: delays.append(s) + "nullrun.transport._sleep", lambda s: delays.append(s) ) calls = {"n": 0} @@ -227,7 +272,7 @@ def test_retry_after_as_an_http_date_is_honoured(monkeypatch): delays: list[float] = [] monkeypatch.setattr( - "nullrun.transport.time.sleep", lambda s: delays.append(s) + "nullrun.transport._sleep", lambda s: delays.append(s) ) when = format_datetime( datetime.now(timezone.utc) + timedelta(seconds=4), usegmt=True @@ -255,7 +300,7 @@ def test_a_retry_after_already_in_the_past_is_not_a_floor(monkeypatch): """A stale date means "no floor stated", not "wait negative seconds".""" delays: list[float] = [] monkeypatch.setattr( - "nullrun.transport.time.sleep", lambda s: delays.append(s) + "nullrun.transport._sleep", lambda s: delays.append(s) ) calls = {"n": 0} @@ -309,7 +354,7 @@ def _handler() -> httpx.Response: def test_a_retry_after_of_a_day_does_not_sit_through_the_day(monkeypatch): """`Retry-After: 86400` must not park a flush thread for 24 hours.""" slept: list[float] = [] - monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: slept.append(s)) + monkeypatch.setattr("nullrun.transport._sleep", lambda s: slept.append(s)) monkeypatch.setattr("nullrun.transport.threading.Event.wait", lambda self, t=None: False) calls = {"n": 0} @@ -375,7 +420,7 @@ def test_a_retry_after_under_the_ceiling_is_still_honoured(monkeypatch): exists to pin. A 5s floor must still produce a >=5s wait. """ delays: list[float] = [] - monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: delays.append(s)) + monkeypatch.setattr("nullrun.transport._sleep", lambda s: delays.append(s)) calls = {"n": 0} def _then_ok() -> httpx.Response: @@ -398,7 +443,7 @@ def _then_ok() -> httpx.Response: def test_the_ceiling_is_configurable(monkeypatch): """An operator whose backend rate-limits for 5 minutes needs to say so.""" slept: list[float] = [] - monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: slept.append(s)) + monkeypatch.setattr("nullrun.transport._sleep", lambda s: slept.append(s)) monkeypatch.setenv("NULLRUN_RETRY_AFTER_CEILING", "600") before = metrics.transport.retry_after_deferred calls = {"n": 0} @@ -439,7 +484,7 @@ def _wait_then_cancel(self, timeout=None): return real_wait(self, 0.02) monkeypatch.setattr("nullrun.transport.threading.Event.wait", _wait_then_cancel) - monkeypatch.setattr("nullrun.transport.time.sleep", lambda s: pytest.fail("slept")) + monkeypatch.setattr("nullrun.transport._sleep", lambda s: pytest.fail("slept")) calls = {"n": 0} diff --git a/tests/test_track_refusal_surfaces.py b/tests/test_track_refusal_surfaces.py new file mode 100644 index 0000000..c0dde84 --- /dev/null +++ b/tests/test_track_refusal_surfaces.py @@ -0,0 +1,796 @@ +""" +SDK-6 — a refused ``/track`` must reach the application. + +The SDK's own runtime raises the refusal correctly: ``runtime.track`` +re-raises a ``NullRunDecision`` when the backend refuses to book the +money (ADR-008's table — the ``/track`` v3 single path is **CLOSED for +enforcement rejections**, so "a typed ``NullRunDecision`` +(``CONSUME_OVERBUDGET`` 422, budget block 402) propagates to the +caller"). + +The auto-instrumentation layer then caught that refusal with a blanket +``except Exception`` and logged it at DEBUG. The application saw a +perfectly ordinary LLM call while the backend had just refused to book +it — a refused ``/track`` looked like a success. + +What these tests pin: + +1. A ``NullRunDecision`` raised by the runtime escapes EVERY + auto-instrumented emission path: the httpx sync transport, the httpx + async transport, the ``requests`` transport, and the OpenAI-Agents + tracer. +2. The refusal the wire parser produces for a refused consume IS a + ``NullRunDecision`` — the rule is written in the class the backend + actually raises. +3. The fail-OPEN posture is UNCHANGED for everything else: a + transport-level failure (a dead backend) is still swallowed at DEBUG + and the caller still gets its response. +4. The refusal the caller sees is the SAME object the runtime raised — + no laundering into a generic error on the way out. + +SDK-1 (security audit 2026-11-06) widened the set of things that +surface. The rule is now one question asked of the WIRE: did the +backend ANSWER? A refusal is any answer the backend gave that was not +an acceptance — a 5xx with no enforcement code in it included, because +the consume did not land either way. A request that never reached the +backend keeps the fail-OPEN posture: the agent must not freeze on a +dead backend, and the undelivered payload goes to the WAL. +""" + +from __future__ import annotations + +import asyncio +import importlib +import json +import sys +from types import ModuleType, SimpleNamespace + +import httpx +import pytest + +from nullrun.breaker.exceptions import ( + NullRunBudgetError, + NullRunConsumeOverbudgetError, + NullRunDecision, +) +from nullrun.instrumentation.auto import ( + NullRunAsyncTransport, + NullRunSyncTransport, + _emit_from_agents_result, +) + +# --------------------------------------------------------------------------- +# Doubles +# --------------------------------------------------------------------------- + + +class _RefusingRuntime: + """A runtime double whose ``track`` raises what the real runtime + raises when the backend refuses the consume. + + ``raise_exc`` is raised AS IS — the test asserts the caller sees the + same object, not a copy. + """ + + def __init__(self, raise_exc: BaseException) -> None: + self.raise_exc = raise_exc + self.tracked: list[dict] = [] + + def check_workflow_budget(self, **kwargs): + # No reservation: the pre-flight is not what this file is about. + return None + + def track(self, event, *, reservation_id=None): + self.tracked.append(event) + raise self.raise_exc + + def track_event(self, **kwargs): + # Span emission — not money, and never a refusal. + return None + + +class _RecordingTransport(httpx.BaseTransport): + """An inner transport that answers with a canned OpenAI-shaped body.""" + + def __init__(self, body: bytes, status: int = 200) -> None: + self.body = body + self.status = status + self.sent: list[httpx.Request] = [] + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.sent.append(request) + return httpx.Response(self.status, content=self.body, request=request) + + +class _RecordingAsyncTransport(httpx.AsyncBaseTransport): + def __init__(self, body: bytes, status: int = 200) -> None: + self.body = body + self.status = status + self.sent: list[httpx.Request] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.sent.append(request) + return httpx.Response(self.status, content=self.body, request=request) + + +def _openai_body() -> bytes: + return json.dumps( + { + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "usage": {"prompt_tokens": 8, "completion_tokens": 2, "total_tokens": 10}, + "choices": [{"message": {"role": "assistant", "content": "hi"}}], + } + ).encode() + + +def _request() -> httpx.Request: + return httpx.Request( + "POST", + "https://api.openai.com/v1/chat/completions", + json={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]}, + headers={"Content-Type": "application/json"}, + ) + + +def _budget_refusal() -> NullRunBudgetError: + """The refusal the backend sends as a 402 budget block.""" + return NullRunBudgetError("wf-1", "period budget exhausted", status_code=402) + + +# --------------------------------------------------------------------------- +# 1. The httpx sync path +# --------------------------------------------------------------------------- + + +class TestTheHttpxSyncPath: + def test_a_refused_track_reaches_the_caller(self): + """The refusal must NOT be swallowed by the transport wrapper. + + Before the fix the wrapper caught it and logged it at DEBUG, so + ``handle_request`` returned a normal response and the + application never learned the backend refused to book the call. + """ + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + transport = NullRunSyncTransport(_RecordingTransport(_openai_body()), runtime) + + with pytest.raises(NullRunBudgetError) as excinfo: + transport.handle_request(_request()) + + assert excinfo.value is refusal + assert runtime.tracked, "the cost event must have been emitted" + + def test_a_successful_track_still_returns_the_response(self): + """The control: no refusal ⇒ the caller gets its response and the + body is intact.""" + runtime = _RefusingRuntime(_budget_refusal()) + runtime.track = lambda event, reservation_id=None: {"ok": True} # type: ignore[method-assign] + transport = NullRunSyncTransport(_RecordingTransport(_openai_body()), runtime) + + response = transport.handle_request(_request()) + + assert response.status_code == 200 + assert json.loads(response.content)["id"] == "chatcmpl-1" + + def test_a_transport_failure_is_still_swallowed(self): + """The fail-OPEN posture is unchanged: a DEAD backend must not + freeze the agent, so a non-decision failure stays swallowed.""" + runtime = _RefusingRuntime(RuntimeError("backend is down")) + transport = NullRunSyncTransport(_RecordingTransport(_openai_body()), runtime) + + response = transport.handle_request(_request()) + + assert response.status_code == 200 + + +# --------------------------------------------------------------------------- +# 2. The httpx async path +# --------------------------------------------------------------------------- + + +class TestTheHttpxAsyncPath: + def test_a_refused_track_reaches_the_caller(self): + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + transport = NullRunAsyncTransport(_RecordingAsyncTransport(_openai_body()), runtime) + + async def _drive(): + await transport.handle_async_request(_request()) + + with pytest.raises(NullRunBudgetError) as excinfo: + asyncio.run(_drive()) + + assert excinfo.value is refusal + + def test_a_transport_failure_is_still_swallowed(self): + runtime = _RefusingRuntime(RuntimeError("backend is down")) + transport = NullRunAsyncTransport(_RecordingAsyncTransport(_openai_body()), runtime) + + async def _drive(): + return await transport.handle_async_request(_request()) + + response = asyncio.run(_drive()) + assert response.status_code == 200 + + +# --------------------------------------------------------------------------- +# 3. The requests path +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fresh_patch_module(): + if "nullrun.instrumentation.auto_requests" in sys.modules: + importlib.reload(sys.modules["nullrun.instrumentation.auto_requests"]) + else: + importlib.import_module("nullrun.instrumentation.auto_requests") + yield + if "nullrun.instrumentation.auto_requests" in sys.modules: + importlib.reload(sys.modules["nullrun.instrumentation.auto_requests"]) + + +def _install_fake_requests(monkeypatch) -> None: + """Install a fake ``requests`` module whose ``Session.send`` answers + with an OpenAI-shaped body.""" + + class _FakeResponse: + def __init__(self, body: bytes, status_code: int): + self.content = body + self.status_code = status_code + self.headers = {"Content-Type": "application/json"} + + class _FakeSession: + send_count = 0 + _nullrun_patched = False + + @staticmethod + def send(self_or_cls, request, **kwargs): + _FakeSession.send_count += 1 + return _FakeResponse(_openai_body(), 200) + + fake_mod = ModuleType("requests") + fake_mod.Session = _FakeSession + monkeypatch.setitem(sys.modules, "requests", fake_mod) + + +class TestTheRequestsPath: + def test_a_refused_track_reaches_the_caller(self, monkeypatch, fresh_patch_module): + _install_fake_requests(monkeypatch) + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + + from requests import Session + + from nullrun.instrumentation.auto_requests import patch_requests + + assert patch_requests(runtime) is True + req = SimpleNamespace( + url="https://api.openai.com/v1/chat/completions", + headers={}, + _nullrun_tracked=False, + ) + + with pytest.raises(NullRunBudgetError) as excinfo: + Session().send(req) + + assert excinfo.value is refusal + + def test_a_transport_failure_is_still_swallowed(self, monkeypatch, fresh_patch_module): + """The pre-existing swallow for a non-decision failure stays.""" + _install_fake_requests(monkeypatch) + runtime = _RefusingRuntime(RuntimeError("down")) + + from requests import Session + + from nullrun.instrumentation.auto_requests import patch_requests + + assert patch_requests(runtime) is True + req = SimpleNamespace(url="https://api.openai.com/v1/chat/completions", headers={}) + + resp = Session().send(req) + assert resp.status_code == 200 + + +# --------------------------------------------------------------------------- +# 4. The OpenAI-Agents tracer +# --------------------------------------------------------------------------- + + +class TestTheAgentsTracer: + def test_a_refused_track_reaches_the_caller(self): + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + + result = SimpleNamespace( + _trace_spans=[ + { + "type": "llm_call", + "id": "span-1", + "model": "gpt-4o-mini", + "usage": { + "input_tokens": 11, + "output_tokens": 4, + "total_tokens": 15, + }, + } + ] + ) + + with pytest.raises(NullRunBudgetError) as excinfo: + _emit_from_agents_result(runtime, result) + + assert excinfo.value is refusal + + +# --------------------------------------------------------------------------- +# 5. The refusal the wire produces IS a decision +# --------------------------------------------------------------------------- + + +class TestTheWireRefusalIsADecision: + def test_a_refused_consume_is_a_decision(self): + """The rule is written in ``NullRunDecision`` — so the class the + BACKEND produces must be one. + + The 422 CONSUME_OVERBUDGET envelope is parsed by the real + transport and must land as a ``NullRunDecision``. + """ + from nullrun.transport import _parse_v3_error_envelope + + response = httpx.Response( + 422, + json={ + "error_code": "CONSUME_OVERBUDGET", + "error_message": "actual 12 > reserved 10 + 1", + "details": { + "execution_id": "0199aaaa-bbbb-7ccc-8ddd-eeeeffff0000", + "reserved_millicents": 10_000, + "max_allowed_millicents": 11_000, + "actual_cost_cents": 12, + "reservation_recorded": True, + }, + }, + request=httpx.Request("POST", "https://api.nullrun.io/api/v1/track"), + ) + + exc = _parse_v3_error_envelope(response, "track") + + assert isinstance(exc, NullRunConsumeOverbudgetError) + assert isinstance(exc, NullRunDecision), ( + "the enforcement refusal must be a NullRunDecision — the " + "class the instrumentation rule is written in" + ) + + +# --------------------------------------------------------------------------- +# 6. The framework integrations the auto-instrumentation layer installs +# --------------------------------------------------------------------------- +# +# ``patch_all()`` installs the httpx transport, the requests transport, +# the OpenAI-Agents tracer AND the crewai / autogen / llama_index / +# langchain integrations. A refusal swallowed in ANY of them is the same +# defect the audit named: the backend refused to book the money and the +# application was told the call succeeded. +# +# The doubles below install SYNTHETIC vendor modules, so the real +# wrapper code runs without the (heavy, optional) dependencies. + + +def _reload(module_name: str) -> None: + if module_name in sys.modules: + importlib.reload(sys.modules[module_name]) + else: + importlib.import_module(module_name) + + +@pytest.fixture +def fresh_crewai(): + _reload("nullrun.instrumentation.crewai") + yield + _reload("nullrun.instrumentation.crewai") + + +@pytest.fixture +def fresh_autogen(): + _reload("nullrun.instrumentation.autogen") + yield + _reload("nullrun.instrumentation.autogen") + + +@pytest.fixture +def fresh_llama_index(): + _reload("nullrun.instrumentation.llama_index") + yield + _reload("nullrun.instrumentation.llama_index") + + +def _install_fake_crewai(monkeypatch) -> None: + class _FakeCrew: + _nullrun_patched = False + usage_metrics: dict = {} + + @staticmethod + def kickoff(self, inputs=None, **kwargs): + return SimpleNamespace(result="ok") + + fake_mod = ModuleType("crewai") + fake_mod.Crew = _FakeCrew + monkeypatch.setitem(sys.modules, "crewai", fake_mod) + + +def _install_fake_autogen(monkeypatch) -> None: + class _Usage: + prompt_tokens = 12 + completion_tokens = 34 + total_tokens = 46 + + class _Result: + usage = _Usage() + + class _FakeOpenAIChatCompletionClient: + _nullrun_patched = False + + @staticmethod + def create(self, *args, **kwargs): + return _Result() + + class _FakeBaseChatAgent: + _nullrun_patched = False + + def on_messages(self, messages, cancellation_token=None): + return SimpleNamespace(content="ok") + + fake_agents_mod = ModuleType("autogen_agentchat.agents") + fake_agents_mod.BaseChatAgent = _FakeBaseChatAgent + monkeypatch.setitem(sys.modules, "autogen_agentchat", ModuleType("autogen_agentchat")) + monkeypatch.setitem(sys.modules, "autogen_agentchat.agents", fake_agents_mod) + + fake_ext_mod = ModuleType("autogen_ext.models.openai") + fake_ext_mod.OpenAIChatCompletionClient = _FakeOpenAIChatCompletionClient + monkeypatch.setitem(sys.modules, "autogen_ext", ModuleType("autogen_ext")) + monkeypatch.setitem(sys.modules, "autogen_ext.models", ModuleType("autogen_ext.models")) + monkeypatch.setitem(sys.modules, "autogen_ext.models.openai", fake_ext_mod) + + +def _install_fake_llama_index(monkeypatch) -> list: + captured: list = [] + + class _FakeDispatcher: + def add_event_handler(self, event_cls, handler): + captured.append((event_cls, handler)) + + def remove_event_handler(self, event_cls, handler): + for i, (cls, h) in enumerate(captured): + if cls is event_cls and h is handler: + del captured[i] + return + + events_mod = ModuleType("llama_index.core.instrumentation.events") + events_mod_llm = ModuleType("llama_index.core.instrumentation.events.llm") + events_mod_llm.LLMChatEndEvent = type("LLMChatEndEvent", (), {}) + events_mod_tool = ModuleType("llama_index.core.instrumentation.events.tool") + events_mod_tool.FunctionCallEvent = type("FunctionCallEvent", (), {}) + events_mod.llm = events_mod_llm + events_mod.tool = events_mod_tool + + inst_mod = ModuleType("llama_index.core.instrumentation") + inst_mod.get_dispatcher = lambda *a, **k: _FakeDispatcher() + monkeypatch.setitem(sys.modules, "llama_index", ModuleType("llama_index")) + monkeypatch.setitem(sys.modules, "llama_index.core", ModuleType("llama_index.core")) + monkeypatch.setitem(sys.modules, "llama_index.core.instrumentation", inst_mod) + monkeypatch.setitem(sys.modules, "llama_index.core.instrumentation.events", events_mod) + monkeypatch.setitem(sys.modules, "llama_index.core.instrumentation.events.llm", events_mod_llm) + monkeypatch.setitem( + sys.modules, "llama_index.core.instrumentation.events.tool", events_mod_tool + ) + return captured + + +class TestTheCrewAiPath: + def test_a_refused_track_reaches_the_caller(self, monkeypatch, fresh_crewai): + _install_fake_crewai(monkeypatch) + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + + from crewai import Crew + + from nullrun.instrumentation.crewai import patch_crewai + + assert patch_crewai(runtime) is True + + crew = Crew() + crew.usage_metrics = { + "gpt-4o": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + } + + with pytest.raises(NullRunBudgetError) as excinfo: + Crew.kickoff(crew, inputs={"q": "hi"}) + + assert excinfo.value is refusal + + def test_a_transport_failure_is_still_swallowed(self, monkeypatch, fresh_crewai): + _install_fake_crewai(monkeypatch) + runtime = _RefusingRuntime(RuntimeError("backend is down")) + + from crewai import Crew + + from nullrun.instrumentation.crewai import patch_crewai + + assert patch_crewai(runtime) is True + crew = Crew() + crew.usage_metrics = { + "gpt-4o": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + } + + result = Crew.kickoff(crew, inputs={"q": "hi"}) + assert result.result == "ok" + + +class TestTheAutogenPath: + def test_a_refused_track_reaches_the_caller(self, monkeypatch, fresh_autogen): + _install_fake_autogen(monkeypatch) + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + + from autogen_ext.models.openai import OpenAIChatCompletionClient + + from nullrun.instrumentation.autogen import patch_autogen + + assert patch_autogen(runtime) is True + + class _Inst: + model = "gpt-4o-mini" + + with pytest.raises(NullRunBudgetError) as excinfo: + OpenAIChatCompletionClient.create(_Inst()) + + assert excinfo.value is refusal + + def test_a_transport_failure_is_still_swallowed(self, monkeypatch, fresh_autogen): + _install_fake_autogen(monkeypatch) + runtime = _RefusingRuntime(RuntimeError("backend is down")) + + from autogen_ext.models.openai import OpenAIChatCompletionClient + + from nullrun.instrumentation.autogen import patch_autogen + + assert patch_autogen(runtime) is True + + class _Inst: + model = "gpt-4o-mini" + + result = OpenAIChatCompletionClient.create(_Inst()) + assert result.usage.prompt_tokens == 12 + + +class TestTheLlamaIndexPath: + def test_a_refused_track_reaches_the_caller(self, monkeypatch, fresh_llama_index): + captured = _install_fake_llama_index(monkeypatch) + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + + from nullrun.instrumentation.llama_index import patch_llama_index + + assert patch_llama_index(runtime) is True + + import llama_index.core.instrumentation.events.llm as _llm_events + + response = SimpleNamespace( + raw={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + model="gpt-4o", + ) + event = SimpleNamespace(response=response) + + fired = False + for cls, handler in captured: + if cls is _llm_events.LLMChatEndEvent: + with pytest.raises(NullRunBudgetError) as excinfo: + handler(event) + assert excinfo.value is refusal + fired = True + break + assert fired, "the LLMChatEndEvent handler was not registered" + + def test_a_transport_failure_is_still_swallowed(self, monkeypatch, fresh_llama_index): + captured = _install_fake_llama_index(monkeypatch) + runtime = _RefusingRuntime(RuntimeError("backend is down")) + + from nullrun.instrumentation.llama_index import patch_llama_index + + assert patch_llama_index(runtime) is True + + import llama_index.core.instrumentation.events.llm as _llm_events + + response = SimpleNamespace( + raw={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + model="gpt-4o", + ) + event = SimpleNamespace(response=response) + + for cls, handler in captured: + if cls is _llm_events.LLMChatEndEvent: + handler(event) # must not raise + break + + +class TestTheLangChainPath: + """The LangChain callback's /track emission. + + ``on_llm_end`` used to swallow the refusal in a blanket + ``except Exception`` INSIDE the method — so the refusal never + reached ``@_only_a_gate_refusal_escapes`` and the SDK-4 machinery + could not do its job. + """ + + def test_a_refused_track_reaches_the_caller(self): + from nullrun.instrumentation.langgraph import NullRunCallback + + refusal = _budget_refusal() + runtime = _RefusingRuntime(refusal) + cb = NullRunCallback(runtime=runtime) + + with pytest.raises(NullRunBudgetError) as excinfo: + cb.on_llm_end( + SimpleNamespace( + usage_metadata={ + "input_tokens": 5, + "output_tokens": 10, + "total_tokens": 15, + } + ), + invocation_params={"model_name": "gpt-4o", "model_provider": "openai"}, + ) + + assert excinfo.value is refusal + + def test_the_refusal_is_also_recorded_for_the_protect_boundary(self): + """The secondary channel: a host framework that ignores + ``raise_error`` still gets the decision at the ``@protect`` + boundary.""" + from nullrun.instrumentation.langgraph import ( + NullRunCallback, + drain_deferred_enforcement, + ) + + drain_deferred_enforcement() + runtime = _RefusingRuntime(_budget_refusal()) + cb = NullRunCallback(runtime=runtime) + + with pytest.raises(NullRunBudgetError): + cb.on_llm_end( + SimpleNamespace(usage_metadata={"total_tokens": 15}), + invocation_params={"model_name": "gpt-4o"}, + ) + + deferred = drain_deferred_enforcement() + assert isinstance(deferred, NullRunBudgetError) + + def test_a_transport_failure_is_still_swallowed(self): + from nullrun.instrumentation.langgraph import NullRunCallback + + runtime = _RefusingRuntime(RuntimeError("backend is down")) + cb = NullRunCallback(runtime=runtime) + + cb.on_llm_end( + SimpleNamespace(usage_metadata={"total_tokens": 15}), + invocation_params={"model_name": "gpt-4o"}, + ) # must not raise + + +# --------------------------------------------------------------------------- +# 7. End to end: the REAL runtime behind the REAL transport +# --------------------------------------------------------------------------- +# +# The two halves are proven separately — the runtime re-raises the +# refusal (``tests/test_route_track_enforcement_propagates.py`` drives +# the real runtime against a mocked wire) and the transport does not +# swallow it (the classes above, with a double runtime). +# +# This joins them, because the seam IS the defect: a real +# ``NullRunRuntime`` behind a real ``NullRunSyncTransport``, with the +# backend mocked at the wire, must surface the refusal to the +# application. A source-pin cannot prove that — the whole defect is +# "what the transport does with what the runtime raised". + +CONSUMED_TOO_MUCH = { + "error_code": "CONSUME_OVERBUDGET", + "error_message": "actual > reserved + epsilon", + "details": { + "execution_id": "0190c5b5-7c9a-7def-8a1b-0123456789ab", + "reserved_millicents": 1000, + "max_allowed_millicents": 1010, + "actual_cost_cents": 150, + "soft_pass": False, + "reservation_recorded": True, + }, +} + + +class TestTheEndToEndPath: + def test_a_refused_track_reaches_the_caller(self, make_runtime): + import respx + from httpx import Response + + from nullrun.instrumentation.auto import NullRunSyncTransport + from nullrun.runtime import _capture_server_minted_execution_id + + rt = make_runtime() + # Bind a server-minted id so ``_route_track`` reaches + # ``track_single`` instead of dropping on "no reservation". + _capture_server_minted_execution_id({"reservation_id": "0190c5b5-7c9a-7def-8a1b-0123456789ab"}) + + respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response(200, json=json.loads(_openai_body())) + ) + respx.post("https://api.test.nullrun.io/api/v1/track").mock( + return_value=Response(422, json=CONSUMED_TOO_MUCH) + ) + + transport = NullRunSyncTransport(_RecordingTransport(_openai_body()), rt) + + with pytest.raises(NullRunConsumeOverbudgetError) as excinfo: + transport.handle_request(_request()) + + assert excinfo.value.status_code == 422 + assert excinfo.value.actual_cost_cents == 150 + + def test_a_transport_failure_is_still_swallowed(self, make_runtime): + """The fail-OPEN half, end to end: a dead backend must not + freeze the agent. + + "A dead backend" is a request that never got an answer. A 5xx + is an ANSWER, and it is in the other half — see the test below. + """ + import respx + from httpx import Response + + from nullrun.instrumentation.auto import NullRunSyncTransport + from nullrun.runtime import _capture_server_minted_execution_id + + rt = make_runtime() + _capture_server_minted_execution_id({"reservation_id": "0190c5b5-7c9a-7def-8a1b-0123456789ab"}) + + respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response(200, json=json.loads(_openai_body())) + ) + respx.post("https://api.test.nullrun.io/api/v1/track").mock( + side_effect=httpx.ConnectError("boom") + ) + + transport = NullRunSyncTransport(_RecordingTransport(_openai_body()), rt) + + response = transport.handle_request(_request()) + + assert response.status_code == 200 + + def test_a_refused_track_surfaces_through_the_auto_layer(self, make_runtime): + """SDK-1 (security audit 2026-11-06). + + The backend ANSWERED and refused the consume. The application + must not see an ordinary LLM call: the spend the backend + refused to book is not a success, and the auto layer that + swallows transport failures must not swallow a refusal. + """ + import respx + from httpx import Response + + from nullrun.instrumentation.auto import NullRunSyncTransport + from nullrun.runtime import _capture_server_minted_execution_id + + rt = make_runtime() + _capture_server_minted_execution_id({"reservation_id": "0190c5b5-7c9a-7def-8a1b-0123456789ab"}) + + respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response(200, json=json.loads(_openai_body())) + ) + respx.post("https://api.test.nullrun.io/api/v1/track").mock( + return_value=Response(500, json={"error_code": "INTERNAL", "error_message": "boom"}) + ) + + transport = NullRunSyncTransport(_RecordingTransport(_openai_body()), rt) + + with pytest.raises(Exception) as ei: + transport.handle_request(_request()) + + assert "track" in str(ei.value).lower() diff --git a/tests/test_v3_track_recovery.py b/tests/test_v3_track_recovery.py new file mode 100644 index 0000000..78ac2b7 --- /dev/null +++ b/tests/test_v3_track_recovery.py @@ -0,0 +1,497 @@ +"""SDK-1 (security audit 2026-11-06) — the money-bearing ``/track`` path. + +The SDK's v3 single-event ``/track`` carries a CONSUME: it settles the +reservation the matching ``/gate`` made. When that call does not land, +the tool call has ALREADY run and the money has ALREADY been spent — +but nothing booked it. The pre-fix SDK swallowed the failure, logged +"event dropped", and returned ``{"allowed": True}`` to the agent. + +Two properties are asserted here, and they are the two halves of the +same invariant: **a consume is either BOOKED or RECOVERABLE, and a +refusal is never reported as a success.** + +1. **Recoverable.** Every failure that did not get an answer leaves the + payload in the WAL, so a restart replays it to ``/api/v1/track`` + (single) and the money lands. +2. **Visible.** When the backend ANSWERED and refused the consume, the + SDK raises. + +Property 2 is the one the pre-fix taxonomy got wrong. The SDK decided +open-vs-closed by exception CLASS, and the class hierarchy puts a +backend refusal (``NullRunBackendError``) under +``NullRunInfrastructureError`` — a SIBLING of ``NullRunDecision`` — so +``isinstance(exc, NullRunDecision)`` could never catch a 503 +``RESERVATION_NOT_FOUND``. A later attempt narrowed the rule to three +classes and still dropped the nine backend codes outside them (the +``PROTOCOL_*`` family, the two idempotency codes and the three +``RATE_LIMIT_*`` codes) — and the backend rate-limits ``/track``, so a +429 there was a dropped spend. + +The rule now is one question asked of the WIRE: **did the backend +answer?** ADR-008's fail-OPEN covers exactly one thing — the gate could +not be REACHED, so there is no answer to honour and a dead backend must +not freeze the agent loop. Every other case is a refusal. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +import pytest +import respx +from httpx import Response + +BASE_URL = "https://api.test.nullrun.io" +TRACK_URL = f"{BASE_URL}/api/v1/track" +BATCH_URL = f"{BASE_URL}/api/v1/track/batch" + +RESERVATION = "0190c5b5-7c9a-7def-8a1b-0123456789ab" +IDEMPOTENCY_KEY = "9f0c1a44-1d2e-4a3b-8c5d-6e7f8091a2b3" + +# The backend's real 422 body (`handlers.rs`, ConsumeOverbudget arm): +# the reservation is in MILLICENTS and the actual cost in cents. +CONSUMED_TOO_MUCH = { + "error_code": "CONSUME_OVERBUDGET", + "error_message": "actual > reserved + epsilon", + "details": { + "execution_id": RESERVATION, + "reserved_millicents": 1000, + "max_allowed_millicents": 1010, + "actual_cost_cents": 150, + "soft_pass": False, + "reservation_recorded": True, + }, +} + + +@pytest.fixture +def wal_runtime(monkeypatch, tmp_path, make_runtime): + """A runtime whose WAL lives in ``tmp_path``. + + The WAL path is resolved ONCE, in the Transport constructor, so the + environment variable has to be in place BEFORE the runtime exists. + """ + wal = tmp_path / "sdk.wal" + monkeypatch.setenv("NULLRUN_WAL_PATH", str(wal)) + rt = make_runtime() + yield rt, Path(f"{wal}.v3") + + +@pytest.fixture +def reservation(): + """Bind a server-minted reservation and its idempotency key. + + The capture goes through the runtime's own helper, because the + runtime also captures the MOMENT of the /check: a /track whose + /check has aged past the reservation TTL is refused, and a capture + with no timestamp reads as infinitely old. + + The idempotency key is not decoration: the backend DELETEs the + reservation key after the first successful consume, so a replay + that arrives without the key is answered 503 + ``RESERVATION_NOT_FOUND`` and the row could never leave the WAL. + """ + from nullrun.context import ( + clear_server_minted_execution_id, + reset_operation_id, + set_operation_id, + ) + from nullrun.runtime import _capture_server_minted_execution_id + + token_op = set_operation_id(IDEMPOTENCY_KEY) + _capture_server_minted_execution_id({"reservation_id": RESERVATION}) + yield RESERVATION, IDEMPOTENCY_KEY + clear_server_minted_execution_id() + reset_operation_id(token_op) + + +def _wal_rows(path: Path) -> list[dict]: + """The rows of the v3 WAL, parsed.""" + if not path.exists(): + return [] + rows = [] + for line in path.read_text().splitlines(): + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return rows + + +def _track(rt, **kwargs) -> None: + """Send one llm_call through the v3 single-track path.""" + rt.track_llm( + input_tokens=kwargs.pop("input_tokens", 60), + output_tokens=kwargs.pop("output_tokens", 40), + model="claude-sonnet-4-6", + **kwargs, + ) + + +class TestARefusalIsNotASuccess: + """The backend answered and refused. The SDK must not say "allowed".""" + + @respx.mock + def test_a_503_reservation_not_found_raises(self, wal_runtime, reservation): + """The audit's named case: a 503 the SDK used to swallow.""" + from nullrun.breaker.exceptions import NullRunBackendError + + rt, _wal = wal_runtime + respx.post(TRACK_URL).mock( + return_value=Response( + 503, + json={ + "error_code": "RESERVATION_NOT_FOUND", + "error_message": "no reservation for this id", + }, + ) + ) + + with pytest.raises(NullRunBackendError): + _track(rt) + + @respx.mock + def test_a_429_rate_limit_raises(self, wal_runtime, reservation): + """The backend rate-limits ``/track``. A 429 here was one of + the nine codes the pre-fix class list dropped — a live lost + spend. + """ + from nullrun.breaker.exceptions import RateLimitError + + rt, _wal = wal_runtime + respx.post(TRACK_URL).mock( + return_value=Response( + 429, + json={ + "error_code": "RATE_LIMIT_EXCEEDED", + "error_message": "too many requests", + }, + headers={"Retry-After": "1"}, + ) + ) + + with pytest.raises(RateLimitError): + _track(rt) + + @respx.mock + def test_a_500_internal_error_raises(self, wal_runtime, reservation): + rt, _wal = wal_runtime + respx.post(TRACK_URL).mock( + return_value=Response( + 500, + json={"error_code": "INTERNAL_ERROR", "error_message": "boom"}, + ) + ) + + with pytest.raises(Exception): + _track(rt) + + @respx.mock + def test_a_refusal_without_an_error_code_raises(self, wal_runtime, reservation): + """The body is not the question. The ANSWER is. + + A bare 502 from a proxy names no enforcement failure, but it is + still an answer, and the consume still did not land. The + pre-fix rule asked "does this code name an enforcement + failure?" — that question let a refused consume through + whenever the body happened to be plain. + """ + rt, _wal = wal_runtime + respx.post(TRACK_URL).mock(return_value=Response(502, text="bad gateway")) + + with pytest.raises(Exception): + _track(rt) + + +class TestTheRefusalRule: + """The rule itself, asked of the wire and not of the class.""" + + def test_a_stamped_wire_status_is_a_refusal(self): + from nullrun.breaker.exceptions import NullRunBackendError + from nullrun.transport import is_backend_refusal + + exc = NullRunBackendError("nope", endpoint="track", status_code=503) + exc.wire_status_code = 503 # type: ignore[attr-defined] + + assert is_backend_refusal(exc) is True + + def test_a_transport_error_is_not_a_refusal(self): + """No answer is not an answer. ADR-008's fail-OPEN row.""" + from nullrun.breaker.exceptions import NullRunTransportError + from nullrun.transport import is_backend_refusal + + exc = NullRunTransportError( + "network error on /track", + source=__import__( + "nullrun.breaker.exceptions", fromlist=["TransportErrorSource"] + ).TransportErrorSource.NETWORK_ERROR, + endpoint="track", + ) + + assert is_backend_refusal(exc) is False + + def test_the_nine_codes_the_pre_fix_list_dropped_are_refusals(self): + """Every code the narrowed class list left out is a refusal. + + The list named three exception classes and left nine backend + codes outside them. The backend rate-limits ``/track``, so + three of the nine were live lost spends. + """ + from nullrun.breaker.exceptions import ( + NullRunAuthError, + NullRunBackendError, + NullRunProtocolError, + NullRunRateLimitRedisError, + RateLimitError, + ) + from nullrun.transport import is_backend_refusal + + cases = [ + NullRunProtocolError("bad protocol header"), + NullRunProtocolError("protocol too new"), + NullRunBackendError("conflict", endpoint="track", status_code=409), + NullRunBackendError("key mismatch", endpoint="track", status_code=409), + RateLimitError( + "rate limited", + source=__import__( + "nullrun.breaker.exceptions", fromlist=["TransportErrorSource"] + ).TransportErrorSource.GATEWAY_ERROR, + endpoint="track", + ), + NullRunRateLimitRedisError("limiter redis down"), + NullRunAuthError("bad key"), + ] + for exc in cases: + exc.wire_status_code = 400 # type: ignore[attr-defined] + assert is_backend_refusal(exc) is True, exc + + def test_a_decision_and_an_infrastructure_error_are_siblings(self): + """The fact that made the pre-fix rule unsound. + + ``NullRunDecision`` and ``NullRunInfrastructureError`` are + SIBLINGS, so ``isinstance(exc, NullRunDecision)`` can never + catch a backend refusal — no matter how the refusal is + classified. + """ + from nullrun.breaker.exceptions import ( + NullRunBackendError, + NullRunDecision, + NullRunInfrastructureError, + ) + + assert issubclass(NullRunBackendError, NullRunInfrastructureError) + assert not issubclass(NullRunInfrastructureError, NullRunDecision) + assert not issubclass(NullRunDecision, NullRunInfrastructureError) + + +class TestTheConsumeIsRecoverable: + """Nothing is dropped. Every undelivered payload is on disk.""" + + @respx.mock + def test_a_transport_failure_leaves_the_payload_in_the_wal(self, wal_runtime, reservation): + """The audit's named verification: an ``httpx.ConnectError`` on + ``/track`` must leave the event recoverable in the WAL. + """ + rt, wal = wal_runtime + respx.post(TRACK_URL).mock(side_effect=httpx.ConnectError("boom")) + + # A dead backend must not freeze the agent — ADR-008's + # fail-OPEN row, unchanged. + _track(rt) + + rows = _wal_rows(wal) + assert len(rows) == 1, "the payload was dropped" + assert rows[0]["_nullrun_wal"] == "v3_track" + + @respx.mock + def test_the_wal_row_carries_the_reservation_and_the_idempotency_key( + self, wal_runtime, reservation + ): + """Verbatim. The replay sends back exactly what was built. + + Without the idempotency key the backend DELETEs the + reservation key after the first consume, so the replay would be + answered 503 ``RESERVATION_NOT_FOUND`` and the row could never + leave the file. + + The row is compared against the bytes that went on the wire, + not against a constant: the idempotency key is compound with + the span id by design, so the value that matters is the one + the backend saw. + """ + rt, wal = wal_runtime + route = respx.post(TRACK_URL).mock(side_effect=httpx.ConnectError("boom")) + + _track(rt) + + sent = json.loads(route.calls[0].request.content) + payload = _wal_rows(wal)[0]["payload"] + assert payload["reservation_id"] == RESERVATION + assert payload["tokens"] == 100 + assert payload["idempotency_key"] == sent["idempotency_key"] + assert payload["idempotency_key"].startswith(IDEMPOTENCY_KEY) + + @respx.mock + def test_a_refusal_leaves_the_payload_in_the_wal(self, wal_runtime, reservation): + """A refusal is raised AND recoverable. + + The backend answered, so the consume did not land, so the + payload is the only copy of an unbooked spend. Raising tells + the operator; the WAL tells the next process. + """ + rt, wal = wal_runtime + respx.post(TRACK_URL).mock( + return_value=Response( + 503, + json={"error_code": "RESERVATION_NOT_FOUND", "error_message": "no"}, + ) + ) + + with pytest.raises(Exception): + _track(rt) + + assert len(_wal_rows(wal)) == 1 + + @respx.mock + def test_a_verdict_is_not_persisted(self, wal_runtime, reservation): + """A ``NullRunDecision`` is the backend's ANSWER, not a failure. + + Re-sending a verdict produces the same verdict, so a row for + it could never leave the WAL — it would grow the file forever + and prove nothing. The consume it refuses was ACCOUNTED for + (the backend charges the overbudget cost), so there is no + unbooked spend to recover. + """ + from nullrun.breaker.exceptions import NullRunConsumeOverbudgetError + + rt, wal = wal_runtime + respx.post(TRACK_URL).mock(return_value=Response(422, json=CONSUMED_TOO_MUCH)) + + with pytest.raises(NullRunConsumeOverbudgetError): + _track(rt) + + assert _wal_rows(wal) == [] + + @respx.mock + def test_a_successful_track_leaves_no_row(self, wal_runtime, reservation): + rt, wal = wal_runtime + respx.post(TRACK_URL).mock(return_value=Response(200, json={"ok": True})) + + _track(rt) + + assert _wal_rows(wal) == [] + + +class TestTheReplayRoute: + """A restart replays to the SINGLE endpoint. + + The batch endpoint does not run the CONSUME script: a reservation + replayed through it is never settled and the money is never + booked. + """ + + @respx.mock + def test_the_replay_posts_to_the_single_track_endpoint(self, wal_runtime, reservation): + rt, wal = wal_runtime + route = respx.post(TRACK_URL).mock(side_effect=httpx.ConnectError("boom")) + _track(rt) + assert len(_wal_rows(wal)) == 1 + + # The next process starts and replays. The route is the SAME + # respx route (one pattern, one route), so the calls before + # the replay are counted off before it starts. + route.mock(return_value=Response(200, json={"idempotent_replay": True})) + before = len(route.calls) + rt._transport._replay_v3_from_wal() + + replayed = route.calls[before:] + assert len(replayed) == 1, "the replay did not reach /api/v1/track" + assert replayed[0].request.url.path == "/api/v1/track" + sent = json.loads(replayed[0].request.content) + assert sent["reservation_id"] == RESERVATION + assert sent["idempotency_key"].startswith(IDEMPOTENCY_KEY) + + # The row cleared: the spend is booked, not pending. + assert _wal_rows(wal) == [] + + @respx.mock + def test_the_replay_never_uses_the_batch_endpoint(self, wal_runtime, reservation): + rt, wal = wal_runtime + respx.post(TRACK_URL).mock(side_effect=httpx.ConnectError("boom")) + _track(rt) + + batch = respx.post(BATCH_URL).mock(return_value=Response(200, json={})) + rt._transport._replay_v3_from_wal() + + assert not batch.called, "the batch endpoint does not settle a reservation" + + @respx.mock + def test_a_refused_row_stays_and_does_not_block_the_rows_behind_it( + self, wal_runtime, reservation + ): + """One stuck row must not block every spend behind it. + + The rows are independent reservations. The refusal is about + this consume, not about the backend, so the next row still + gets its chance. + """ + rt, wal = wal_runtime + + def _answer(request): + body = json.loads(request.content) + if body.get("reservation_id") == RESERVATION: + return Response( + 503, + json={ + "error_code": "RESERVATION_NOT_FOUND", + "error_message": "no", + }, + ) + return Response(200, json={"idempotent_replay": True}) + + route = respx.post(TRACK_URL).mock(side_effect=_answer) + with pytest.raises(Exception): + _track(rt) + + # A second, different spend lands behind the stuck one. + rt._transport.persist_v3_track( + { + "reservation_id": "0190c5b5-7c9a-7def-8a1b-000000000000", + "workflow_id": "wf-1", + "tokens": 10, + "idempotency_key": "key-2", + } + ) + + before = len(route.calls) + rt._transport._replay_v3_from_wal() + + assert len(route.calls) - before == 2, "the stuck row blocked the row behind it" + rows = _wal_rows(wal) + assert len(rows) == 1 + assert rows[0]["payload"]["reservation_id"] == RESERVATION + + +class TestTheWalPathIsFixed: + def test_the_path_does_not_move_while_the_transport_lives( + self, monkeypatch, tmp_path, make_runtime + ): + """The path is resolved ONCE, in the constructor. + + A path that moves mid-life would split one agent's WAL across + two files, and the recovery would find neither half. + """ + wal = tmp_path / "sdk.wal" + monkeypatch.setenv("NULLRUN_WAL_PATH", str(wal)) + rt = make_runtime() + + before = rt._transport._wal_path() + rt._transport.persist_v3_track( + {"reservation_id": RESERVATION, "workflow_id": "wf-1", "tokens": 1} + ) + after = rt._transport._wal_path() + + assert before == after + assert Path(f"{wal}.v3").exists() diff --git a/tests/test_v3_wire_contract.py b/tests/test_v3_wire_contract.py index 93be48b..e6e4844 100644 --- a/tests/test_v3_wire_contract.py +++ b/tests/test_v3_wire_contract.py @@ -465,6 +465,27 @@ def test_chain_cross_org_maps_to_chain_error(self): exc = _parse_v3_error_envelope(resp, "check") assert isinstance(exc, NullRunChainError) + def test_chain_key_mismatch_maps_to_chain_error(self): + """ADR-180 — the chain belongs to the API key that created it. + + The backend refuses a chain presented by another key of the + same organization with ``CHAIN_KEY_MISMATCH`` (403). Without a + map entry the dispatcher falls through to the base class and a + caller branching on ``NullRunChainError`` cannot catch it. + """ + resp = self._make_response( + 403, + { + "error_code": "CHAIN_KEY_MISMATCH", + "error_message": "chain belongs to another api key", + "details": {"chain_id": "abc"}, + }, + ) + exc = _parse_v3_error_envelope(resp, "check") + assert isinstance(exc, NullRunChainError) + assert exc.backend_code == "CHAIN_KEY_MISMATCH" + assert exc.chain_id == "abc" + def test_workflow_inactive_maps_to_workflow_inactive_error(self): resp = self._make_response( 403, @@ -614,6 +635,10 @@ def test_catalog_covers_all_documented_codes(self): "CHAIN_MAX_DURATION_EXCEEDED", "CHAIN_CROSS_ORG", "CHAIN_ORG_MISMATCH", + # ADR-180 (2026-11-07): a chain belongs to the API key that + # created it. The backend emits this code when another key + # of the same organization presents a chain it does not own. + "CHAIN_KEY_MISMATCH", "WORKFLOW_INACTIVE", "API_KEY_REVOKED", "CONSUME_OVERBUDGET", @@ -1095,234 +1120,86 @@ def test_execution_id_is_uuidv7_format(self): # ───────────────────────────────────────────────────────────────────── -# BUG #5 — In-process gate cache for chain-mode +# SDK-1 — the in-process gate cache is GONE (regression pins) # ───────────────────────────────────────────────────────────────────── +# +# There was a chain-mode short-circuit keyed on +# ``(workflow_id, chain_id, call_model, estimated_tokens)`` that served a +# previous ``/gate`` answer for up to five seconds. It was not a cache, +# it was a second authorisation path with no gate behind it: +# +# * the key carried no org id, no api key id and no tool set, so one +# chain's "allow" could be served to another chain, or to a +# different tool inside the same chain, whose call was never +# checked; +# * the cached body carried call #1's ``reservation_id`` and the +# runtime re-captured it for every later step, so every step of a +# chain ran under ONE server-minted execution id. The backend +# settles an execution id at most once (``consume_v3.lua`` answers +# IDEMPOTENT_REPLAY for an execution that is already ``consumed`` +# and books nothing), so a three-step chain performed three tool +# executions and produced ONE accounting event — the money for +# steps 2 and 3 was never billed; +# * the tool-block check never ran for a cached step either. +# +# The backend owns the chain: ``chain_op`` (start / continue / end) and +# the chain TTL are enforced there, and every ``/gate`` inside a chain +# reserves and settles its own envelope. The backend's own idempotency +# store (``idempotency_key`` = the SDK's ``operation_id``) dedupes a +# genuine RETRY of one logical action and nothing else. +# Three distinct server-minted reservation ids (uuidv7 layout — the +# version nibble is the first hex char of the third group). +_SDK1_RESERVATIONS = [ + "0190c5b5-7c9a-7def-8a1b-0123456789ab", + "0190c5b5-7c9a-7def-8a1b-fedcba987654", + "0190c5b5-7c9a-7def-8a1b-0123456789cd", +] -class TestGateCache: - """BUG #5 (2026-07-04): chain-mode /check calls should be served - from an in-process 5s TTL cache, not hit /gate every time. - Single-shot (Hard mode) callers MUST NOT cache. - - These tests pin the cache data-structure invariants + opt-out - behavior. The runtime-level integration (10 chain-mode calls - collapse to 1 HTTP roundtrip) is covered by an end-to-end smoke - against the live API per docs/runbooks/budget-blue-green-smoke.sh - Invariant 12. The runtime construction needed for in-process - respx-mocked tests has its own env-bypass quirks; the data - structure tests below are the durable contract.""" - - def setup_method(self): - from nullrun import runtime - runtime._GATE_CACHE.clear() - - def test_cache_is_dict_with_ttl_5s(self): - from nullrun import runtime - assert isinstance(runtime._GATE_CACHE, dict) - assert runtime._GATE_CACHE_TTL_SECONDS == 5.0 - def test_store_and_retrieve_within_ttl(self): - import time as _time +class TestNoInProcessGateCache: + """The gate cache must not come back in any shape.""" + def test_the_runtime_module_has_no_gate_cache(self): from nullrun import runtime - k = ("wf-x", "chain-y", "model-z", 1) - runtime._GATE_CACHE[k] = (_time.monotonic(), {"decision": "allow"}) - cached = runtime._GATE_CACHE.get(k) - assert cached is not None - assert cached[1]["decision"] == "allow" - - def test_per_chain_cache_key_isolation(self): - import time as _time - from nullrun import runtime - k1 = ("wf-x", "chain-A", "model-z", 1) - k2 = ("wf-x", "chain-B", "model-z", 1) - runtime._GATE_CACHE[k1] = (_time.monotonic(), {"decision": "allow"}) - runtime._GATE_CACHE[k2] = (_time.monotonic(), {"decision": "block"}) - assert runtime._GATE_CACHE.get(k1)[1]["decision"] == "allow" - assert runtime._GATE_CACHE.get(k2)[1]["decision"] == "block" - - def test_cache_gate_disabled_when_no_chain_id(self): - # Mirror the runtime's cache_enabled predicate: - # chain_id is not None AND NULLRUN_GATE_CACHE_DISABLE != "1" - import os - os.environ["NULLRUN_GATE_CACHE_DISABLE"] = "" - chain_id = None - cache_enabled = ( - chain_id is not None - and not os.environ.get("NULLRUN_GATE_CACHE_DISABLE", "").strip() == "1" + assert not hasattr(runtime, "_GATE_CACHE"), ( + "the in-process gate cache is gone (SDK-1): a client-side " + "short-circuit is a second authorisation path with no gate " + "behind it" ) - assert cache_enabled is False + assert not hasattr(runtime, "_GATE_CACHE_TTL_SECONDS") + assert not hasattr(runtime, "_invalidate_gate_cache_for_chain") - def test_cache_gate_disabled_via_env(self): - import os - os.environ["NULLRUN_GATE_CACHE_DISABLE"] = "1" - chain_id = "chain-y" - cache_enabled = ( - chain_id is not None - and not os.environ.get("NULLRUN_GATE_CACHE_DISABLE", "").strip() == "1" - ) - assert cache_enabled is False - os.environ.pop("NULLRUN_GATE_CACHE_DISABLE", None) - - def test_invalidate_drops_only_matching_chain(self): - # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): the - # invalidation must drop ONLY entries whose chain_id matches. - # Pre-fix the invalidation read chain_id from - # ``wire_event.get("chain_id")`` which is always None - # (chain_id lives in the contextvar, not on the per-call - # wire_event dict). That bug over-invalidated to all chains - # for the same workflow_id — safe but wasteful, and it - # leaked cache pressure onto unrelated chains. Pin that - # the helper accepts an explicit chain_id and only drops - # entries with that exact chain_id. - import time as _time + def test_check_workflow_budget_has_no_use_gate_cache_switch(self): + import inspect - from nullrun import runtime - - k_failing = ("wf-x", "chain-fail", "model-z", 1) - k_other = ("wf-x", "chain-other", "model-z", 1) - runtime._GATE_CACHE[k_failing] = (_time.monotonic(), {"decision": "allow"}) - runtime._GATE_CACHE[k_other] = (_time.monotonic(), {"decision": "allow"}) - - dropped = runtime._invalidate_gate_cache_for_chain("wf-x", "chain-fail") - assert dropped == 1 - assert k_failing not in runtime._GATE_CACHE - assert k_other in runtime._GATE_CACHE # unrelated chain preserved - - def test_invalidate_with_none_chain_id_drops_all_for_workflow(self): - # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): the - # helper's ``chain_id is None`` branch drops EVERY entry - # for the matching workflow_id regardless of chain. This - # is the rare-but-defensive path — the caller chose not to - # scope by chain (e.g. transport-layer aggregate that - # doesn't know the chain). Pin the branch so a future - # refactor that breaks the None-OR-match predicate fails - # here before the wire layer regresses. - import time as _time - - from nullrun import runtime - - k_a = ("wf-y", "chain-a", "model-z", 1) - k_b = ("wf-y", "chain-b", "model-z", 1) - k_other_wf = ("wf-other", "chain-a", "model-z", 1) - runtime._GATE_CACHE[k_a] = (_time.monotonic(), {"decision": "allow"}) - runtime._GATE_CACHE[k_b] = (_time.monotonic(), {"decision": "allow"}) - runtime._GATE_CACHE[k_other_wf] = (_time.monotonic(), {"decision": "allow"}) - - dropped = runtime._invalidate_gate_cache_for_chain("wf-y", None) - assert dropped == 2 - assert k_a not in runtime._GATE_CACHE - assert k_b not in runtime._GATE_CACHE - assert k_other_wf in runtime._GATE_CACHE # unrelated workflow preserved - - def test_invalidate_with_none_workflow_id_drops_all_for_chain(self): - # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): the - # mirror branch — ``workflow_id is None`` drops EVERY entry - # for the matching chain_id regardless of workflow. Same - # defensive-rationale as the chain_id=None case but in the - # opposite dimension. - import time as _time - - from nullrun import runtime + from nullrun.runtime import NullRunRuntime - k_a = ("wf-a", "chain-z", "model-z", 1) - k_b = ("wf-b", "chain-z", "model-z", 1) - k_other_chain = ("wf-a", "chain-other", "model-z", 1) - runtime._GATE_CACHE[k_a] = (_time.monotonic(), {"decision": "allow"}) - runtime._GATE_CACHE[k_b] = (_time.monotonic(), {"decision": "allow"}) - runtime._GATE_CACHE[k_other_chain] = (_time.monotonic(), {"decision": "allow"}) - - dropped = runtime._invalidate_gate_cache_for_chain(None, "chain-z") - assert dropped == 2 - assert k_a not in runtime._GATE_CACHE - assert k_b not in runtime._GATE_CACHE - assert k_other_chain in runtime._GATE_CACHE # unrelated chain preserved - - def test_invalidate_with_both_none_drops_everything(self): - # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): both - # dimensions None → drop the whole cache. Reserved for the - # never-seen-in-prod "drop everything, the world is on fire" - # path (e.g. manual operator flush via debug API). - import time as _time + params = inspect.signature(NullRunRuntime.check_workflow_budget).parameters + assert "use_gate_cache" not in params, ( + "the cache switch is gone with the cache: a switch that " + "re-enables it is the same defect one level down" + ) - from nullrun import runtime + def test_the_cache_disable_env_var_is_gone(self): + # The opt-out env var is dead. It must not linger as a way to + # re-enable (or, worse, to be mistaken for) the cache. + import inspect - runtime._GATE_CACHE[("wf-1", "c-1", "m", 1)] = (_time.monotonic(), {"decision": "allow"}) - runtime._GATE_CACHE[("wf-2", "c-2", "m", 1)] = (_time.monotonic(), {"decision": "allow"}) - - dropped = runtime._invalidate_gate_cache_for_chain(None, None) - assert dropped == 2 - assert runtime._GATE_CACHE == {} - - def test_invalidate_on_empty_cache_returns_zero(self): - # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): the - # ``if not _GATE_CACHE: return 0`` short-circuit when the - # cache is empty. Without this branch the helper would - # still scan an empty dict and return 0, but the early - # return avoids building the (empty) list comprehension - # at all. Pin the branch so a future refactor that drops - # the short-circuit (e.g. someone replaces it with a - # `len(...) == 0` check that's a different shape) is caught - # before it ships. from nullrun import runtime - # Cache is empty (setup_method clears it). - assert runtime._GATE_CACHE == {} - dropped = runtime._invalidate_gate_cache_for_chain("wf-anything", "chain-anything") - assert dropped == 0 + source = inspect.getsource(runtime) + assert "NULLRUN_GATE_CACHE_DISABLE" not in source -# ───────────────────────────────────────────────────────────────────── -# BUG #5 — chain-mode gate cache at the runtime level -#`) -# ───────────────────────────────────────────────────────────────────── -# -# The TestGateCache data-structure tests above pin the runtime's -# `_GATE_CACHE` dict invariants in isolation; this class drives the -# full NullRunRuntime.check_workflow_budget path so the -# cache_enabled predicate + cache hit/miss branches in -# ``runtime.py:1287-1310`` are actually exercised end-to-end. Without -# these tests ``pytest-cov`` reports that exact range as uncovered -# which dragged patch coverage on PR #52 below the 70% Codecov floor. - - -class TestGateCacheRuntimeFlow: - """Runtime-level chain-mode gate cache coverage. - - Drives ``NullRunRuntime.check_workflow_budget `` inside - ``with workflow(...) + with chain(...)`` and verifies the - /gate roundtrip count vs. expected after the 5s in-process - cache is applied. +class TestChainStepsAreAdmittedIndividually: + """SDK-1 at the runtime level: every step of a chain is its own + ``/gate`` call and its own reservation. """ - def setup_method(self): - from nullrun import runtime as rt_mod - - rt_mod._GATE_CACHE.clear() - - def teardown_method(self): - from nullrun import runtime as rt_mod - - rt_mod._GATE_CACHE.clear() - # Always unset the gate-cache-disable opt-out so tests don't - # leak state between runs. - import os - - os.environ.pop("NULLRUN_GATE_CACHE_DISABLE", None) - @respx.mock - def test_chain_mode_collapses_three_checks_to_one_gate_call(self): - """3 consecutive check_workflow_budget inside `with chain(...)` - must hit /gate exactly ONCE — the 2nd and 3rd calls fall - into the cache hit branch (runtime.py:1302). - - Covers: - runtime.py:1291-1310 (cache_enabled predicate) - runtime.py:1302 (cache hit `response = cached[1]`) - runtime.py:1306 (cache miss → transport.check + store). - """ - + def test_three_chain_checks_make_three_gate_calls(self): respx.post(f"{BASE_URL}/api/v1/gate").mock( return_value=Response( 200, @@ -1336,20 +1213,16 @@ def test_chain_mode_collapses_three_checks_to_one_gate_call(self): polling=False, # no background WS/HTTP poll thread ) try: - with workflow("wf-runtime-cache") as _wf_id, chain( - str(uuid.uuid4()) - ) as _cid: - # Direct calls in chain scope — bypasses @protect but - # exercises the same check_workflow_budget codepath. + with workflow("wf-sdk1"), chain(str(uuid.uuid4())): rt_inst.check_workflow_budget() rt_inst.check_workflow_budget() rt_inst.check_workflow_budget() gate_calls = [ c for c in respx.calls if c.request.url.path.endswith("/gate") ] - assert len(gate_calls) == 1, ( - f"chain-mode cache must collapse 3 calls into 1 /gate " - f"roundtrip; got {len(gate_calls)}" + assert len(gate_calls) == 3, ( + f"each chain step must be admitted by its own /gate " + f"call; got {len(gate_calls)}" ) finally: try: @@ -1358,73 +1231,94 @@ def test_chain_mode_collapses_three_checks_to_one_gate_call(self): pass @respx.mock - def test_chain_mode_emits_fresh_uuid7_execution_id_per_call(self): - """BUG #4 wire at the runtime level: every /gate payload must - carry a fresh execution_id == uuid7 (NOT workflow_id). - - Disables the chain-mode cache so both ``check_workflow_budget`` - calls actually POST a /gate body — the cache would otherwise - collapse the second call into a hit and we'd never see the - second payload. - - Covers: - runtime.py:1247-1255 (execution_id = uuid7_str ) - runtime.py:1310-1323 (no-cache branch — direct transport.check). - """ + def test_each_chain_step_gets_its_own_server_minted_reservation(self): + # The money-relevant half of SDK-1. Pre-fix the cache served + # call #1's ``reservation_id`` to steps 2 and 3, so the + # backend settled ONE execution for THREE tool calls and the + # money for steps 2 and 3 was never billed. + respx.post(f"{BASE_URL}/api/v1/gate").mock( + side_effect=[ + Response( + 200, + json={ + "decision": "allow", + "decision_source": "gateway", + "reservation_id": reservation, + }, + ) + for reservation in _SDK1_RESERVATIONS + ] + ) + rt_inst = NullRunRuntime( + api_key="nr_live_abc123", + api_url=BASE_URL, + _test_mode=True, + polling=False, + ) + try: + with workflow("wf-sdk1"), chain(str(uuid.uuid4())): + got = [ + rt_inst.check_workflow_budget() for _ in range(3) + ] + assert got == _SDK1_RESERVATIONS, ( + "every chain step must carry the reservation the server " + "minted for THAT step; sharing one reservation id means " + "the backend settles the chain once and bills one step" + ) + finally: + try: + rt_inst.shutdown() + except Exception: + pass + + @respx.mock + def test_every_chain_step_carries_the_tool_list(self): + # The cache key had no tool set in it, so a cached step never + # ran the tool-block check. Each step must put its tools on the + # wire so the backend can block on them. import json as _json - import os + from nullrun.context import set_call_context - os.environ["NULLRUN_GATE_CACHE_DISABLE"] = "1" + respx.post(f"{BASE_URL}/api/v1/gate").mock( + return_value=Response( + 200, + json={"decision": "allow", "decision_source": "gateway"}, + ) + ) + rt_inst = NullRunRuntime( + api_key="nr_live_abc123", + api_url=BASE_URL, + _test_mode=True, + polling=False, + ) try: - respx.post(f"{BASE_URL}/api/v1/gate").mock( - return_value=Response( - 200, - json={"decision": "allow", "decision_source": "gateway"}, + with workflow("wf-sdk1"), chain(str(uuid.uuid4())): + set_call_context(tools=["dangerous_tool"]) + rt_inst.check_workflow_budget() + rt_inst.check_workflow_budget() + gate_calls = [ + c for c in respx.calls if c.request.url.path.endswith("/gate") + ] + assert len(gate_calls) == 2 + for call in gate_calls: + body = _json.loads(call.request.content) + assert body.get("tools") == ["dangerous_tool"], ( + "every step must present its tools to the gate" ) - ) - rt_inst = NullRunRuntime( - api_key="nr_live_abc123", - api_url=BASE_URL, - _test_mode=True, - polling=False, - ) - try: - with workflow("wf-runtime-uuid7"), chain(str(uuid.uuid4())): - rt_inst.check_workflow_budget() - rt_inst.check_workflow_budget() - gate_calls = [ - c for c in respx.calls if c.request.url.path.endswith("/gate") - ] - assert len(gate_calls) == 2 - first = _json.loads(gate_calls[0].request.content)["execution_id"] - second = _json.loads(gate_calls[1].request.content)["execution_id"] - assert first != second - assert uuid.UUID(first).version == 7 - assert uuid.UUID(second).version == 7 - assert first != "wf-runtime-uuid7" - assert second != "wf-runtime-uuid7" - finally: - try: - rt_inst.shutdown() - except Exception: - pass finally: - os.environ.pop("NULLRUN_GATE_CACHE_DISABLE", None) + try: + rt_inst.shutdown() + except Exception: + pass @respx.mock - def test_chain_mode_disabled_via_env_bypasses_cache(self): - """NULLRUN_GATE_CACHE_DISABLE=1 → cache_enabled=False → every - call hits /gate (runtime.py:1275-1277 fallback, runtime.py:1324 - direct transport.check path). - - Covers: - runtime.py:1294-1295 (cache_enabled=False exit) - runtime.py:1310-1323 (no-cache branch). - """ + def test_the_dead_env_var_cannot_bring_the_cache_back(self): + # The old opt-out was "the cache is disabled". Now that the + # cache is gone, the variable must be inert: it may not change + # the number of /gate calls. import os - os.environ["NULLRUN_GATE_CACHE_DISABLE"] = "1" try: respx.post(f"{BASE_URL}/api/v1/gate").mock( @@ -1440,16 +1334,13 @@ def test_chain_mode_disabled_via_env_bypasses_cache(self): polling=False, ) try: - with workflow("wf-no-cache"), chain(str(uuid.uuid4())): + with workflow("wf-sdk1"), chain(str(uuid.uuid4())): rt_inst.check_workflow_budget() rt_inst.check_workflow_budget() gate_calls = [ c for c in respx.calls if c.request.url.path.endswith("/gate") ] - assert len(gate_calls) == 2, ( - f"with NULLRUN_GATE_CACHE_DISABLE=1 every call must " - f"hit /gate; got {len(gate_calls)}" - ) + assert len(gate_calls) == 2 finally: try: rt_inst.shutdown() @@ -1459,61 +1350,35 @@ def test_chain_mode_disabled_via_env_bypasses_cache(self): os.environ.pop("NULLRUN_GATE_CACHE_DISABLE", None) @respx.mock - def test_runtime_chain_end_invalidates_cache(self, make_runtime): - # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): - # ``Runtime.chain_end()`` must invalidate the gate cache - # for the closing chain AFTER the wire call succeeds. - # The chain is closed on the server; the in-process cache - # for that chain is no longer reachable, so dropping it - # frees memory and prevents any hypothetical re-use of - # the same (workflow_id, chain_id) UUID from hitting the - # cached entry. Pin that the invalidation happens after - # the wire call (not before) — invalidating before would - # free the cache even if the server-side chain-close - # failed, leaking cache state into a "chain is closed on - # the client side but still open on the server" half-state. - import time as _time - import uuid as _uuid - - from nullrun import runtime as _rt_mod - - rt = make_runtime() - - # Mock /gate (chain_end uses the same endpoint) to return - # success. - respx.post(f"{BASE_URL}/api/v1/gate").mock( + def test_chain_end_still_closes_the_chain_on_the_server(self): + # The cache invalidation that used to hang off ``chain_end`` + # is gone with the cache. The server-side close must still + # happen — that is the part that was never the SDK's to skip. + rt_inst = NullRunRuntime( + api_key="nr_live_abc123", + api_url=BASE_URL, + _test_mode=True, + polling=False, + ) + gate = respx.post(f"{BASE_URL}/api/v1/gate").mock( return_value=Response( 200, json={"decision": "allow", "decision_source": "gateway"}, ) ) - - workflow_id = rt.workflow_id - chain_id = str(_uuid.uuid4()) - cache_key = (str(workflow_id), chain_id, "claude-sonnet-4-6", 1) - _rt_mod._GATE_CACHE[cache_key] = (_time.monotonic(), {"decision": "allow"}) - - # Set the chain_id contextvar so chain_end's post-wire - # invalidation helper can resolve it. Matches what - # ``with chain(...)`` would set. - from nullrun.context import set_chain_id - - token = set_chain_id(chain_id) try: - assert cache_key in _rt_mod._GATE_CACHE - rt.chain_end(chain_id) - - # The chain is closed on the server and the wire call - # returned 200, so the cache must be dropped for - # this chain. - assert cache_key not in _rt_mod._GATE_CACHE, ( - "Runtime.chain_end must invalidate the matching " - "chain's cache entry after the wire call succeeds " - "(DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET)" - ) + # `_test_mode` skips the auth handshake, so the org binding + # that ``chain_end`` forwards has to be set by hand. + rt_inst.organization_id = "org-1" + chain_id = str(uuid.uuid4()) + result = rt_inst.chain_end(chain_id) + assert isinstance(result, dict) + assert gate.call_count == 1 finally: - _rt_mod._GATE_CACHE.pop(cache_key, None) - _chain_id_var.reset(token) + try: + rt_inst.shutdown() + except Exception: + pass # ─── server-minted execution_id ────────────────────────────────── @@ -1724,21 +1589,44 @@ def test_clears_on_none_field(self): assert result is None assert get_server_minted_execution_id() is None - def test_drops_malformed_uuid_with_warning(self, caplog): + def test_refuses_a_malformed_uuid(self, caplog): + """SDK-10 — STRENGTHENED, not weakened. + + This test used to pin the old behaviour: a ``reservation_id`` + that is not a UUID was DROPPED with a warning, and the capture + returned ``None``. Dropping it read "the server reserved + nothing" when the truth was "the server reserved something I + cannot bind" — so the call proceeded with no reservation, the + ``/track`` event was dropped, and the spend was never booked. + + The backend's ``/gate`` Allow arm always carries the Step-9 + reserve's server-minted ``uuidv7`` (and the ``/execute`` arm + reuses the SAME id, ADR-073), so a real answer never lands + here. A present-but-unusable id is an answer the SDK cannot + act on, and it is REFUSED. + + The full money path is in ``test_sdk10_malformed_reservation.py``. + """ import logging - # Pre-seed so we can verify clear happens even on - # malformed input. + from nullrun.breaker.exceptions import NullRunMalformedGateResponseError + + # Pre-seed so we can verify the refusal does not leave a + # previous scope's id in place. set_server_minted_execution_id(SERVER_MINTED_V1) - with caplog.at_level(logging.WARNING, logger="nullrun.runtime"): - result = _capture_server_minted_execution_id( - {"reservation_id": "not-a-uuid"} - ) - assert result is None - assert get_server_minted_execution_id() is None + with caplog.at_level(logging.ERROR, logger="nullrun.runtime"): + with pytest.raises(NullRunMalformedGateResponseError): + _capture_server_minted_execution_id( + {"reservation_id": "not-a-uuid"} + ) + assert get_server_minted_execution_id() == SERVER_MINTED_V1, ( + "a refusal must not silently clear the scope's id — the " + "caller is about to abort, and the id belongs to whoever " + "set it" + ) assert any( - "is not a valid UUID" in record.message + "cannot be bound" in record.message for record in caplog.records ) @@ -1749,14 +1637,20 @@ def test_tolerates_non_dict_response(self): assert result is None assert get_server_minted_execution_id() is None - def test_drops_non_string_field(self): - # Backend is the source of truth and only emits strings - # but a buggy proxy could echo an int. Defensive parse. - result = _capture_server_minted_execution_id( - {"reservation_id": 123456} # type: ignore[dict-item] - ) - assert result is None - assert get_server_minted_execution_id() is None + def test_refuses_a_non_string_field(self): + """SDK-10 — STRENGTHENED, not weakened. + + Was: "a buggy proxy could echo an int → drop it". The drop is + the hole — a dropped reservation id means the call runs with no + reservation and its cost event is dropped, so the spend is + never billed. A present-but-unusable id is REFUSED. + """ + from nullrun.breaker.exceptions import NullRunMalformedGateResponseError + + with pytest.raises(NullRunMalformedGateResponseError): + _capture_server_minted_execution_id( + {"reservation_id": 123456} # type: ignore[dict-item] + ) # ───────────────────────────────────────────────────────────────── @@ -2168,20 +2062,23 @@ def test_v3_track_disable_env_forces_legacy(self, make_runtime, monkeypatch): assert batch_route.call_count == 1 @respx.mock - def test_track_single_422_invalidates_chain_cache(self, make_runtime): + def test_track_single_422_refused_consume_propagates(self, make_runtime): # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): when # ``/track`` returns HTTP 422 (CONSUME_OVERBUDGET) the chain - # has hit its budget ceiling. Any cached ``allow`` for the - # same (workflow_id, chain_id) must NOT be served for the - # next 0–5 s — otherwise a chain firing faster than the - # cache TTL over-reserves against a budget the server has - # just rejected. _route_track's except handler invalidates - # the cache before logging the failure. - import time as _time + # has hit its budget ceiling. + # + # Pre-fix the SDK's answer to a refused consume was to drop + # an in-process gate cache, so the next step of the chain + # could not be served a stale "allow". That cache is gone + # (SDK-1) and the refusal is handled where it belongs: the + # server refuses the next step's own /gate call, and this + # refusal reaches the agent as a typed error. + # + # The property that matters is unchanged and is the one this + # test pins: a refused consume is NOT laundered into an + # allowed one (DEF-TC6-006). import uuid as _uuid - from nullrun import runtime as _rt_mod - rt = make_runtime() # Mock /track to return 422 CONSUME_OVERBUDGET. @@ -2201,22 +2098,11 @@ def test_track_single_422_invalidates_chain_cache(self, make_runtime): ) ) - # Pre-populate the cache so the invalidation has something - # to drop. The chain_id matches the contextvar we'll set - # below; the workflow_id matches the runtime's bound - # workflow. Use uuid4() so the chain_id validator accepts it - # (CLAUDE.md §6). - workflow_id = rt.workflow_id - chain_id = str(_uuid.uuid4()) - cache_key = (str(workflow_id), chain_id, "claude-sonnet-4-6", 1) - _rt_mod._GATE_CACHE[cache_key] = (_time.monotonic(), {"decision": "allow"}) - assert cache_key in _rt_mod._GATE_CACHE - # Pin the chain_id in the contextvar (matches what - # ``with chain(...)`` would set) so the invalidation - # helper can resolve it without falling back to None. + # ``with chain(...)`` would set). from nullrun.context import set_chain_id + chain_id = str(_uuid.uuid4()) token = set_chain_id(chain_id) try: # Capture a server-minted id so _route_track reaches @@ -2225,14 +2111,6 @@ def test_track_single_422_invalidates_chain_cache(self, make_runtime): {"reservation_id": SERVER_MINTED_V1} ) - # DEF-TC6-006 (2026-10-02): the 422 CONSUME_OVERBUDGET - # now PROPAGATES to the caller as a typed - # NullRunConsumeOverbudgetError instead of being logged - # and swallowed. The invalidation this test was written - # to pin is unchanged -- it runs above the re-raise in - # the same handler -- so the assertion is strengthened, - # not relaxed: the blast radius is still closed, and the - # agent additionally learns its consume was refused. from nullrun.breaker.exceptions import NullRunConsumeOverbudgetError with pytest.raises(NullRunConsumeOverbudgetError): @@ -2241,39 +2119,20 @@ def test_track_single_422_invalidates_chain_cache(self, make_runtime): output_tokens=40, model="claude-sonnet-4-6", ) - - assert cache_key not in _rt_mod._GATE_CACHE, ( - "422 from /track must invalidate the matching chain's " - "cache entry (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET)" - ) finally: - # Reset the cache + the chain contextvar so we don't - # leak state into the next test. - _rt_mod._GATE_CACHE.pop(cache_key, None) _chain_id_var.reset(token) @respx.mock - def test_track_single_402_invalidates_chain_cache(self, make_runtime): - # 2026-09-12 (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET): the - # 402 path must trigger the same cache invalidation as 422. - # Pre-fix code only handled 422 — a 402 with the same - # chain-bleed blast radius would have silently kept the - # cached allow alive for 5 s while the backend was refusing - # consume. We use BUDGET_HARD_BLOCKED (catalog-mapped → - # NullRunBudgetError with status_code=402) so the - # ``getattr(exc, "status_code", None)`` branch in - # ``_route_track``'s except handler fires the same way - # the 422 path does. Note: REDIS_UNAVAILABLE is absent - # from the v3 catalog (removed 2026-09-10) and would - # fall through to the generic NullRunBackendError, whose - # status_code lives in ``details`` not as a direct - # attribute — a different shape than the runtime - # branch's getattr reads. - import time as _time + def test_track_single_402_refused_consume_propagates(self, make_runtime): + # Same shape as the 422 test above: a 402 on the consume path + # must reach the agent too. We use BUDGET_HARD_BLOCKED + # (catalog-mapped → NullRunBudgetError with status_code=402). + # Note: REDIS_UNAVAILABLE is absent from the v3 catalog + # (removed 2026-09-10) and would fall through to the generic + # NullRunBackendError, whose status_code lives in ``details`` + # not as a direct attribute. import uuid as _uuid - from nullrun import runtime as _rt_mod - rt = make_runtime() # Mock /track to return 402 (BUDGET_HARD_BLOCKED). @@ -2288,13 +2147,9 @@ def test_track_single_402_invalidates_chain_cache(self, make_runtime): ) ) - workflow_id = rt.workflow_id - chain_id = str(_uuid.uuid4()) - cache_key = (str(workflow_id), chain_id, "claude-sonnet-4-6", 1) - _rt_mod._GATE_CACHE[cache_key] = (_time.monotonic(), {"decision": "allow"}) - from nullrun.context import set_chain_id + chain_id = str(_uuid.uuid4()) token = set_chain_id(chain_id) try: _capture_server_minted_execution_id( @@ -2303,8 +2158,7 @@ def test_track_single_402_invalidates_chain_cache(self, make_runtime): # DEF-TC6-006: same shape as the 422 test above -- # NullRunBudgetError is a NullRunBlockedException, which - # is a NullRunDecision, so the 402 propagates too. The - # cache invalidation still runs before the raise. + # is a NullRunDecision, so the 402 propagates too. from nullrun.breaker.exceptions import NullRunBudgetError with pytest.raises(NullRunBudgetError): @@ -2313,13 +2167,7 @@ def test_track_single_402_invalidates_chain_cache(self, make_runtime): output_tokens=10, model="claude-sonnet-4-6", ) - - assert cache_key not in _rt_mod._GATE_CACHE, ( - "402 from /track must invalidate the matching chain's " - "cache entry (DEF-CACHE-STALE-ALLOW-AFTER-OVERBUDGET)" - ) finally: - _rt_mod._GATE_CACHE.pop(cache_key, None) _chain_id_var.reset(token) @@ -2764,6 +2612,11 @@ def test_check_workflow_budget_soft_pass_branch_logs_overdraft_telemetry(): ("APPROVAL_DIGEST_MISMATCH", exc.NullRunApprovalDigestMismatchError), ("APPROVAL_TOOL_DIGEST_MISMATCH", exc.NullRunApprovalToolDigestMismatchError), ("APPROVAL_REPLAY_REJECTED", exc.NullRunApprovalReplayRejectedError), + # DEC-5 / DEC-7 (audit 2026-11-02): the approval binds a TOOL + # SET. The executing tool must be a member of the set the + # /gate request declared — a money approval used to bind only + # the AMOUNT, so an approval for tool A authorised tool B. + ("APPROVAL_TOOL_MISMATCH", exc.NullRunApprovalToolMismatchError), ], ) def test_v3_error_code_map_covers_all_post_approval_outcomes( @@ -2803,6 +2656,8 @@ def test_v3_error_code_map_covers_all_post_approval_outcomes( ("APPROVAL_DIGEST_MISMATCH", "NR-A013"), ("APPROVAL_TOOL_DIGEST_MISMATCH", "NR-A014"), ("APPROVAL_REPLAY_REJECTED", "NR-A015"), + # DEC-5 / DEC-7: the tool-binding refusal. + ("APPROVAL_TOOL_MISMATCH", "NR-A017"), ], ) def test_post_approval_exceptions_carry_stable_nr_codes(wire_code, expected_nr_code): @@ -2957,6 +2812,53 @@ def test_parse_v3_error_envelope_dispatches_approval_tool_digest_mismatch(): assert err.error_code == "NR-A014" +def test_parse_v3_error_envelope_dispatches_approval_tool_mismatch(): + """``APPROVAL_TOOL_MISMATCH`` → ``NullRunApprovalToolMismatchError``. + + DEC-5 / DEC-7 (audit 2026-11-02): the approval binds a TOOL SET, + and the executing tool must be a member. HTTP 409 — a conflict, not + a permission denial: the approval is valid, it just covers a + different tool than the one now running. + """ + response = httpx.Response( + 409, + json={ + "error_code": "APPROVAL_TOOL_MISMATCH", + "error_message": "The executing tool is not covered by the operator's approval", + "details": {"approval_id": "apr-uuid-5555"}, + }, + ) + err = _parse_v3_error_envelope(response, "execute") + assert isinstance(err, exc.NullRunApprovalToolMismatchError) + assert err.status_code == 409 + assert err.error_code == "NR-A017" + + +def test_approval_tool_mismatch_is_not_part_of_the_403_family(): + """``APPROVAL_TOOL_MISMATCH`` is 409, NOT 403. + + The other six grant-consume outcomes are 403 ("you are not allowed + to do this"). The tool mismatch is a CONFLICT: the operator's + approval is valid and unconsumed — it simply covers a different + tool. A cookbook recipe that branches on the status code (403 → + "permission problem", 409 → "state conflict") must be able to tell + them apart, so the 409 has to survive the dispatch verbatim. + """ + response = httpx.Response( + 409, + json={ + "error_code": "APPROVAL_TOOL_MISMATCH", + "error_message": "tool not covered", + "details": {"approval_id": "apr-uuid-6666"}, + }, + ) + err = _parse_v3_error_envelope(response, "execute") + assert err.status_code == 409, ( + "APPROVAL_TOOL_MISMATCH must dispatch with status_code=409 — the " + f"approval covers a different tool. Got {err.status_code}." + ) + + def test_approval_exceptions_inherit_from_nullrun_blocked_exception(): """All six typed exceptions inherit from ``NullRunBlockedException`` so legacy ``except NullRunBlockedException:`` clauses keep @@ -2972,6 +2874,7 @@ def test_approval_exceptions_inherit_from_nullrun_blocked_exception(): exc.NullRunApprovalDigestMismatchError, exc.NullRunApprovalToolDigestMismatchError, exc.NullRunApprovalReplayRejectedError, + exc.NullRunApprovalToolMismatchError, ): assert issubclass(cls, exc.NullRunBlockedException), ( f"{cls.__name__} must inherit from NullRunBlockedException " @@ -2990,6 +2893,7 @@ def test_approval_exceptions_inherit_from_nullrun_error(): exc.NullRunApprovalDigestMismatchError, exc.NullRunApprovalToolDigestMismatchError, exc.NullRunApprovalReplayRejectedError, + exc.NullRunApprovalToolMismatchError, ): assert issubclass(cls, exc.NullRunError), ( f"{cls.__name__} must inherit from NullRunError so " diff --git a/tests/test_value_level_redaction.py b/tests/test_value_level_redaction.py new file mode 100644 index 0000000..d45142e --- /dev/null +++ b/tests/test_value_level_redaction.py @@ -0,0 +1,91 @@ +"""Redaction is not only about the NAME an argument arrives under. + +``SENSITIVE_ARG_KEYS`` masks the arguments whose name someone listed. +It cannot catch a secret that arrives under a name nobody listed -- +``{"note": ""}`` prints the key in full, and the key-name +pass looks straight past it. + +This file pins the value-shaped half: the credential itself is +matched, whatever key it travels in. The shapes are deliberately +narrow -- a shape that cannot be told apart from ordinary data is a +guess, and a guess that scrubs real data is a bug of its own. +""" + +from __future__ import annotations + +import pytest + +from nullrun.decorators import _safe_kwargs, _safe_repr +from nullrun.observability.redaction import scrub_secret_values + +# A key in the shape this product mints. The random part is long +# enough to match the pattern and is not a real credential. +_OWN_KEY = "nr_live_0123456789abcdefghij" +_JWT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVP" +_AWS_KEY_ID = "AKIAIOSFODNN7EXAMPLE" +_OPENAI_KEY = "sk-abcdef0123456789abcdef0123456789" +_PEM = ( + "-----BEGIN RSA PRIVATE KEY-----\n" + "MIIEowIBAAKCAQEAx1234567890abcdef\n" + "-----END RSA PRIVATE KEY-----" +) + + +class TestValueShapes: + @pytest.mark.parametrize( + "secret", + [_OWN_KEY, _JWT, _AWS_KEY_ID, _OPENAI_KEY, _PEM], + ) + def test_a_credential_is_scrubbed_whatever_the_key(self, secret): + """The secret travels under a name nobody listed.""" + rendered = _safe_kwargs({"note": secret}) + assert secret not in str(rendered), ( + f"a credential survived the value pass under a benign key: {secret!r}" + ) + assert "***" in str(rendered) + + def test_a_bearer_credential_is_scrubbed(self): + rendered = _safe_kwargs({"headers": f"Bearer {_JWT}"}) + assert _JWT not in str(rendered) + + def test_ordinary_data_is_untouched(self): + """A scrubber that eats real data is its own defect.""" + rendered = _safe_kwargs({"query": "SELECT id FROM users WHERE name = 'x'"}) + assert "SELECT id FROM users" in str(rendered) + + @pytest.mark.parametrize( + "value", + [ + "task-abcdef0123456789abcdef0123456789", + "sk-short", + "AKIASHORT", + "eyJnotajwt", + "risk-assessment-report", + ], + ) + def test_a_near_miss_is_not_scrubbed(self, value): + """The patterns are anchored so an ordinary word that happens + to contain the shape survives.""" + assert scrub_secret_values(value) == value + + def test_the_key_name_pass_still_works(self): + """The value pass is added, not substituted.""" + assert _safe_kwargs({"password": "hunter2"})["password"] == "***" + + +class TestOrder: + def test_the_scrub_runs_before_the_truncation(self): + """A credential that straddles the cut must be caught whole. + + The credential starts INSIDE the truncated region and runs + past it. Truncate-then-scrub would cut the credential in half + and scrub neither half -- the first eighteen characters would + reach the audit log.""" + payload = "x" * 30 + _OWN_KEY + rendered = _safe_repr(payload) + assert "nr_live_" not in rendered + assert _OWN_KEY not in rendered + + def test_a_short_credential_is_scrubbed_too(self): + rendered = _safe_repr(_OWN_KEY) + assert _OWN_KEY not in rendered diff --git a/tests/test_wal_scope.py b/tests/test_wal_scope.py new file mode 100644 index 0000000..387bd58 --- /dev/null +++ b/tests/test_wal_scope.py @@ -0,0 +1,417 @@ +"""SDK-4 — the write-ahead log must not be shared across tenants. + +Before the fix the WAL lived at a FIXED path in the shared OS temp +directory — ``/nullrun.wal`` — for every user and every org on +the host. Two runtimes therefore resolved to the SAME file, and the +atomic whole-file replace in ``Transport._persist_to_wal`` replaced the +other runtime's undelivered events with its own buffer. Spend that had +already happened was then never booked. + +The default is now scoped per user AND per API KEY. + +Why the KEY and not the org id +------------------------------ +The path is resolved ONCE, when the transport is constructed, and never +changes afterwards. The org id is only known AFTER authentication; a +path that changed at authentication would orphan every file written +before it, and no test would catch it — the path is "right" at every +instant and the money is lost anyway. + +One API key belongs to exactly one organisation, so a key-scoped WAL is +an org-scoped WAL in money terms. Two organisations therefore never +share a WAL. + +Two workers of the SAME organisation using DIFFERENT keys never share +one either. That is the common multi-worker case, and it is covered by +the default. + +The audit's literal check — "two Transports with different org ids must +not share a WAL path" — is therefore satisfied by scoping on the key: +in a real deployment an org is reachable only through its own key, so +two orgs always carry two keys. The same key with two different org ids +is ONE money pool and MUST share one WAL: splitting it would strand +half the spend in a file nobody replays. + +Every assertion here runs against REAL files on disk. A mock would pass +while the destructive whole-file replace is still in place, which is the +exact thing this finding is about. +""" + +from __future__ import annotations + +import json +import os +import tempfile + +import pytest + +from nullrun.transport import Transport +from nullrun.wal_paths import ( + LEGACY_WAL_BASENAME, + default_wal_path, + legacy_wal_path, + scoped_wal_paths, + user_token, +) + +FAKE_KEY = "nr_live_FAKEKEYFORTESTSONLY_0123456789" +OTHER_FAKE_KEY = "nr_live_OTHERKEYFORTESTSONLY_012345678" + + +@pytest.fixture +def tempdir_as_tmp(monkeypatch, tmp_path): + """Point the OS temp directory at this test's own directory. + + Patching the FUNCTION (not the module attribute of a single caller) + means the OLD unscoped default and the NEW scoped default both + resolve under ``tmp_path``. Without this the RED run of this file + would write into the real shared temp directory. + """ + monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) + monkeypatch.delenv("NULLRUN_WAL_PATH", raising=False) + return tmp_path + + +def _transport( + org_id: str | None = None, + api_key: str | None = FAKE_KEY, +) -> Transport: + return Transport( + api_url="https://api.test.nullrun.io", + api_key=api_key, + organization_id=org_id, + ) + + +def _event(n: int) -> dict: + return {"event_id": f"evt-{n}", "type": "llm_call", "tokens": 100} + + +class TestThePathIsScoped: + def test_two_transports_with_different_keys_do_not_share_a_path( + self, tempdir_as_tmp + ): + """Two orgs are reached through two keys, and two keys must not + share a file.""" + a = _transport("org_alpha") + b = _transport("org_beta", api_key=OTHER_FAKE_KEY) + try: + assert a._wal_path() != b._wal_path() + finally: + a._client.close() + b._client.close() + + def test_the_same_key_resolves_to_the_same_path(self, tempdir_as_tmp): + """A restart must find the WAL the previous run left behind.""" + a = _transport("org_alpha") + b = _transport("org_alpha") + try: + assert a._wal_path() == b._wal_path() + finally: + a._client.close() + b._client.close() + + def test_one_key_with_two_org_ids_shares_one_path(self, tempdir_as_tmp): + """One key is ONE money pool, so it gets ONE WAL. + + The audit asked for the split on the org id. Splitting here would + strand half the spend in a file nothing replays — the same class + of loss the finding is about, in the other direction. + """ + a = _transport("org_alpha") + b = _transport("org_beta") + try: + assert a._wal_path() == b._wal_path() + finally: + a._client.close() + b._client.close() + + def test_the_path_is_scoped_per_user(self, tempdir_as_tmp, monkeypatch): + """Two users on one host must not share a WAL either.""" + a = _transport("org_alpha") + try: + first = a._wal_path() + finally: + a._client.close() + + import nullrun.wal_paths as wal_paths + + monkeypatch.setattr(wal_paths, "user_token", lambda: "u999999") + second = default_wal_path(org_id="org_alpha", api_key=FAKE_KEY) + assert first != second + + def test_the_api_key_is_not_in_the_path_in_cleartext(self, tempdir_as_tmp): + a = _transport("org_alpha") + try: + path = a._wal_path() + finally: + a._client.close() + assert FAKE_KEY not in path + assert "org_alpha" not in path + + def test_the_path_carries_the_user_token(self, tempdir_as_tmp): + a = _transport("org_alpha") + try: + path = a._wal_path() + finally: + a._client.close() + assert user_token() in path + + def test_the_legacy_unscoped_path_is_not_the_default(self, tempdir_as_tmp): + a = _transport("org_alpha") + try: + path = a._wal_path() + finally: + a._client.close() + assert path != legacy_wal_path() + assert LEGACY_WAL_BASENAME not in os.path.basename(path) + + +class TestThePathNeverChanges: + """SDK-4 (security audit 2026-11-06). The path must be resolved + ONCE and never change during a transport's lifetime. + + A path that changed at authentication would orphan every file + written before it. The early file would hold real undelivered + spend and nothing would ever read it again — and no test would + catch it, because at every instant the path is "the right one". + """ + + def test_the_org_becoming_known_does_not_move_the_path(self, tempdir_as_tmp): + a = _transport(None) + try: + before = a._wal_path() + # The org id arrives after authentication. + a.organization_id = "org_alpha" + assert a._wal_path() == before + finally: + a._client.close() + + def test_two_transports_of_one_key_agree_even_after_the_org_is_known( + self, tempdir_as_tmp + ): + """The file written before auth and the file written after it + must be the SAME file, or the early one is orphaned.""" + early = _transport(None) + later = _transport(None) + try: + early.organization_id = "org_alpha" + later.organization_id = "org_alpha" + assert early._wal_path() == later._wal_path() + finally: + early._client.close() + later._client.close() + + def test_a_keyless_transport_does_not_move_its_path_when_the_org_arrives( + self, tempdir_as_tmp + ): + """The once-only resolution, pinned where it actually bites. + + With a key the scope IS the key, so the org cannot move the path + and the property is structural — no test can catch a regression + there, because nothing can break it. + + Without a key the scope falls back to the org id. A path + re-derived on every call WOULD move here, and the file written + before authentication would be orphaned with real undelivered + spend inside it — and nothing would ever read it again. + """ + a = _transport(None, api_key=None) + try: + before = a._wal_path() + a.organization_id = "org_alpha" + assert a._wal_path() == before + finally: + a._client.close() + + +class TestTheEnvironmentOverrideStillWins: + def test_nullrun_wal_path_beats_the_scoped_default( + self, tempdir_as_tmp, monkeypatch + ): + chosen = str(tempdir_as_tmp / "chosen.wal") + monkeypatch.setenv("NULLRUN_WAL_PATH", chosen) + a = _transport("org_alpha") + try: + assert a._wal_path() == chosen + finally: + a._client.close() + + def test_the_override_ignores_the_key_and_the_org( + self, tempdir_as_tmp, monkeypatch + ): + """The operator's explicit path wins over everything. + + It is therefore also the operator's responsibility to make it + unique per runtime — the SDK cannot detect two runtimes pointing + at one path. + """ + chosen = str(tempdir_as_tmp / "chosen.wal") + monkeypatch.setenv("NULLRUN_WAL_PATH", chosen) + a = _transport("org_alpha") + b = _transport("org_beta", api_key=OTHER_FAKE_KEY) + try: + assert a._wal_path() == b._wal_path() == chosen + finally: + a._client.close() + b._client.close() + + +class TestOneRuntimeDoesNotDestroyAnother: + """The audit's integration check, on real files. + + A write from runtime B must not remove runtime A's buffered events. + """ + + def test_a_write_from_runtime_b_leaves_runtime_a_events_alone( + self, tempdir_as_tmp + ): + a = _transport("org_alpha") + b = _transport("org_beta", api_key=OTHER_FAKE_KEY) + try: + a.track(_event(1)) + b.track(_event(2)) + assert a._persist_to_wal() + assert b._persist_to_wal() + finally: + a._client.close() + b._client.close() + + a_rows = _read_rows(a._wal_path()) + b_rows = _read_rows(b._wal_path()) + + assert [r["event_id"] for r in a_rows] == ["evt-1"], ( + "runtime B destroyed runtime A's buffered events" + ) + assert [r["event_id"] for r in b_rows] == ["evt-2"] + + def test_the_two_runtimes_wrote_to_two_different_files( + self, tempdir_as_tmp + ): + a = _transport("org_alpha") + b = _transport("org_beta", api_key=OTHER_FAKE_KEY) + try: + a.track(_event(1)) + b.track(_event(2)) + a._persist_to_wal() + b._persist_to_wal() + a_file = a._wal_path() + b_file = b._wal_path() + finally: + a._client.close() + b._client.close() + + assert a_file != b_file + assert os.path.exists(a_file) + assert os.path.exists(b_file) + + +class TestTheLegacyWalIsReportedButNotStolen: + def test_a_legacy_wal_is_warned_about_and_left_alone( + self, tempdir_as_tmp, caplog + ): + """The old unscoped file may hold ANOTHER org's events. + + Reading it into this org's WAL would attribute a stranger's spend + to this tenant. So the SDK reports it and leaves it where it is. + """ + legacy = legacy_wal_path() + with open(legacy, "w") as f: + f.write(json.dumps(_event(99)) + "\n") + + a = _transport("org_alpha") + try: + with caplog.at_level("WARNING", logger="nullrun.transport"): + a.start() + finally: + a._client.close() + a.stop(flush=False) + + assert any(legacy in r.getMessage() for r in caplog.records), ( + "the legacy WAL was not reported" + ) + # Untouched: still exactly the one row it had. + assert [r["event_id"] for r in _read_rows(legacy)] == ["evt-99"] + # And it was not replayed into this org's WAL. + assert _read_rows(a._wal_path()) == [] + + +class TestTheListingHelper: + def test_it_lists_only_this_users_wal_files(self, tempdir_as_tmp): + a = _transport("org_alpha") + b = _transport("org_beta", api_key=OTHER_FAKE_KEY) + try: + a.track(_event(1)) + b.track(_event(2)) + a._persist_to_wal() + b._persist_to_wal() + finally: + a._client.close() + b._client.close() + + listed = scoped_wal_paths() + assert a._wal_path() in listed + assert b._wal_path() in listed + assert legacy_wal_path() not in listed + + +class TestTheCli: + """The operator's tool must not point at the wrong file. + + Before SDK-4 the CLI's default was the ONE fixed path, so every + command acted on every user's events. After it, the user may own one + scoped WAL per API key — so ``status`` lists them and ``replay`` + refuses to guess. A guess could send one tenant's events into + another tenant's ledger. + """ + + def test_status_without_a_path_lists_every_wal_of_this_user( + self, tempdir_as_tmp, capsys + ): + from nullrun.wal_admin import main as wal_main + + a = _transport("org_alpha") + b = _transport("org_beta", api_key=OTHER_FAKE_KEY) + try: + a.track(_event(1)) + b.track(_event(2)) + a._persist_to_wal() + b._persist_to_wal() + a_path = a._wal_path() + b_path = b._wal_path() + finally: + a._client.close() + b._client.close() + + assert wal_main(["status"]) == 0 + + out = capsys.readouterr().out + assert a_path in out + assert b_path in out + + def test_replay_without_a_path_refuses_to_guess(self, tempdir_as_tmp, capsys): + from nullrun.wal_admin import main as wal_main + + a = _transport("org_alpha") + b = _transport("org_beta", api_key=OTHER_FAKE_KEY) + try: + a.track(_event(1)) + b.track(_event(2)) + a._persist_to_wal() + b._persist_to_wal() + finally: + a._client.close() + b._client.close() + + assert wal_main(["replay"]) == 1 + + err = capsys.readouterr().err + assert "--wal" in err + + +def _read_rows(path: str) -> list[dict]: + try: + with open(path) as f: + return [json.loads(line) for line in f if line.strip()] + except FileNotFoundError: + return [] diff --git a/tests/test_ws_approval_forgery.py b/tests/test_ws_approval_forgery.py new file mode 100644 index 0000000..53646ab --- /dev/null +++ b/tests/test_ws_approval_forgery.py @@ -0,0 +1,573 @@ +"""tests/test_ws_approval_forgery.py — a forged approval frame must not +release a parked gate. + +SDK-5 (reliability audit 2026-11-02). + +The defect +---------- +The HMAC on the control-plane WebSocket covers the bytes inside +``signed_payload`` — NOT the flattened outer wire body. The outer body +is therefore attacker-writable: anyone who can inject bytes into the +socket may rewrite ``type``, ``outcome``, ``execution_id`` … while +keeping a captured ``(signed_payload, signature)`` pair intact. The +signature check still PASSES, because it verifies bytes the attacker +never touched. + +The ``approval_resolved`` branch read its fields from the OUTER body +and handed that dict to ``on_approval_resolved``. In the runtime that +callback pops the pending approval and sets its ``threading.Event`` — +releasing a gate that was parked waiting for a human. + +The fix +------- +1. ONE source of truth: the branch type and every field are read from + the verified ``signed_payload``; the outer body is a fallback only + for frames that could not be authenticated at all. +2. The privileged branch requires an AUTHENTICATED frame. A frame the + SDK cannot authenticate never releases a gate — the waiter stays + parked and fails closed on the approval timeout. + +WHAT THIS FILE DRIVES +--------------------- +The real ``WebSocketConnection._handle_message`` with the real +envelope shape the backend produces (``SignedWsMessage``: the payload +is flattened into the outer body AND published hex-encoded as +``signed_payload``). A source-pin could not prove the forgery is +neutralised — the whole point is which dict the dispatcher reads. +""" + +from __future__ import annotations + +import json +import threading +import time +from typing import Any + +import pytest + +from nullrun.observability import metrics +from nullrun.runtime import ( + NullRunApprovalDeniedError, + NullRunApprovalExpiredError, + NullRunRuntime, +) +from nullrun.transport import generate_hmac_signature +from nullrun.transport_websocket import WebSocketConnection + +API_KEY = "nr_live_test_key" +SECRET = "s3cret" +# The backend mints the reservation id as a UUID (RFC-4122 v7). +RESERVATION_ID = "018f3c5a-7b2d-7c4e-9f10-2b3c4d5e6f70" + +# The genuine, server-signed payload: the operator DENIED this call. +SIGNED_DENIAL: dict[str, Any] = { + "type": "approval_resolved", + "approval_id": "ap-1", + "workflow_id": "wf-1", + "execution_id": "ex-1", + "outcome": "denied", + "note": "not this one", + "resolved_at": 1730000000, + "message_id": "m-1", +} + +# The genuine, server-signed payload: the operator APPROVED this call. +SIGNED_APPROVAL: dict[str, Any] = { + "type": "approval_resolved", + "approval_id": "ap-1", + "workflow_id": "wf-1", + "execution_id": "ex-1", + "outcome": "approved", + "note": "ok", + "resolved_at": 1730000000, + "message_id": "m-1", +} + + +def _signed_envelope(message: dict, api_key: str, secret_key: str) -> dict: + """The server's ``SignedWsMessage`` shape. + + The payload is flattened into the outer body AND published + hex-encoded as ``signed_payload``; the HMAC covers the INNER + payload bytes only. That asymmetry is the whole attack surface. + """ + timestamp = int(time.time()) + payload_json = json.dumps(message, separators=(",", ":"), sort_keys=True) + signature = generate_hmac_signature( + api_key, secret_key, timestamp, payload_json.encode("utf-8") + ) + envelope = dict(message) + envelope["signature"] = signature + envelope["timestamp"] = timestamp + envelope["api_key"] = api_key + envelope["signed_payload"] = payload_json.encode("utf-8").hex() + return envelope + + +class _StubWS: + """Captures what the SDK writes back (the ACK).""" + + def __init__(self) -> None: + self.sent: list[bytes] = [] + self.closed = False + + async def send(self, data) -> None: + if isinstance(data, str): + self.sent.append(data.encode("utf-8")) + else: + self.sent.append(data) + + async def close(self) -> None: + self.closed = True + + +def _conn( + *, + api_key: str | None = API_KEY, + secret_key: str | None = SECRET, + on_approval_resolved=None, + on_state_change=None, +) -> tuple[WebSocketConnection, _StubWS]: + conn = WebSocketConnection( + url="wss://example.invalid/ws/control/org-1", + headers={}, + api_key=api_key, + secret_key=secret_key, + on_state_change=on_state_change, + on_approval_resolved=on_approval_resolved, + ) + stub = _StubWS() + conn._conn = stub + conn._running = True + return conn, stub + + +def _park_a_gate( + rt: NullRunRuntime, approval_id: str = "ap-1" +) -> tuple[threading.Event, dict]: + """Register a pending approval the way ``_wait_for_approval_resolution`` + does. + + The runtime POPS the entry from ``_approval_pending`` and stashes the + outcome on the entry dict itself (the waiter holds the reference), so + both the event and the entry are handed back. + """ + event = threading.Event() + entry = { + "approval_id": approval_id, + "workflow_id": "wf-1", + "execution_id": "ex-1", + "event": event, + "timeout_seconds": 300, + } + with rt._approval_lock: + rt._approval_pending[approval_id] = entry + return event, entry + + +class TestTheOuterBodyIsNotATrustedSource: + """The signature covers ``signed_payload``, not the outer body.""" + + @pytest.mark.asyncio + async def test_a_spliced_outer_body_cannot_flip_a_denial_into_an_approval(self): + """The attacker rewrites the OUTER body only. + + The signature verifies (it covers bytes the attacker did not + touch), so the frame is accepted — and the SDK must dispatch + the SIGNED outcome, not the spliced one. + """ + received: list[dict] = [] + conn, stub = _conn(on_approval_resolved=received.append) + + envelope = _signed_envelope(SIGNED_DENIAL, API_KEY, SECRET) + # The splice: the outer body now says "approved". + envelope["outcome"] = "approved" + envelope["note"] = "approved by the operator" + envelope["message_id"] = "m-attacker" + + await conn._handle_message(json.dumps(envelope)) + + assert len(received) == 1, "the signed frame must still be dispatched" + assert received[0]["outcome"] == "denied", ( + "the OUTER body decided the outcome — the signed payload " + f"said 'denied', the SDK dispatched {received[0]['outcome']!r}" + ) + assert received[0]["note"] == "not this one", ( + "the note came from the unsigned outer body" + ) + # The acknowledgement correlates on the SIGNED message id. + acks = [json.loads(s) for s in stub.sent] + assert acks, "no acknowledgement was sent for a genuine signed frame" + assert all(a.get("message_id") == "m-1" for a in acks), ( + "the acknowledgement correlated on the unsigned outer body's " + f"message_id — got {acks}" + ) + + @pytest.mark.asyncio + async def test_the_branch_is_chosen_from_the_verified_payload(self): + """ANY signed frame must not be relabelled as an approval. + + The attacker keeps a captured ``(signed_payload, signature)`` + pair from a harmless frame and rewrites the outer ``type``. + The branch must come from the signed payload. + """ + received: list[dict] = [] + conn, _ = _conn(on_approval_resolved=received.append) + + envelope = _signed_envelope( + {"type": "subscribed", "organization_id": "org-1"}, API_KEY, SECRET + ) + # The relabel. + envelope["type"] = "approval_resolved" + envelope["approval_id"] = "ap-2" + envelope["execution_id"] = "ex-2" + envelope["outcome"] = "approved" + + await conn._handle_message(json.dumps(envelope)) + + assert received == [], ( + "a signed 'subscribed' frame was dispatched as an approval — " + "the branch was chosen from the unsigned outer body" + ) + + @pytest.mark.asyncio + async def test_the_execution_id_comes_from_the_signed_payload(self): + """The execution id decides WHICH reservation is released.""" + received: list[dict] = [] + conn, _ = _conn(on_approval_resolved=received.append) + + envelope = _signed_envelope(SIGNED_APPROVAL, API_KEY, SECRET) + envelope["execution_id"] = "ex-attacker" + + await conn._handle_message(json.dumps(envelope)) + + assert len(received) == 1 + assert received[0]["execution_id"] == "ex-1", ( + "the execution id came from the unsigned outer body — it " + "decides which gate reservation is released" + ) + + +class TestAnUnauthenticatedFrameNeverReleasesAGate: + """A frame the SDK cannot authenticate must not release a gate.""" + + @pytest.mark.asyncio + async def test_an_unsigned_approval_frame_is_refused(self): + received: list[dict] = [] + conn, stub = _conn(on_approval_resolved=received.append) + + before = metrics.transport.ws_unverified_approval_total + await conn._handle_message( + json.dumps( + { + "type": "approval_resolved", + "approval_id": "ap-1", + "workflow_id": "wf-1", + "execution_id": "ex-1", + "outcome": "approved", + } + ) + ) + + assert received == [], ( + "an UNSIGNED approval frame released a gate — the SDK cannot " + "tell it from a genuine one, so it must refuse it" + ) + assert metrics.transport.ws_unverified_approval_total == before + 1, ( + "the refusal must be counted — an operator has to be able to " + "see it" + ) + assert stub.sent == [], "a refused frame must not be ACKed" + + @pytest.mark.asyncio + async def test_a_signed_but_tampered_approval_frame_is_refused(self): + """A frame whose signature does not verify must not release a gate.""" + received: list[dict] = [] + conn, _ = _conn(on_approval_resolved=received.append) + + envelope = _signed_envelope(SIGNED_APPROVAL, API_KEY, SECRET) + envelope["signature"] = "de" * 32 # 64 hex chars, wrong + + await conn._handle_message(json.dumps(envelope)) + + assert received == [], "a frame with a bad signature released a gate" + + @pytest.mark.asyncio + async def test_a_signed_frame_whose_payload_does_not_parse_is_dropped(self): + """The signature verifies, but the signed bytes are not JSON. + + The outer body is NOT what was signed — the signature covers the + bytes inside ``signed_payload`` — so the outer body is not a + fallback source. The frame is dropped, and the drop is counted. + """ + received: list[dict] = [] + conn, _ = _conn(on_approval_resolved=received.append) + + payload_bytes = b"this is not json" + timestamp = int(time.time()) + envelope = { + "type": "approval_resolved", + "approval_id": "ap-1", + "workflow_id": "wf-1", + "execution_id": "ex-1", + "outcome": "approved", + "timestamp": timestamp, + "api_key": API_KEY, + "signature": generate_hmac_signature( + API_KEY, SECRET, timestamp, payload_bytes + ), + "signed_payload": payload_bytes.hex(), + } + + before = metrics.transport.hmac_verify_failures_total + await conn._handle_message(json.dumps(envelope)) + + assert received == [], ( + "a frame whose signed payload does not parse was dispatched from " + "the outer body — the outer body is not what was signed" + ) + assert metrics.transport.hmac_verify_failures_total == before + 1, ( + "the drop must be counted — an operator has to be able to see it" + ) + + +class TestTheParkedGateItself: + """The end the audit names: a parked gate must not be released BY A FORGERY.""" + + @pytest.mark.asyncio + async def test_a_spliced_outer_body_cannot_release_the_gate_as_approved(self): + """The signed payload says DENIED. The gate must wake up on THAT. + + A denial is a legitimate wake-up: the waiter raises + ``NullRunApprovalDeniedError`` and the agent stops. What must + never happen is the gate waking up on the spliced ``approved``. + """ + rt = NullRunRuntime(api_key="test-key", _test_mode=True) + try: + event, entry = _park_a_gate(rt) + conn, _ = _conn(on_approval_resolved=rt._handle_approval_resolved) + + envelope = _signed_envelope(SIGNED_DENIAL, API_KEY, SECRET) + envelope["outcome"] = "approved" # the splice + + await conn._handle_message(json.dumps(envelope)) + + assert entry["outcome"] == "denied", ( + "the parked gate was released with the SPLICED outcome " + f"({entry['outcome']!r}) — the operator denied this call" + ) + assert entry["note"] == "not this one" + assert event.is_set(), ( + "a signed denial must still wake the waiter — it is the " + "fail-CLOSED wake-up" + ) + finally: + rt.shutdown() + + @pytest.mark.asyncio + async def test_an_unsigned_approval_does_not_release_the_parked_gate(self): + """An unauthenticated frame never wakes the waiter at all. + + The gate stays parked; the waiter then fails closed on its own + approval timeout (``NullRunApprovalExpiredError``). + """ + rt = NullRunRuntime(api_key="test-key", _test_mode=True) + try: + event, _entry = _park_a_gate(rt) + conn, _ = _conn(on_approval_resolved=rt._handle_approval_resolved) + + await conn._handle_message( + json.dumps( + { + "type": "approval_resolved", + "approval_id": "ap-1", + "workflow_id": "wf-1", + "execution_id": "ex-1", + "outcome": "approved", + } + ) + ) + + assert not event.is_set(), ( + "the parked gate was RELEASED by an unauthenticated frame — " + "the approval was never given by the operator" + ) + with rt._approval_lock: + assert "ap-1" in rt._approval_pending, ( + "the pending approval was consumed by an unauthenticated " + "frame" + ) + finally: + rt.shutdown() + + @pytest.mark.asyncio + async def test_a_genuine_approval_releases_the_parked_gate(self): + """The control: the real path still works.""" + rt = NullRunRuntime(api_key="test-key", _test_mode=True) + try: + event, entry = _park_a_gate(rt) + conn, stub = _conn(on_approval_resolved=rt._handle_approval_resolved) + + envelope = _signed_envelope(SIGNED_APPROVAL, API_KEY, SECRET) + await conn._handle_message(json.dumps(envelope)) + + assert event.is_set(), ( + "a genuine, signed approval did NOT release the parked gate " + "— the fix broke the feature it was meant to protect" + ) + assert entry["outcome"] == "approved" + assert entry["note"] == "ok" + assert any(b'"type": "ack"' in s for s in stub.sent), ( + "the acknowledgement was not sent for a genuine approval" + ) + finally: + rt.shutdown() + + +class TestTheGateCallItself: + """The end the audit names: a parked GATE must not be released.""" + + @staticmethod + def _runtime() -> NullRunRuntime: + rt = NullRunRuntime(api_key="test-key", _test_mode=True) + rt.workflow_id = "wf-1" + rt.organization_id = "org-1" + rt._transport.check = lambda req: { # type: ignore[method-assign] + "decision": "require_approval", + "decision_source": "gateway", + "approval_id": "ap-1", + "approval_timeout_seconds": 1, + "reservation_id": RESERVATION_ID, + } + return rt + + @staticmethod + def _run_the_gate(rt: NullRunRuntime) -> dict[str, Any]: + """Run the gate call on a thread (it blocks on the approval).""" + box: dict[str, Any] = {} + + def run() -> None: + try: + box["value"] = rt.check_workflow_budget() + except BaseException as exc: # noqa: BLE001 — the outcome IS the test + box["error"] = exc + + t = threading.Thread(target=run, daemon=True) + t.start() + # wait until the gate has parked on the approval + for _ in range(100): + with rt._approval_lock: + if "ap-1" in rt._approval_pending: + break + time.sleep(0.02) + box["thread"] = t + return box + + @pytest.mark.asyncio + async def test_a_forged_approval_does_not_release_the_gate(self): + """A gate that got ``require_approval`` must not proceed.""" + rt = self._runtime() + try: + box = self._run_the_gate(rt) + conn, _ = _conn(on_approval_resolved=rt._handle_approval_resolved) + + envelope = _signed_envelope(SIGNED_DENIAL, API_KEY, SECRET) + envelope["outcome"] = "approved" # the splice + + await conn._handle_message(json.dumps(envelope)) + box["thread"].join(timeout=10) + + assert "error" in box, ( + "the gate call PROCEEDED after a forged approval — the " + "spend was authorised without the operator" + ) + assert isinstance(box["error"], NullRunApprovalDeniedError), ( + f"expected the signed DENIAL to surface, got {box['error']!r}" + ) + finally: + rt.shutdown() + + @pytest.mark.asyncio + async def test_an_unsigned_approval_does_not_release_the_gate(self): + """Unauthenticated ⇒ the waiter times out ⇒ fail-CLOSED.""" + rt = self._runtime() + try: + box = self._run_the_gate(rt) + conn, _ = _conn(on_approval_resolved=rt._handle_approval_resolved) + + await conn._handle_message( + json.dumps( + { + "type": "approval_resolved", + "approval_id": "ap-1", + "workflow_id": "wf-1", + "execution_id": "ex-1", + "outcome": "approved", + } + ) + ) + box["thread"].join(timeout=15) + + assert "error" in box, ( + "the gate call PROCEEDED after an unauthenticated approval" + ) + assert isinstance(box["error"], NullRunApprovalExpiredError), ( + f"expected the approval to time out, got {box['error']!r}" + ) + finally: + rt.shutdown() + + @pytest.mark.asyncio + async def test_a_genuine_approval_releases_the_gate(self): + """The control: the real path still works end to end.""" + rt = self._runtime() + try: + box = self._run_the_gate(rt) + conn, _ = _conn(on_approval_resolved=rt._handle_approval_resolved) + + envelope = _signed_envelope(SIGNED_APPROVAL, API_KEY, SECRET) + await conn._handle_message(json.dumps(envelope)) + box["thread"].join(timeout=10) + + assert "error" not in box, ( + f"a genuine, signed approval did NOT release the gate: " + f"{box.get('error')!r}" + ) + assert box.get("value") == RESERVATION_ID, ( + f"the gate call returned {box.get('value')!r} — the " + "reservation the backend minted" + ) + finally: + rt.shutdown() + + +class TestTheKillSwitchStillWorks: + """The safety direction must not regress. + + A kill/pause is delivered on the SAME socket. Refusing to deliver + it would be the dangerous direction, so an unauthenticated + ``state_change`` is still dispatched — it can only stop the agent, + never release it. + """ + + @pytest.mark.asyncio + async def test_an_unsigned_state_change_is_still_delivered(self): + received: list[dict] = [] + conn, _ = _conn(api_key="k", secret_key=None, on_state_change=received.append) + + await conn._handle_message( + json.dumps( + { + "type": "state_change", + "workflow_id": "wf-1", + "state": "Killed", + "version": 1, + } + ) + ) + + assert len(received) == 1, ( + "the kill switch stopped working — a kill must be delivered " + "even when the frame carries no signature" + ) + assert received[0]["state"] == "Killed" diff --git a/tests/test_zero_code_preflight.py b/tests/test_zero_code_preflight.py index 0796775..c399ae8 100644 --- a/tests/test_zero_code_preflight.py +++ b/tests/test_zero_code_preflight.py @@ -206,8 +206,10 @@ def test_preflight_declares_the_calls_own_numbers(): # The historical value was the literal 1 — one token. The whole # point of ADR-101 is that the call's own size is declared. assert kwargs["estimated_tokens"] >= 100 - # A cached "allow" carries the PREVIOUS call's reservation id. - assert kwargs["use_gate_cache"] is False + # There is no in-process gate cache, so the pre-flight never + # passes a "do not cache me" flag — every call is its own /gate + # call with its own reservation (SDK-1). + assert "use_gate_cache" not in kwargs def test_no_output_cap_means_no_output_cap_is_declared(): @@ -583,7 +585,7 @@ def test_the_requests_path_preflights_and_carries_the_reservation( ) assert runtime.gate_kwargs["model"] == "gpt-4o-mini" assert runtime.gate_kwargs["max_output_tokens"] == 128 - assert runtime.gate_kwargs["use_gate_cache"] is False + assert "use_gate_cache" not in runtime.gate_kwargs assert len(runtime.tracked) == 1 assert runtime.tracked[0][1] == "req-1-1" diff --git a/uv.lock b/uv.lock index c1715b6..53b420a 100644 --- a/uv.lock +++ b/uv.lock @@ -92,6 +92,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, ] +[[package]] +name = "backports-tarfile" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", size = 86406, upload-time = "2024-05-28T17:01:54.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, +] + +[[package]] +name = "build" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "os_name == 'nt'" }, + { name = "importlib-metadata", marker = "python_full_version < '3.10.2'" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/67/4898a44ea4f3f8e213b0954ec0aa0a16971d62a6212d6ea3931e97115b99/build-1.6.1.tar.gz", hash = "sha256:51cc11666391ab6f092070437ac747002ff46f3e4113a3622177ee6b488bfc53", size = 113427, upload-time = "2026-09-10T07:56:00.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/9b/9fb3585dabcd73a1b2a6267f63f62649347c9e6d072c9fde365b105abb2c/build-1.6.1-py3-none-any.whl", hash = "sha256:ecd351a4be9d35a9eaaba244a7687143c9c7d4aea6ac964e7e7ddab20cbcf4e7", size = 31179, upload-time = "2026-09-10T07:55:59.148Z" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -101,6 +126,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, ] +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", version = "3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "3.12", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/0b/ba385d8ccedf926c3cd06e8e2f327027da5afe5f0eb30f1f7bc43ac55125/cffi-2.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004", size = 211037, upload-time = "2026-08-03T21:19:17.705Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b9/0f2e58b2cefa33255bff36935d42b13180fe559bba82596540eb404bde7d/cffi-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9", size = 218652, upload-time = "2026-08-03T21:19:18.735Z" }, + { url = "https://files.pythonhosted.org/packages/75/77/60bebf6f818bec84210ac5b6979ce4eeadce6fbbaabc9c7ab23e506d1ce5/cffi-2.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6", size = 218742, upload-time = "2026-08-03T21:19:22.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ae/679bf47e73fd77b352171727f07de559a003f14de5d02b904a6ec1fa73ca/cffi-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf", size = 221054, upload-time = "2026-08-03T21:19:23.694Z" }, + { url = "https://files.pythonhosted.org/packages/09/b8/eefc0e06913b70aa153bf74c946094a18f58fd4aff11b7f372bfdfdca050/cffi-2.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659", size = 213489, upload-time = "2026-08-03T21:19:24.922Z" }, + { url = "https://files.pythonhosted.org/packages/6f/13/4e56852824a03cdf68523a35686f1c28eacd4bd30a7b0a78e682e6e6e1d3/cffi-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9", size = 220241, upload-time = "2026-08-03T21:19:26.214Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5a/4707a0dc1f203f5dde5a907b0d4e3c25d71120241048bd5bc6f1bb9d4e71/cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0", size = 211805, upload-time = "2026-08-03T21:19:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/ad/66/c19feabb28485b6e0bbaaafa90837a1ef5d302e90f2178bd33f17a49879b/cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813", size = 218716, upload-time = "2026-08-03T21:19:32.896Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a4/4399daaf8f7dfee9d7c3327fdb0426ee041cc63edc358b93911ceb2bfc7a/cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632", size = 217807, upload-time = "2026-08-03T21:19:36.286Z" }, + { url = "https://files.pythonhosted.org/packages/28/f7/dabe6da2466ecbd82dc62e7342dc6b1065dad990c06f00f0ede9ebf2a0ed/cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd", size = 221252, upload-time = "2026-08-03T21:19:37.416Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/616202d8e51342c07d2534c510111c4cc37201775ce8f60802c9335d1edd/cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a", size = 214214, upload-time = "2026-08-03T21:19:38.507Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c6/ab025d75d2c26c19b087c0124e75ee31cb65032f4fe345d356d8c507ab97/cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa", size = 219408, upload-time = "2026-08-03T21:19:39.809Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.9" @@ -300,6 +381,58 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11'" }, ] +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/19/8e/aa1fc533d4546b127b45de8aa024eb5933d23eff9debfe25931e56861095/cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047", size = 4710431, upload-time = "2026-09-30T14:44:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/6a/64/72bc3f75176e7e406b748a3e3830432b8c51297b38368713df04dc04898a/cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539", size = 4694824, upload-time = "2026-09-30T14:44:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c6/62c77550edfa5ca3f14bf44a1e6739b9fa09d6e998a11d97ed8213bccc98/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1", size = 4716967, upload-time = "2026-09-30T14:44:19.661Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9a/6f2f0304d634ceafdeaf23e84537336664ac419b5d07611675c2ad3f6b7a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18", size = 4727698, upload-time = "2026-09-30T14:44:24.178Z" }, + { url = "https://files.pythonhosted.org/packages/1d/de/66bcf9244d118663b2e1aaded8990f4640e3d7b7411870a5765f252074d2/cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37", size = 4354821, upload-time = "2026-09-30T14:44:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e6/db28a28c7b6c676addce89136de3d8db49ea825a8c863472e36e42ead4ad/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2", size = 4716748, upload-time = "2026-09-30T14:44:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/6c/01/03263395f74d50b071e9e66daace3f8bef80493e5d410726f2ba8554736b/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05", size = 4727268, upload-time = "2026-09-30T14:44:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/2bfe8f29ec0cc9c0d99359c4161adf32858e4934b72c6d100d2ac0bbe962/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e", size = 4849503, upload-time = "2026-09-30T14:44:34.969Z" }, + { url = "https://files.pythonhosted.org/packages/54/44/e80651ecbf0e42b62e2bb5f5768916e07eea72e1297338956a61df361f88/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e", size = 5004057, upload-time = "2026-09-30T14:44:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/1d/7a/f08d34ce09d60f89ebd391e2ebc6ba2b995e6dd7552f41820f8085f94e53/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67", size = 4716035, upload-time = "2026-09-30T15:29:48.681Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/e18fb65592451a2acb76e9f2fbe14e0f47a8318b4c5430f1633851d03daa/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a", size = 4726917, upload-time = "2026-09-30T15:29:50.608Z" }, + { url = "https://files.pythonhosted.org/packages/83/28/38fdce17e60f6b825e69fc3b7f75e70a6612759980704697e1de4cbfaf6e/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48", size = 4715341, upload-time = "2026-09-30T15:29:52.522Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b1/d9121a717e0f893c64bd6ca7702614778d7df2a5c309128a002421788516/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42", size = 4726322, upload-time = "2026-09-30T15:29:54.263Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -309,6 +442,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docutils" +version = "0.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/a4/5180d9afc57e8fca05601dd652bdff19604c218814037fe90ffc7625a50a/docutils-0.23.tar.gz", hash = "sha256:746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e", size = 2303823, upload-time = "2026-05-27T17:41:06.934Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/91/30151a39f7570f448ed84529390628a651d7f27c87d73c9b887f8189695e/docutils-0.23-py3-none-any.whl", hash = "sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea", size = 634701, upload-time = "2026-05-27T17:40:58.442Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" @@ -383,6 +525,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "id" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/04/c2156091427636080787aac190019dc64096e56a23b7364d3c1764ee3a06/id-1.6.1.tar.gz", hash = "sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069", size = 18088, upload-time = "2026-02-04T16:19:41.26Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/77/de194443bf38daed9452139e960c632b0ef9f9a5dd9ce605fdf18ca9f1b1/id-1.6.1-py3-none-any.whl", hash = "sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca", size = 14689, upload-time = "2026-02-04T16:19:40.051Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -413,6 +567,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools", version = "11.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "more-itertools", version = "11.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-tarfile", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools", version = "11.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "more-itertools", version = "11.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/1f/c23395957d41ccf27c4e535c3d334c4051e5395b3752057ba4cbaec35c56/jaraco_functools-4.6.0.tar.gz", hash = "sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280", size = 20837, upload-time = "2026-07-14T01:28:02.544Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/36/ecc85bc96c273dc8a11273ed4782272975e6338d4a3e9228621175edf0e3/jaraco_functools-4.6.0-py3-none-any.whl", hash = "sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30", size = 11677, upload-time = "2026-07-14T01:28:01.59Z" }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -434,9 +635,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, ] +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + [[package]] name = "langchain-core" -version = "0.3.86" +version = "0.3.76" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -446,11 +665,10 @@ dependencies = [ { name = "pyyaml" }, { name = "tenacity" }, { name = "typing-extensions" }, - { name = "uuid-utils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/8d/d54586b8f65c6fc209db93916ff9e919e1cc14bad8fe66880ea4d7ea9d6c/langchain_core-0.3.86.tar.gz", hash = "sha256:671cbc96a325fe47f7dbab421236ada2d437bc4bfad0038102264885d0b462e2", size = 603154, upload-time = "2026-05-07T16:48:08.14Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/4d/5e2ea7754ee0a1f524c412801c6ba9ad49318ecb58b0d524903c3d9efe0a/langchain_core-0.3.76.tar.gz", hash = "sha256:71136a122dd1abae2c289c5809d035cf12b5f2bb682d8a4c1078cd94feae7419", size = 573568, upload-time = "2025-09-10T14:49:39.863Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/93/ba19ca54701c6118e68f8785949b6c0eab1df3a5cfa5310508cc86877994/langchain_core-0.3.86-py3-none-any.whl", hash = "sha256:7d2a1c50d2d2a139dbc6465cd339f32d14aa43db5ac9bd232e5b567a238709e8", size = 461306, upload-time = "2026-05-07T16:48:06.283Z" }, + { url = "https://files.pythonhosted.org/packages/77/b5/501c0ffcb09c734457ceaa86bc7b1dd37b6a261147bd653add03b838aacb/langchain_core-0.3.76-py3-none-any.whl", hash = "sha256:46e0eb48c7ac532432d51f8ca1ece1804c82afe9ae3dcf027b867edadf82b3ec", size = 447508, upload-time = "2025-09-10T14:49:38.179Z" }, ] [[package]] @@ -565,6 +783,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/5d/3dcec2884ba1b0806d1408612555c38dd5d68e90156b59f75f6e36435c3a/librt-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99", size = 110771, upload-time = "2026-07-08T12:26:12.303Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "more-itertools" +version = "11.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/de/1d/f4da6f02cdffe04d6362210b807146a26044c88d839208aec273bb0d9184/more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", size = 145772, upload-time = "2026-05-22T14:14:29.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, +] + +[[package]] +name = "more-itertools" +version = "11.2.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b8/c63995bebbb9ac9c059350e0fb8c82c2420c85aeca7d34e40aecf3df9281/more_itertools-11.2.1.tar.gz", hash = "sha256:cbf08fd0af284dc69718b9b76a8e6203df624d70209ea511e4219d350c856f63", size = 154133, upload-time = "2026-10-14T18:50:30.407Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/57/3c4b1c6f2e0cc29ba023802458ea3d2a996f87085559e4a5f2192d549532/more_itertools-11.2.1-py3-none-any.whl", hash = "sha256:35a7377edd1dd6608dcb2cdf534ded55ea32d49448875ad042cd3f879fb1ded0", size = 74405, upload-time = "2026-10-14T18:50:29.116Z" }, +] + [[package]] name = "mypy" version = "2.3.0" @@ -634,9 +901,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "nh3" +version = "0.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/2f/022b27146d52d24b1b353b003359134788ecbcd6fcdf6283adbd57c0fbc8/nh3-0.3.7.tar.gz", hash = "sha256:71860d01c16f4d8c72e334e0674beb2b0899dbd0bf760de18932ef4390303848", size = 25662, upload-time = "2026-08-23T14:26:30.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/88/b594f0e86856b37e182fb663283da419eea6424972506e640e890885467f/nh3-0.3.7-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:91a4dab4e94d9fc54b9f67b1adfb23e81fab7ab43f33c3b8c97be9aa38f789ba", size = 1471147, upload-time = "2026-08-23T14:25:55.259Z" }, + { url = "https://files.pythonhosted.org/packages/1e/60/847a21339f095c4d4c655af31fa2d18b174585bcc210709facacc7ce205c/nh3-0.3.7-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eae64328e46a25785535afcb6885b6f182ecaf5ee8c88f8c075422db8aacc65b", size = 820463, upload-time = "2026-08-23T14:25:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7f/1a103e00aaf5e59f2dee4c2709aac609bb2d4bb74fddaf0dcfade11ed87b/nh3-0.3.7-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4968fe8d2db97c6f047659bf46a449fd8ec377f44ebf3e0a1b96c0d3a333ae32", size = 861456, upload-time = "2026-08-23T14:25:58.087Z" }, + { url = "https://files.pythonhosted.org/packages/d8/4a/e9c436089a0c80b928011ead0efd156aa7639a19b6064ef58dcedcab8369/nh3-0.3.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:be53a4825585f701955cb9baf49f478f56eb81e20294329fe4bc689dd5dd81fa", size = 1023930, upload-time = "2026-08-23T14:25:59.465Z" }, + { url = "https://files.pythonhosted.org/packages/04/5c/aa1468e3e281e78d2b3b7d762ccba59f681af355e971dbd255d5903f7b86/nh3-0.3.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:94fd6e59553fbb9ffd8ba71bbd5a54e3126ba01799a097ae30d5341d750bc6ac", size = 1102614, upload-time = "2026-08-23T14:26:00.869Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/57d186d9d3dd38905dc12dddb3484406cdf6aa0b1ce33639a2d277d4ee1c/nh3-0.3.7-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:18f4278ecd157d43cb35acd5aae9f35cfa79f546b4922bd86536adc0f6312102", size = 1059915, upload-time = "2026-08-23T14:26:02.388Z" }, + { url = "https://files.pythonhosted.org/packages/6b/53/097a5ad0b34b15d67a472ef849165a54209fa5fbd3e639801c6fe439ba28/nh3-0.3.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:808def0c8c07843e6e50dc84f532457bfa2cfd17417b219a5d9e7c773709331a", size = 1047402, upload-time = "2026-08-23T14:26:03.897Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/c57a2c70534418310889a65ccfac3525e62f0bc0a8613225903403755ce7/nh3-0.3.7-cp314-cp314t-win32.whl", hash = "sha256:874b7d67a067bd29a59223f6270fc30da4edd8e6d87fd219fc93bcbaa662c946", size = 619895, upload-time = "2026-08-23T14:26:05.105Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b7/efda1d0a611d940bdfde6893bde1ea6b7b7d48c31273aea48e35b822fd58/nh3-0.3.7-cp314-cp314t-win_amd64.whl", hash = "sha256:614dac4a4c36ad084e78447d16fe898dedd762e354a7ab9cda2984e82f67883d", size = 633456, upload-time = "2026-08-23T14:26:06.661Z" }, + { url = "https://files.pythonhosted.org/packages/1d/18/3ab564595cb88196f50d26e163ed0fd2acc731ab26ac615df91981885887/nh3-0.3.7-cp314-cp314t-win_arm64.whl", hash = "sha256:157ec1eb7a62f3d9a7badb8d82d89aa810e3e24e097eedfa481a25d0c8a99877", size = 611003, upload-time = "2026-08-23T14:26:07.813Z" }, + { url = "https://files.pythonhosted.org/packages/94/0d/c257754bf57f829f307aa226bbe136d3a1356b5a0d08324c7b6bd2a8aacd/nh3-0.3.7-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6c3aa50eb26e9228238271db9f983cbc3b006dfbfeca2d4dc34c33ddc6ac5ea5", size = 1493959, upload-time = "2026-08-23T14:26:09.025Z" }, + { url = "https://files.pythonhosted.org/packages/07/42/a687e7091928806e514f89fa2666f25ec9bfe0a902fc4402b25e51ce408b/nh3-0.3.7-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f266d3f1b3647449923a8e406524632220dd5d8b647078dfe45b885d33d10479", size = 859615, upload-time = "2026-08-23T14:26:10.606Z" }, + { url = "https://files.pythonhosted.org/packages/85/05/b0e6bef633549a23347d5462aa288fcc42381e7918482062ca3cb456242a/nh3-0.3.7-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e8fd1ab205258b29254f72db377d99e2c96aa7653ef3b015ccab0420b094b506", size = 839872, upload-time = "2026-08-23T14:26:12.037Z" }, + { url = "https://files.pythonhosted.org/packages/17/40/2a0921d45b20828708bcb56887e47dcf8cae13818de5bf9a01308d348712/nh3-0.3.7-cp38-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:19f288c938ec6eef1f5d2c6cab47838e71fef8097e1c1233802be5a6230ba086", size = 1091325, upload-time = "2026-08-23T14:26:13.34Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d1/9d70e0e418a48280ec0ddc6c1b08b4b1136ebcc31a1625e57ff5c665fa51/nh3-0.3.7-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de2b2aab32ea303405debefdcfc58043d3e635fa3f67b9eb140d2b0e0c0d2563", size = 1042482, upload-time = "2026-08-23T14:26:14.667Z" }, + { url = "https://files.pythonhosted.org/packages/93/a7/02dd159d4e71f98607d8d4249cddb7561e77be1a8e4dec77d76e1b68fc99/nh3-0.3.7-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b7279d43323a25225df23576af6594a16693f61431170848b8b2ac21ad4f174", size = 946868, upload-time = "2026-08-23T14:26:16.094Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ed/c5510c615dce55b6fcc364aa1838142f938beed64f5e4927490dfcaf4405/nh3-0.3.7-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70f5ac8626e899a4bab0ef74ca2f5bd602f49c7b739e6e5026b4afc6d63dac42", size = 832161, upload-time = "2026-08-23T14:26:17.272Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e3/3212c1a5b5745245d7f18885207bbddb34c56075f34dd682bd539aad55cc/nh3-0.3.7-cp38-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:5ffdfcb9a686ffb12765376bcfb6b5b55728516d3c0ee317d29982381ded3df8", size = 849791, upload-time = "2026-08-23T14:26:18.498Z" }, + { url = "https://files.pythonhosted.org/packages/20/64/9e36594efad6c290de4240d02cb2bd80c339a4ab1c4de66e599ffa6d9d81/nh3-0.3.7-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc42bb1193c1e28a1e74c2cabaca178e118a7103e8832699fef8a2b3e2496493", size = 875473, upload-time = "2026-08-23T14:26:19.908Z" }, + { url = "https://files.pythonhosted.org/packages/00/0c/1a8985fd43fea5530c0ac890b6f0b423770ee72f111b70b7a77f2dec243a/nh3-0.3.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d56e76bd3cadb09b6b0cef364850811663734b348a25f5f587a2819c495367bd", size = 1036463, upload-time = "2026-08-23T14:26:21.536Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5d/891e533b716cf00df76ad0ba6485dcfd14d59a6430a3cc99057c4c04004e/nh3-0.3.7-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:fd4a70efb45d5372174f718878eb7a35c12677626a63b2f103b23b833457dcac", size = 1116029, upload-time = "2026-08-23T14:26:22.907Z" }, + { url = "https://files.pythonhosted.org/packages/42/e5/ae8c0782fce74fb6fcf7234bb3d4017f37ce181b4f9d29369eab21c50a04/nh3-0.3.7-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:15f5fbf090f5c88d61c820e1fc1fceecb6520cca9fe85649c06b57ef9dc9ff62", size = 1076589, upload-time = "2026-08-23T14:26:24.302Z" }, + { url = "https://files.pythonhosted.org/packages/26/a4/c3423351e8d864ad756e85e15f0c01433361f14d34e4ed156482c0518f2a/nh3-0.3.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6698a822132beedab80f131c08d8d0ac5a178ddeb488d02ca4b67716ecfac7af", size = 1058871, upload-time = "2026-08-23T14:26:25.674Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6a/478f153f1d7c0baaa3d1e8bb5fdcee3a6235f90fe44ea969a9d4e2b8c47a/nh3-0.3.7-cp38-abi3-win32.whl", hash = "sha256:6e4280115d44c3b278eef712a86748c1a723105cd79feec46952383117ab4e59", size = 630729, upload-time = "2026-08-23T14:26:26.932Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b9/34433ccb1f0fe6968dabbb7d4bf5721c6221878ef07832748c06655a6a80/nh3-0.3.7-cp38-abi3-win_amd64.whl", hash = "sha256:618e3059caf41ccdf5dcccb3fa9df4cf6e4efe23d1382a8bbfca272a8a4f8bfc", size = 644462, upload-time = "2026-08-23T14:26:28.294Z" }, + { url = "https://files.pythonhosted.org/packages/f9/70/e140dffff6e808dc6343598df76e7e2407fd0f581de3524c75fba2e0cf24/nh3-0.3.7-cp38-abi3-win_arm64.whl", hash = "sha256:f04b7d333b27f13ca439da3cf1c75c2fba34f104969f6ce4ac8e7079699c2f4a", size = 621867, upload-time = "2026-08-23T14:26:29.547Z" }, +] + [[package]] name = "nullrun" -version = "0.23.0" +version = "0.24.0" source = { editable = "." } dependencies = [ { name = "httpx" }, @@ -667,6 +968,12 @@ websocket = [ { name = "websockets" }, ] +[package.dev-dependencies] +publish = [ + { name = "build" }, + { name = "twine" }, +] + [package.metadata] requires-dist = [ { name = "coverage", extras = ["toml"], marker = "extra == 'dev'", specifier = ">=7.0" }, @@ -690,6 +997,12 @@ requires-dist = [ ] provides-extras = ["opentelemetry", "websocket", "dev"] +[package.metadata.requires-dev] +publish = [ + { name = "build", specifier = "==1.6.1" }, + { name = "twine", specifier = "==7.0.0" }, +] + [[package]] name = "opentelemetry-api" version = "1.34.1" @@ -798,11 +1111,11 @@ wheels = [ [[package]] name = "packaging" -version = "25.0" +version = "26.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] [[package]] @@ -823,6 +1136,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pycparser" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/da/a8/c5fdbeee588bb8ada9458774f43adf1bdd30bd59157055142183e769a024/pycparser-3.11.tar.gz", hash = "sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc", size = 113796, upload-time = "2026-10-09T12:56:59.539Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/11/0e6f11117525ff0eec40ebac3d313376f102df93ca44ad9e893ee85e4f89/pycparser-3.11-py3-none-any.whl", hash = "sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80", size = 51178, upload-time = "2026-10-09T12:56:58.131Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + [[package]] name = "pydantic" version = "2.11.10" @@ -934,6 +1275,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyproject-hooks" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/5d/f2ddeef4a855a102aaae5e97826a0260007522ab504421b75addfdb1517c/pyproject_hooks-1.3.3.tar.gz", hash = "sha256:defda19b854fa0d3bd4f76ea4ddcba8abd7dcfcdd585a6690ade050744fc5f43", size = 21013, upload-time = "2026-09-16T08:58:03.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/11/044d1ae1b4ec0d7af88ee5bc91e081be1022533032b906a7bdabdbb60977/pyproject_hooks-1.3.3-py3-none-any.whl", hash = "sha256:5fc53fdac9f7bd63fbcdc868fb5f90b4784d78a53a3d3388cd738b807441a20b", size = 10724, upload-time = "2026-09-16T08:58:02.96Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -1006,6 +1356,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1070,6 +1429,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "readme-renderer" +version = "46.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "nh3" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/d7/9309494fad74ee831d4546f69325b5519f37c6dfb2d9ba495db8c6d4f4ca/readme_renderer-46.0.tar.gz", hash = "sha256:af3e964914f6310a33ff67b72a4bdd940bed8d7c3bdecd2d14f40edf284bfe90", size = 38382, upload-time = "2026-08-28T15:18:32.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/72/ac5ca81fe9121fcaa9d828d21017cba00a16a98e4ea5fb60c878f93dda4f/readme_renderer-46.0-py3-none-any.whl", hash = "sha256:d0dae1f74bb273b534770cb4cccb6bb78735540afdb03c2146f4e19dcd412560", size = 14239, upload-time = "2026-08-28T15:18:31.132Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1109,6 +1482,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/4a/221da6ca167db45693d8d26c7dc79ccfc978a440251bf6721c9aaf251ac0/respx-0.23.1-py2.py3-none-any.whl", hash = "sha256:b18004b029935384bccfa6d7d9d74b4ec9af73a081cc28600fffc0447f4b8c1a", size = 25557, upload-time = "2026-04-08T14:37:14.613Z" }, ] +[[package]] +name = "rfc3986" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", size = 49026, upload-time = "2022-01-10T00:52:30.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", size = 31326, upload-time = "2022-01-10T00:52:29.594Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "ruff" version = "0.16.1" @@ -1134,6 +1529,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, ] +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -1174,6 +1582,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/db/ce8eda256fa131af12e0a76d481711abe4681b6923c27efb9a255c9e4594/tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38", size = 13237, upload-time = "2024-10-02T10:46:11.806Z" }, ] +[[package]] +name = "twine" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "id" }, + { name = "keyring", marker = "platform_machine != 'ppc64le' and platform_machine != 's390x'" }, + { name = "packaging" }, + { name = "readme-renderer" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "rfc3986" }, + { name = "rich" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/3c/58f808a359700f39a967dffede33efeac809262c03303fa3eec6afff8f49/twine-7.0.0.tar.gz", hash = "sha256:85cdb29c518efef867360ae4acd4b0dfd61c8654a22fca08e6f8539f05022177", size = 215032, upload-time = "2026-07-27T15:59:00.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/08/ddcdc06225eaad6de0e48e1002b06d919dbde20582d0662c7af51308e5d6/twine-7.0.0-py3-none-any.whl", hash = "sha256:b854164df26db268af05f49aa5c0344b10e27a494343ff05b1e0bad3b135f5a7", size = 43204, upload-time = "2026-07-27T15:58:59.26Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0"