diff --git a/docs/seaweedfs-iam-diagnostics.md b/docs/seaweedfs-iam-diagnostics.md new file mode 100644 index 0000000..fa16e69 --- /dev/null +++ b/docs/seaweedfs-iam-diagnostics.md @@ -0,0 +1,59 @@ + + +# Private Filer IAM verification diagnostics + +Persisted anonymous grants still have to match the public web allowlist exactly. +Failure raises `RuntimeError` and stops reconciliation before ingress publication. +The readiness wait and retry ordering remain those of the existing deployment +flow; this diagnostic change does not retry or accept an invalid policy. + +Errors keep their original message prefix and add a safe, fixed classification: + +```text +Could not verify anonymous S3 web-only grants [phase=read_policy category=http_error status=503] +``` + +| Category | Phase | Meaning | +| --- | --- | --- | +| `transport_timeout` | `read_policy` | Request connection/read timeout | +| `transport_error` | `read_policy` | Other request/connection error | +| `http_error` | `read_policy` | HTTP error; includes validated numeric status or `unknown` | +| `json_error` | `decode_policy` | Response is not decodable JSON | +| `schema_error` | `validate_policy` | Invalid IAM document/action structure | +| `anonymous_identity` | `validate_policy` | Missing or duplicated anonymous identity | +| `grant_mismatch` | `validate_policy` | Persisted grants differ from the exact expected set | + +Response bodies, exception messages, URLs, credentials, expected bucket names and +IAM documents are not included. Underlying exception chaining is suppressed. +Consumers must accept the old generic message too; its cause is unclassified. +An observed error in historical logs alone does not establish present health or +the outcome of the handler's subsequent retry. + +The unit suite covers valid/scoped policies, malformed documents, duplicated +identities, incorrect grants, transport failures and safe diagnostic output. +Normal and optimized Python runs pass the 167-test suite. Runtime qualification +uses synthetic loopback HTTP responses and a read-only check of the actual Filer +policy; synthetic invalid policies are never installed in the real datastore. + +On Linux AMD64 WSL2, the exact candidate module passed nine synthetic loopback +cases plus the actual policy readback. Operator resume succeeded, all 18 +persistent pods remained Ready, and all 11 PVC UIDs were preserved. No historical +Job or pod was removed. This is focused diagnostic/readback qualification, not +full application acceptance, fresh-installation or ARM64 qualification. diff --git a/openserverless/seaweedfs_util.py b/openserverless/seaweedfs_util.py index c0cf6c5..3b6be1e 100644 --- a/openserverless/seaweedfs_util.py +++ b/openserverless/seaweedfs_util.py @@ -152,21 +152,44 @@ def remove_anonymous_access(self, bucket): def verify_anonymous_access(self, buckets): """Fail closed if persisted anonymous IAM differs from the web allowlist.""" + def fail(category, phase, status=None): + # Only fixed categories and a validated HTTP number may reach logs. + detail = f"phase={phase} category={category}" + if status is not None: + detail += f" status={status}" + raise RuntimeError(f"Could not verify anonymous S3 web-only grants [{detail}]") from None + try: response = requests.get(f"{self.filer_url}/etc/iam/identity.json", timeout=10) response.raise_for_status() - identities = response.json()["identities"] + except requests.Timeout: + fail("transport_timeout", "read_policy") + except requests.HTTPError as error: + status = getattr(error.response, "status_code", None) + fail("http_error", "read_policy", status if type(status) is int and 100 <= status <= 599 else "unknown") + except requests.RequestException: + fail("transport_error", "read_policy") + try: + document = response.json() + except ValueError: + fail("json_error", "decode_policy") + try: + identities = document["identities"] if not isinstance(identities, list): raise ValueError() anonymous = [item for item in identities if item["name"] == "anonymous"] - expected = {f"Read:{bucket}" for bucket in buckets} - if len(anonymous) != 1: - raise ValueError() + except (ValueError, KeyError, TypeError): + fail("schema_error", "validate_policy") + if len(anonymous) != 1: + fail("anonymous_identity", "validate_policy") + try: actions = anonymous[0]["actions"] - if not isinstance(actions, list) or any(not isinstance(action, str) for action in actions) or set(actions) != expected: + if not isinstance(actions, list) or any(not isinstance(action, str) for action in actions): raise ValueError() - except (requests.RequestException, ValueError, KeyError, TypeError): - raise RuntimeError("Could not verify anonymous S3 web-only grants") from None + except (ValueError, KeyError, TypeError): + fail("schema_error", "validate_policy") + if set(actions) != {f"Read:{bucket}" for bucket in buckets}: + fail("grant_mismatch", "validate_policy") def make_public_bucket(self, bucket_name): """ diff --git a/tests/test_s3_anonymous_access.py b/tests/test_s3_anonymous_access.py index 011772d..54c29c9 100644 --- a/tests/test_s3_anonymous_access.py +++ b/tests/test_s3_anonymous_access.py @@ -48,10 +48,46 @@ def test_persisted_policy_must_match_web_only_even_if_native_shell_succeeded(sel client.verify_anonymous_access({"custom-web"}) get.assert_called_with("http://127.0.0.1:41237/etc/iam/identity.json", timeout=10) get.side_effect = requests.ConnectionError("PRIVATE_CANARY") - with self.assertRaisesRegex(RuntimeError, "^Could not verify anonymous S3 web-only grants$") as raised: + with self.assertRaisesRegex(RuntimeError, r"^Could not verify anonymous S3 web-only grants \[phase=read_policy category=transport_error\]$") as raised: client.verify_anonymous_access({"custom-web"}) self.assertNotIn("PRIVATE_CANARY", str(raised.exception)) + @patch.object(seaweedfs_deploy.SeaweedfsClient, "__init__", return_value=None) + @patch("openserverless.seaweedfs_util.requests.get") + def test_safe_diagnostic_categories_preserve_fail_closed_policy(self, get, init): + import traceback + client = seaweedfs_deploy.SeaweedfsClient() + client.filer_url = "http://PRIVATE_CANARY" + valid = {"identities": [{"name": "anonymous", "actions": ["Read:custom-web"]}]} + scenarios = [ + (requests.Timeout("PRIVATE_CANARY"), None, None, "transport_timeout"), + (requests.ConnectionError("PRIVATE_CANARY"), None, None, "transport_error"), + (None, requests.HTTPError("PRIVATE_CANARY", response=Mock(status_code=503)), None, "http_error"), + (None, None, ValueError("PRIVATE_CANARY"), "json_error"), + (None, None, {"identities": [None]}, "schema_error"), + (None, None, {"identities": []}, "anonymous_identity"), + (None, None, {"identities": valid['identities'] * 2}, "anonymous_identity"), + (None, None, {"identities": [{"name": "anonymous", "actions": ["Read"]}]}, "grant_mismatch"), + ] + for transport, http_error, document, category in scenarios: + with self.subTest(category=category): + get.side_effect = transport + get.return_value.raise_for_status.side_effect = http_error + get.return_value.json.side_effect = document if isinstance(document, Exception) else None + get.return_value.json.return_value = document + with self.assertRaisesRegex(RuntimeError, "category=" + category) as raised: + client.verify_anonymous_access({"custom-web"}) + rendered = ''.join(traceback.format_exception(raised.exception)) + self.assertNotIn("PRIVATE_CANARY", rendered) + self.assertNotIn('"identities"', rendered) + if http_error: + self.assertIn("status=503", str(raised.exception)) + get.side_effect = None + get.return_value.raise_for_status.side_effect = None + get.return_value.json.side_effect = None + get.return_value.json.return_value = valid + client.verify_anonymous_access({"custom-web"}) + def test_failed_iam_reconciliation_does_not_publish_ingress_or_bootstrap_storage(self): owner = {"metadata": {"uid": "synthetic"}} seaweedfs_deploy.cfg.configure({"components": {"seaweedfs": True}})