fix: classify private Filer IAM verification failures safely

docs/seaweedfs-iam-diagnostics.md

@@ -0,0 +1,59 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Private Filer IAM verification diagnostics
+
+Persisted anonymous grants still have to match the public web allowlist exactly.
+Failure raises `RuntimeError` and stops reconciliation before ingress publication.
+The readiness wait and retry ordering remain those of the existing deployment
+flow; this diagnostic change does not retry or accept an invalid policy.
+
+Errors keep their original message prefix and add a safe, fixed classification:
+
+```text
+Could not verify anonymous S3 web-only grants [phase=read_policy category=http_error status=503]
+```
+
+| Category | Phase | Meaning |
+| --- | --- | --- |
+| `transport_timeout` | `read_policy` | Request connection/read timeout |
+| `transport_error` | `read_policy` | Other request/connection error |
+| `http_error` | `read_policy` | HTTP error; includes validated numeric status or `unknown` |
+| `json_error` | `decode_policy` | Response is not decodable JSON |
+| `schema_error` | `validate_policy` | Invalid IAM document/action structure |
+| `anonymous_identity` | `validate_policy` | Missing or duplicated anonymous identity |
+| `grant_mismatch` | `validate_policy` | Persisted grants differ from the exact expected set |
+
+Response bodies, exception messages, URLs, credentials, expected bucket names and
+IAM documents are not included. Underlying exception chaining is suppressed.
+Consumers must accept the old generic message too; its cause is unclassified.
+An observed error in historical logs alone does not establish present health or
+the outcome of the handler's subsequent retry.
+
+The unit suite covers valid/scoped policies, malformed documents, duplicated
+identities, incorrect grants, transport failures and safe diagnostic output.
+Normal and optimized Python runs pass the 167-test suite. Runtime qualification
+uses synthetic loopback HTTP responses and a read-only check of the actual Filer
+policy; synthetic invalid policies are never installed in the real datastore.
+
+On Linux AMD64 WSL2, the exact candidate module passed nine synthetic loopback
+cases plus the actual policy readback. Operator resume succeeded, all 18
+persistent pods remained Ready, and all 11 PVC UIDs were preserved. No historical
+Job or pod was removed. This is focused diagnostic/readback qualification, not
+full application acceptance, fresh-installation or ARM64 qualification.

openserverless/seaweedfs_util.py

@@ -152,21 +152,44 @@ def remove_anonymous_access(self, bucket):
 
     def verify_anonymous_access(self, buckets):
         """Fail closed if persisted anonymous IAM differs from the web allowlist."""
+        def fail(category, phase, status=None):
+            # Only fixed categories and a validated HTTP number may reach logs.
+            detail = f"phase={phase} category={category}"
+            if status is not None:
+                detail += f" status={status}"
+            raise RuntimeError(f"Could not verify anonymous S3 web-only grants [{detail}]") from None
+
         try:
             response = requests.get(f"{self.filer_url}/etc/iam/identity.json", timeout=10)
             response.raise_for_status()
-            identities = response.json()["identities"]
+        except requests.Timeout:
+            fail("transport_timeout", "read_policy")
+        except requests.HTTPError as error:
+            status = getattr(error.response, "status_code", None)
+            fail("http_error", "read_policy", status if type(status) is int and 100 <= status <= 599 else "unknown")
+        except requests.RequestException:
+            fail("transport_error", "read_policy")
+        try:
+            document = response.json()
+        except ValueError:
+            fail("json_error", "decode_policy")
+        try:
+            identities = document["identities"]
             if not isinstance(identities, list):
                 raise ValueError()
             anonymous = [item for item in identities if item["name"] == "anonymous"]
-            expected = {f"Read:{bucket}" for bucket in buckets}
-            if len(anonymous) != 1:
-                raise ValueError()
+        except (ValueError, KeyError, TypeError):
+            fail("schema_error", "validate_policy")
+        if len(anonymous) != 1:
+            fail("anonymous_identity", "validate_policy")
+        try:
             actions = anonymous[0]["actions"]
-            if not isinstance(actions, list) or any(not isinstance(action, str) for action in actions) or set(actions) != expected:
+            if not isinstance(actions, list) or any(not isinstance(action, str) for action in actions):
                 raise ValueError()
-        except (requests.RequestException, ValueError, KeyError, TypeError):
-            raise RuntimeError("Could not verify anonymous S3 web-only grants") from None
+        except (ValueError, KeyError, TypeError):
+            fail("schema_error", "validate_policy")
+        if set(actions) != {f"Read:{bucket}" for bucket in buckets}:
+            fail("grant_mismatch", "validate_policy")
 
     def make_public_bucket(self, bucket_name):
         """

tests/test_s3_anonymous_access.py

@@ -48,10 +48,46 @@ def test_persisted_policy_must_match_web_only_even_if_native_shell_succeeded(sel
                 client.verify_anonymous_access({"custom-web"})
         get.assert_called_with("http://127.0.0.1:41237/etc/iam/identity.json", timeout=10)
         get.side_effect = requests.ConnectionError("PRIVATE_CANARY")
-        with self.assertRaisesRegex(RuntimeError, "^Could not verify anonymous S3 web-only grants$") as raised:
+        with self.assertRaisesRegex(RuntimeError, r"^Could not verify anonymous S3 web-only grants \[phase=read_policy category=transport_error\]$") as raised:
             client.verify_anonymous_access({"custom-web"})
         self.assertNotIn("PRIVATE_CANARY", str(raised.exception))
 
+    @patch.object(seaweedfs_deploy.SeaweedfsClient, "__init__", return_value=None)
+    @patch("openserverless.seaweedfs_util.requests.get")
+    def test_safe_diagnostic_categories_preserve_fail_closed_policy(self, get, init):
+        import traceback
+        client = seaweedfs_deploy.SeaweedfsClient()
+        client.filer_url = "http://PRIVATE_CANARY"
+        valid = {"identities": [{"name": "anonymous", "actions": ["Read:custom-web"]}]}
+        scenarios = [
+            (requests.Timeout("PRIVATE_CANARY"), None, None, "transport_timeout"),
+            (requests.ConnectionError("PRIVATE_CANARY"), None, None, "transport_error"),
+            (None, requests.HTTPError("PRIVATE_CANARY", response=Mock(status_code=503)), None, "http_error"),
+            (None, None, ValueError("PRIVATE_CANARY"), "json_error"),
+            (None, None, {"identities": [None]}, "schema_error"),
+            (None, None, {"identities": []}, "anonymous_identity"),
+            (None, None, {"identities": valid['identities'] * 2}, "anonymous_identity"),
+            (None, None, {"identities": [{"name": "anonymous", "actions": ["Read"]}]}, "grant_mismatch"),
+        ]
+        for transport, http_error, document, category in scenarios:
+            with self.subTest(category=category):
+                get.side_effect = transport
+                get.return_value.raise_for_status.side_effect = http_error
+                get.return_value.json.side_effect = document if isinstance(document, Exception) else None
+                get.return_value.json.return_value = document
+                with self.assertRaisesRegex(RuntimeError, "category=" + category) as raised:
+                    client.verify_anonymous_access({"custom-web"})
+                rendered = ''.join(traceback.format_exception(raised.exception))
+                self.assertNotIn("PRIVATE_CANARY", rendered)
+                self.assertNotIn('"identities"', rendered)
+                if http_error:
+                    self.assertIn("status=503", str(raised.exception))
+        get.side_effect = None
+        get.return_value.raise_for_status.side_effect = None
+        get.return_value.json.side_effect = None
+        get.return_value.json.return_value = valid
+        client.verify_anonymous_access({"custom-web"})
+
     def test_failed_iam_reconciliation_does_not_publish_ingress_or_bootstrap_storage(self):
         owner = {"metadata": {"uid": "synthetic"}}
         seaweedfs_deploy.cfg.configure({"components": {"seaweedfs": True}})