fix(seaweedfs): reconcile restored IAM before exposing resumed ingress

openserverless/seaweedfs_deploy.py

@@ -149,6 +149,10 @@ def reconcile(owner=None):
     kopf.append_owner_reference(resources, owner)
     datastore_security.apply(policy)
     result = kube.apply(services[0])
+    # Existing deployments and restored filer data can still carry global Read.
+    # Normalize IAM before publishing ingress, without bootstrapping buckets.
+    with service_endpoint(data['name'], 9090) as (host, port):
+        reconcile_anonymous_web_access(SeaweedfsClient(f"http://{host}:{port}"))
     seaweedfs_ingress.create_seaweedfs_ingresses(data, owner)
     return result
 
@@ -203,6 +207,9 @@ def reconcile_anonymous_web_access(client):
         raise RuntimeError("Could not grant public access to web buckets")
     if web_buckets & private_buckets:
         raise ValueError("Public and private S3 bucket names overlap")
+    # Native shell commands may return success even when an operation failed.
+    # Read back the persisted policy before accepting reconciliation.
+    client.verify_anonymous_access(safe_web_buckets)
 
 def create_seaweedfs_nuv_storage(data, filer_url=None):
     """

openserverless/seaweedfs_util.py

@@ -150,6 +150,24 @@ def remove_anonymous_access(self, bucket):
         command = f's3.configure -user=anonymous -actions=Read -buckets={bucket} -delete -apply'
         return util.check(self._exec_weed_command(command), "remove_anonymous_web_access", True)
 
+    def verify_anonymous_access(self, buckets):
+        """Fail closed if persisted anonymous IAM differs from the web allowlist."""
+        try:
+            response = requests.get(f"{self.filer_url}/etc/iam/identity.json", timeout=10)
+            response.raise_for_status()
+            identities = response.json()["identities"]
+            if not isinstance(identities, list):
+                raise ValueError()
+            anonymous = [item for item in identities if item["name"] == "anonymous"]
+            expected = {f"Read:{bucket}" for bucket in buckets}
+            if len(anonymous) != 1:
+                raise ValueError()
+            actions = anonymous[0]["actions"]
+            if not isinstance(actions, list) or any(not isinstance(action, str) for action in actions) or set(actions) != expected:
+                raise ValueError()
+        except (requests.RequestException, ValueError, KeyError, TypeError):
+            raise RuntimeError("Could not verify anonymous S3 web-only grants") from None
+
     def make_public_bucket(self, bucket_name):
         """
         assign the specified buckets to the given users

tests/test_datastore_security.py

@@ -204,18 +204,24 @@ def test_existing_cluster_reconcile_secures_before_service_update(self):
                    'spec': {'type': 'ClusterIP', 'ports': []}}
         events = []
         config.configure({'components': {'seaweedfs': True}})
-        with patch.object(seaweedfs_deploy.util, 'get_seaweedfs_config_data', return_value={}), \
+        with patch.object(seaweedfs_deploy.util, 'get_seaweedfs_config_data', return_value={'name': 'seaweedfs'}), \
                 patch.object(seaweedfs_deploy, '_render_seaweedfs_spec', return_value={
                     'apiVersion': 'v1', 'kind': 'List', 'items': [service]}), \
                 patch.object(seaweedfs_deploy.datastore_security, 'apply',
                              side_effect=lambda obj: events.append(('policy', obj))), \
                 patch.object(seaweedfs_deploy.kube, 'apply',
                              side_effect=lambda obj: events.append(('service', obj))), \
+                patch.object(seaweedfs_deploy, 'service_endpoint', return_value=
+                             contextlib.nullcontext(('127.0.0.1', 41237))), \
+                patch.object(seaweedfs_deploy, 'SeaweedfsClient') as client, \
+                patch.object(seaweedfs_deploy, 'reconcile_anonymous_web_access',
+                             side_effect=lambda obj: events.append(('iam', obj))), \
                 patch.object(seaweedfs_deploy.seaweedfs_ingress,
                              'create_seaweedfs_ingresses',
                              side_effect=lambda data, obj: events.append(('ingress', obj))):
             seaweedfs_deploy.reconcile(owner)
-        self.assertEqual([event[0] for event in events], ['policy', 'service', 'ingress'])
+        self.assertEqual([event[0] for event in events], ['policy', 'service', 'iam', 'ingress'])
+        client.assert_called_once_with('http://127.0.0.1:41237')
         self.assertEqual(events[0][1]['kind'], 'NetworkPolicy')
         self.assertEqual(events[1][1]['spec']['type'], 'ClusterIP')
 

tests/test_s3_anonymous_access.py

@@ -16,13 +16,60 @@
 # under the License.
 
 import json
+import contextlib
+import requests
 import unittest
 from unittest.mock import Mock, call, patch
 
 from openserverless import seaweedfs_deploy
 
 
 class AnonymousS3AccessTest(unittest.TestCase):
+    @patch.object(seaweedfs_deploy.SeaweedfsClient, "__init__", return_value=None)
+    @patch("openserverless.seaweedfs_util.requests.get")
+    def test_persisted_policy_must_match_web_only_even_if_native_shell_succeeded(self, get, init):
+        client = seaweedfs_deploy.SeaweedfsClient()
+        client.filer_url = "http://127.0.0.1:41237"
+        get.return_value.json.return_value = {"identities": [
+            {"name": "anonymous", "actions": ["Read:custom-web"]},
+            {"name": "tenant", "actions": ["Read:private-data", "Write:private-data"]},
+        ]}
+        client.verify_anonymous_access({"custom-web"})
+        for actions in (["Read"], ["Read:custom-web", "Read:private-data"], [], ["Write:custom-web"]):
+            get.return_value.json.return_value = {"identities": [{"name": "anonymous", "actions": actions}]}
+            with self.subTest(actions=actions), self.assertRaisesRegex(RuntimeError, "verify"):
+                client.verify_anonymous_access({"custom-web"})
+        for document in ({}, {"identities": None}, {"identities": []},
+                         {"identities": [None]}, {"identities": [{"name": "anonymous", "actions": None}]},
+                         {"identities": [{"name": "anonymous", "actions": {"Read:custom-web": True}}]},
+                         {"identities": [{"name": "anonymous", "actions": ["Read:custom-web"]}] * 2}):
+            get.return_value.json.return_value = document
+            with self.subTest(document=document), self.assertRaises(RuntimeError):
+                client.verify_anonymous_access({"custom-web"})
+        get.assert_called_with("http://127.0.0.1:41237/etc/iam/identity.json", timeout=10)
+        get.side_effect = requests.ConnectionError("PRIVATE_CANARY")
+        with self.assertRaisesRegex(RuntimeError, "^Could not verify anonymous S3 web-only grants$") as raised:
+            client.verify_anonymous_access({"custom-web"})
+        self.assertNotIn("PRIVATE_CANARY", str(raised.exception))
+
+    def test_failed_iam_reconciliation_does_not_publish_ingress_or_bootstrap_storage(self):
+        owner = {"metadata": {"uid": "synthetic"}}
+        seaweedfs_deploy.cfg.configure({"components": {"seaweedfs": True}})
+        with patch.object(seaweedfs_deploy.util, "get_seaweedfs_config_data", return_value={"name": "seaweedfs"}), \
+                patch.object(seaweedfs_deploy, "_render_seaweedfs_spec", return_value={"items": [
+                    {"kind": "Service", "metadata": {"name": "seaweedfs"}}]}), \
+                patch.object(seaweedfs_deploy.datastore_security, "apply"), \
+                patch.object(seaweedfs_deploy.kube, "apply"), \
+                patch.object(seaweedfs_deploy, "service_endpoint", return_value=contextlib.nullcontext(("seaweedfs", 9090))), \
+                patch.object(seaweedfs_deploy, "SeaweedfsClient"), \
+                patch.object(seaweedfs_deploy, "reconcile_anonymous_web_access", side_effect=RuntimeError("IAM unavailable")), \
+                patch.object(seaweedfs_deploy.seaweedfs_ingress, "create_seaweedfs_ingresses") as ingress, \
+                patch.object(seaweedfs_deploy, "create_seaweedfs_nuv_storage") as bootstrap:
+            with self.assertRaisesRegex(RuntimeError, "IAM unavailable"):
+                seaweedfs_deploy.reconcile(owner)
+        ingress.assert_not_called()
+        bootstrap.assert_not_called()
+
     def test_grants_are_bucket_scoped_and_reject_invalid_names(self):
         client = seaweedfs_deploy.SeaweedfsClient.__new__(seaweedfs_deploy.SeaweedfsClient)
         client._exec_weed_command = Mock(return_value=True)