fix: remove orphaned mesh test containers

tests/mesh/run.sh

@@ -99,6 +99,20 @@ diagnose() {
         break
       fi
     done
+    echo "----- etcd membership and per-voter peer metrics"
+    # etcdctl's simple output prints member IDs in the same hexadecimal form as the metrics' Remote labels. Read both before teardown so an extra active series can be compared with actual membership.
+    etcd_members "${api_node:-1}" 2>&1 || echo "could not read etcd membership"
+    for index in $(seq 1 "$node_count"); do
+      docker_on "$index" inspect "$(container "$index")" >/dev/null 2>&1 || continue
+      if reports=$(kubectl_on "$index" --namespace "$platform_namespace" get pods -l app.kubernetes.io/name=fleet-role-probe -o json </dev/null 2>/dev/null); then
+        while IFS=$'\t' read -r probe node; do
+          [ -n "$probe" ] || continue
+          echo "----- ${node}: etcd server and peer series"
+          kubectl_on "$index" --namespace "$platform_namespace" exec "$probe" -- sh -c 'curl -fsS --max-time 5 http://127.0.0.1:2381/metrics | grep -E "^etcd_(server_id|network_active_peers|network_known_peers)(\\{| )"' </dev/null 2>&1 || echo "could not read peer metrics for ${node}"
+        done < <(jq -r '.items[] | [.metadata.name, .spec.nodeName] | @tsv' <<< "$reports")
+        break
+      fi
+    done
     echo "----- the role controller's status and log"
     for index in $(seq 1 "$node_count"); do
       docker_on "$index" inspect "$(container "$index")" >/dev/null 2>&1 || continue
@@ -144,7 +158,7 @@ docker_on() {
 teardown() {
   for index in $(seq 1 "$node_count"); do
     if [ -f "${work}/$(node_name "$index")/docker-compose.yml" ]; then
-      docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server --profile agent --profile regrade down -v --rmi local >/dev/null 2>&1 || true
+      docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server --profile agent --profile regrade down --remove-orphans -v --rmi local || return
     fi
   done
   if [ -f "${work}/headscale/docker-compose.yml" ]; then
@@ -309,7 +323,7 @@ EOF
 run_role() {
   log "Running the cluster role"
   # Returns the playbook's own status, which a caller testing it (compose_faulty) relies on, errexit being off there. Any arguments go to ansible-playbook.
-  ansible_run -i "${work}/inventory.yml" "${repo_root}/tests/mesh/cluster.yml" "$@" || return
+  ansible_run -i "${work}/inventory.yml" "${repo_root}/tests/mesh/cluster.yml" "$@" || { diagnose >&2 || true; return 1; }
   if [ "$pin_context" = 1 ]; then grtest_assert_current_context_unchanged "$docker_config" || fail "the role changed the current Docker context"; fi
 }
 

tests/unit/test_mesh_teardown.py

@@ -0,0 +1,50 @@
+"""Exercise mesh teardown with a Compose project holding an orphan."""
+
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class MeshTeardownTest(unittest.TestCase):
+    def test_removes_orphans_before_deleting_work(self):
+        source = (ROOT / "tests/mesh/run.sh").read_text()
+        teardown = source.split("teardown() {\n", 1)[1].split("\n}\n", 1)[0]
+        with tempfile.TemporaryDirectory() as directory:
+            base = Path(directory)
+            work = base / "work"
+            project = work / "grtest-node1"
+            project.mkdir(parents=True)
+            (project / "docker-compose.yml").write_text("services: {}\n")
+            orphan = base / "orphan"
+            orphan.touch()
+            docker = base / "docker"
+            docker.write_text(
+                '#!/bin/bash\n'
+                'if [ "$1" = compose ]; then\n'
+                '  case " $* " in *" --remove-orphans "*) rm "$ORPHAN" ;; esac\n'
+                'fi\n'
+            )
+            docker.chmod(0o755)
+            script = (
+                'set -euo pipefail\n'
+                f'work="$1"; node_count=1; native_host=grtest-native; network=grtest-mesh\n'
+                'node_name() { printf "grtest-node%s" "$1"; }\n'
+                'vm() { printf "grtest-vm%s" "$1"; }\n'
+                'teardown() {\n' + teardown + '\n}\n'
+                'teardown\n'
+            )
+            env = os.environ.copy()
+            env["PATH"] = f'{base}{os.pathsep}{env["PATH"]}'
+            env["ORPHAN"] = str(orphan)
+            subprocess.run(["bash", "-c", script, "test", str(work)], env=env, check=True)
+            self.assertFalse(orphan.exists())
+            self.assertFalse(work.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()