diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 4ce92ae..8085a34 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -99,6 +99,20 @@ diagnose() { break fi done + echo "----- etcd membership and per-voter peer metrics" + # etcdctl's simple output prints member IDs in the same hexadecimal form as the metrics' Remote labels. Read both before teardown so an extra active series can be compared with actual membership. + etcd_members "${api_node:-1}" 2>&1 || echo "could not read etcd membership" + for index in $(seq 1 "$node_count"); do + docker_on "$index" inspect "$(container "$index")" >/dev/null 2>&1 || continue + if reports=$(kubectl_on "$index" --namespace "$platform_namespace" get pods -l app.kubernetes.io/name=fleet-role-probe -o json /dev/null); then + while IFS=$'\t' read -r probe node; do + [ -n "$probe" ] || continue + echo "----- ${node}: etcd server and peer series" + kubectl_on "$index" --namespace "$platform_namespace" exec "$probe" -- sh -c 'curl -fsS --max-time 5 http://127.0.0.1:2381/metrics | grep -E "^etcd_(server_id|network_active_peers|network_known_peers)(\\{| )"' &1 || echo "could not read peer metrics for ${node}" + done < <(jq -r '.items[] | [.metadata.name, .spec.nodeName] | @tsv' <<< "$reports") + break + fi + done echo "----- the role controller's status and log" for index in $(seq 1 "$node_count"); do docker_on "$index" inspect "$(container "$index")" >/dev/null 2>&1 || continue @@ -144,7 +158,7 @@ docker_on() { teardown() { for index in $(seq 1 "$node_count"); do if [ -f "${work}/$(node_name "$index")/docker-compose.yml" ]; then - docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server --profile agent --profile regrade down -v --rmi local >/dev/null 2>&1 || true + docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server --profile agent --profile regrade down --remove-orphans -v --rmi local || return fi done if [ -f "${work}/headscale/docker-compose.yml" ]; then @@ -309,7 +323,7 @@ EOF run_role() { log "Running the cluster role" # Returns the playbook's own status, which a caller testing it (compose_faulty) relies on, errexit being off there. Any arguments go to ansible-playbook. - ansible_run -i "${work}/inventory.yml" "${repo_root}/tests/mesh/cluster.yml" "$@" || return + ansible_run -i "${work}/inventory.yml" "${repo_root}/tests/mesh/cluster.yml" "$@" || { diagnose >&2 || true; return 1; } if [ "$pin_context" = 1 ]; then grtest_assert_current_context_unchanged "$docker_config" || fail "the role changed the current Docker context"; fi } diff --git a/tests/unit/test_mesh_teardown.py b/tests/unit/test_mesh_teardown.py new file mode 100644 index 0000000..b4642ec --- /dev/null +++ b/tests/unit/test_mesh_teardown.py @@ -0,0 +1,50 @@ +"""Exercise mesh teardown with a Compose project holding an orphan.""" + +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class MeshTeardownTest(unittest.TestCase): + def test_removes_orphans_before_deleting_work(self): + source = (ROOT / "tests/mesh/run.sh").read_text() + teardown = source.split("teardown() {\n", 1)[1].split("\n}\n", 1)[0] + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + work = base / "work" + project = work / "grtest-node1" + project.mkdir(parents=True) + (project / "docker-compose.yml").write_text("services: {}\n") + orphan = base / "orphan" + orphan.touch() + docker = base / "docker" + docker.write_text( + '#!/bin/bash\n' + 'if [ "$1" = compose ]; then\n' + ' case " $* " in *" --remove-orphans "*) rm "$ORPHAN" ;; esac\n' + 'fi\n' + ) + docker.chmod(0o755) + script = ( + 'set -euo pipefail\n' + f'work="$1"; node_count=1; native_host=grtest-native; network=grtest-mesh\n' + 'node_name() { printf "grtest-node%s" "$1"; }\n' + 'vm() { printf "grtest-vm%s" "$1"; }\n' + 'teardown() {\n' + teardown + '\n}\n' + 'teardown\n' + ) + env = os.environ.copy() + env["PATH"] = f'{base}{os.pathsep}{env["PATH"]}' + env["ORPHAN"] = str(orphan) + subprocess.run(["bash", "-c", script, "test", str(work)], env=env, check=True) + self.assertFalse(orphan.exists()) + self.assertFalse(work.exists()) + + +if __name__ == "__main__": + unittest.main()