diff --git a/deploy/README.md b/deploy/README.md index 53c7216..360a8fd 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -12,6 +12,7 @@ does the work: accounts, venv, env files, units, timers, and migrating an older | `zendesk-relay.service` | Always on, `127.0.0.1:8080`: Zendesk's `claude:` note webhooks. | | `session-ops-discord.service` | Always on, `127.0.0.1:8081`: the `/run` and `/versions` slash commands. See [Discord commands](#discord-commands). | | `session-ops-alert@.service` | Every unit's `OnFailure=` backstop; see [session-ops-silence](../docs/jobs/session-ops-silence.md). | +| `session-ops-alert-fallback@.service` | The backstop's own `OnFailure=`: shell and curl, no venv, at most one post per 6 hours. | `session-ops list` shows the jobs; `session-ops run [--dry-run] [-- job arguments]` runs one as its timer does. State is under `/var/lib/session-ops//`, copies of a diff --git a/deploy/session-ops-alert-fallback.sh b/deploy/session-ops-alert-fallback.sh new file mode 100755 index 0000000..ace0ef1 --- /dev/null +++ b/deploy/session-ops-alert-fallback.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Report that session-ops-alert@ itself failed, from session-ops-alert-fallback@.service. +# +# Shell and curl only: the alert runs from the same venv as every job, so a venv that +# stops importing (a Python upgrade under it) fails the jobs and silences their alerts. +# +# Usage: session-ops-alert-fallback.sh +set -u + +unit=${1:?usage: session-ops-alert-fallback.sh } +# One post per window: a broken venv fails every run of every job, each a new alert. +WINDOW_SECONDS=${WINDOW_SECONDS:-21600} +marker="${STATE_DIRECTORY:?}/last-post" + +if [ -z "${ALERT_DISCORD_WEBHOOK_URL:-}" ]; then + echo "No ALERT_DISCORD_WEBHOOK_URL in alerts.env, so $unit is not reported." + exit 0 +fi + +now=$(date +%s) +last=$(cat "$marker" 2>/dev/null || echo 0) +case $last in '' | *[!0-9]*) last=0 ;; esac +if [ $((now - last)) -lt "$WINDOW_SECONDS" ]; then + echo "Posted at $last, under ${WINDOW_SECONDS}s ago, so $unit is not reported again." + exit 0 +fi + +# JSON string content: backslashes and quotes escaped, control characters dropped. +json() { + printf '%s' "$1" | tr -d '\000-\037' | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +line=$(journalctl "_SYSTEMD_UNIT=$unit" -n 1 -q -o cat --no-pager 2>/dev/null | cut -c1-400) +host=$(hostname) +text="🚨 **$(json "$unit")** failed on \`$(json "$host")\`, so failures of session-ops units there are not being reported." +[ -n "$line" ] && text="$text\\n> $(json "$line")" +text="$text\\nA ModuleNotFoundError means the venv no longer imports, usually after a Python upgrade: run \`/opt/session-ops/deploy/install.sh\` as root." +text="$text\\n\`journalctl -u $(json "$unit") -n 50 --no-pager\`" + +payload="{\"content\": \"$text\", \"allowed_mentions\": {\"parse\": []}}" +if ! printf '%s' "$payload" | curl -fsS --max-time 30 -H 'Content-Type: application/json' \ + --data-binary @- "$ALERT_DISCORD_WEBHOOK_URL" >/dev/null; then + echo "Could not post the failure of $unit to Discord." >&2 + exit 1 +fi +printf '%s\n' "$now" >"$marker" diff --git a/deploy/session-ops-alert-fallback@.service b/deploy/session-ops-alert-fallback@.service new file mode 100644 index 0000000..f8876c1 --- /dev/null +++ b/deploy/session-ops-alert-fallback@.service @@ -0,0 +1,18 @@ +[Unit] +Description=Report a failed session-ops-alert to Discord, without the venv +Documentation=https://github.com/session-foundation/session-shared-scripts + +[Service] +Type=oneshot +User=sessionops +Group=sessionops +EnvironmentFile=/etc/session-ops/alerts.env +ExecStart=/opt/session-ops/deploy/session-ops-alert-fallback.sh %i +# Holds the time of the last post, which keeps a broken venv to one post per window. +StateDirectory=session-ops-alert-fallback +SupplementaryGroups=systemd-journal + +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes diff --git a/deploy/session-ops-alert@.service b/deploy/session-ops-alert@.service index aad5bd8..b21bfc8 100644 --- a/deploy/session-ops-alert@.service +++ b/deploy/session-ops-alert@.service @@ -1,6 +1,7 @@ [Unit] Description=Report a failed unit to Discord, unless it already did Documentation=https://github.com/session-foundation/session-shared-scripts +OnFailure=session-ops-alert-fallback@%n.service [Service] Type=oneshot diff --git a/docs/jobs/session-ops-silence.md b/docs/jobs/session-ops-silence.md index 38f82d3..7ea7b71 100644 --- a/docs/jobs/session-ops-silence.md +++ b/docs/jobs/session-ops-silence.md @@ -29,3 +29,9 @@ the first check that found it missing, so a job left disabled alerts once that i than its `max_age_hours`: every scheduled job is meant to run. A job in `jobs.toml` with a `schedule`, or in its `[queue]`, is watched; the `session-ops@` template stamps it. + +**A venv that no longer imports.** The jobs, `session-ops-alert` and this checker all +run from `/opt/session-ops/.venv`, which a Python upgrade under it breaks until +`install.sh` rebuilds it. `session-ops-alert` failing starts +`session-ops-alert-fallback`, a shell script that posts the failed alert, its last +line and that fix to `ALERT_DISCORD_WEBHOOK_URL`, once per 6 hours while it lasts. diff --git a/tests/ops/test_deploy.py b/tests/ops/test_deploy.py index 02dcf2f..bb1a9a5 100644 --- a/tests/ops/test_deploy.py +++ b/tests/ops/test_deploy.py @@ -6,6 +6,7 @@ """ import glob import importlib +import json import os import re import stat @@ -50,11 +51,24 @@ def test_the_template_stamps_each_success_for_the_silence_checker(self): def test_every_unit_reports_its_failure(self): for path in glob.glob(os.path.join(ROOT, "deploy", "*.service")): name = os.path.basename(path) - if name == "session-ops-alert@.service": + if name in ("session-ops-alert@.service", "session-ops-alert-fallback@.service"): continue with self.subTest(unit=name): self.assertIn("OnFailure=session-ops-alert@%n.service", unit_text(name)) + def test_a_failed_alert_is_reported_by_a_unit_that_runs_nothing_from_the_venv(self): + self.assertIn("\nOnFailure=session-ops-alert-fallback@%n.service\n", + unit_text("session-ops-alert@.service")) + fallback = unit_text("session-ops-alert-fallback@.service") + self.assertNotIn(VENV_BIN, fallback) + self.assertNotIn("OnFailure=", fallback) + self.assertIn("\nExecStart=/opt/session-ops/deploy/session-ops-alert-fallback.sh %i\n", + fallback) + script = os.path.join(ROOT, "deploy", "session-ops-alert-fallback.sh") + self.assertTrue(os.stat(script).st_mode & stat.S_IXUSR) + with open(script, encoding="utf-8") as handle: + self.assertNotIn("python", handle.read()) + def test_a_scheduled_job_gets_a_timer_and_an_unscheduled_one_does_not(self): files = units.dropins(registry.load(), registry.load_queue()) for job in registry.load(): @@ -127,5 +141,67 @@ def test_a_clone_anywhere_but_opt_session_ops_is_refused_before_anything_runs(se self.assertIn(f"run the clone at /opt/session-ops, not {ROOT}", done.stderr) +class TestAlertFallback(unittest.TestCase): + UNIT = "session-ops-alert@session-ops@store-versions.service.service" + + def setUp(self): + self.stubs = tempfile.mkdtemp() + self.state = tempfile.mkdtemp() + self.posted = os.path.join(self.stubs, "posted") + self.stub("curl", f'cat > "{self.posted}"\nexit "${{CURL_EXIT:-0}}"\n') + self.stub("journalctl", "echo \"ModuleNotFoundError: No module named 'session_ops'\"\n") + self.stub("hostname", "echo angus\n") + + def stub(self, name, body): + path = os.path.join(self.stubs, name) + with open(path, "w", encoding="utf-8") as handle: + handle.write("#!/bin/sh\n" + body) + os.chmod(path, stat.S_IRWXU) + + def run_script(self, **env): + if os.path.exists(self.posted): + os.remove(self.posted) + return subprocess.run( + ["sh", os.path.join(ROOT, "deploy", "session-ops-alert-fallback.sh"), self.UNIT], + capture_output=True, text=True, check=False, + env={"PATH": f"{self.stubs}:{os.environ['PATH']}", "STATE_DIRECTORY": self.state, + "ALERT_DISCORD_WEBHOOK_URL": "https://discord.invalid/hook", **env}) + + def test_it_posts_the_unit_host_and_last_line_as_json(self): + done = self.run_script() + self.assertEqual(done.returncode, 0, done.stderr) + with open(self.posted, encoding="utf-8") as handle: + payload = json.load(handle) + self.assertEqual(payload["allowed_mentions"], {"parse": []}) + self.assertIn(f"**{self.UNIT}** failed on `angus`", payload["content"]) + self.assertIn("\n> ModuleNotFoundError: No module named 'session_ops'\n", + payload["content"]) + + def test_it_posts_once_per_window(self): + self.assertEqual(self.run_script().returncode, 0) + again = self.run_script() + self.assertEqual(again.returncode, 0) + self.assertFalse(os.path.exists(self.posted)) + self.assertEqual(self.run_script(WINDOW_SECONDS="0").returncode, 0) + self.assertTrue(os.path.exists(self.posted)) + + def test_a_failed_post_fails_the_unit_and_is_retried_next_time(self): + self.assertEqual(self.run_script(CURL_EXIT="22").returncode, 1) + self.assertEqual(self.run_script().returncode, 0) + self.assertTrue(os.path.exists(self.posted)) + + def test_quotes_and_backslashes_in_the_line_stay_valid_json(self): + self.stub("journalctl", "printf '%s\\n' 'said \"no\" at C:\\path'\n") + self.assertEqual(self.run_script().returncode, 0) + with open(self.posted, encoding="utf-8") as handle: + self.assertIn('> said "no" at C:\\path', json.load(handle)["content"]) + + def test_without_a_webhook_it_says_so_and_succeeds(self): + done = self.run_script(ALERT_DISCORD_WEBHOOK_URL="") + self.assertEqual(done.returncode, 0) + self.assertIn("No ALERT_DISCORD_WEBHOOK_URL", done.stdout) + self.assertFalse(os.path.exists(self.posted)) + + if __name__ == "__main__": unittest.main()