fix(alerts): report a failed session-ops-alert without the venv

deploy/README.md

@@ -12,6 +12,7 @@ does the work: accounts, venv, env files, units, timers, and migrating an older
 | `zendesk-relay.service` | Always on, `127.0.0.1:8080`: Zendesk's `claude:` note webhooks. |
 | `session-ops-discord.service` | Always on, `127.0.0.1:8081`: the `/run` and `/versions` slash commands. See [Discord commands](#discord-commands). |
 | `session-ops-alert@.service` | Every unit's `OnFailure=` backstop; see [session-ops-silence](../docs/jobs/session-ops-silence.md). |
+| `session-ops-alert-fallback@.service` | The backstop's own `OnFailure=`: shell and curl, no venv, at most one post per 6 hours. |
 
 `session-ops list` shows the jobs; `session-ops run <job> [--dry-run] [-- job arguments]`
 runs one as its timer does. State is under `/var/lib/session-ops/<job>/`, copies of a

deploy/session-ops-alert-fallback.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+# Report that session-ops-alert@ itself failed, from session-ops-alert-fallback@.service.
+#
+# Shell and curl only: the alert runs from the same venv as every job, so a venv that
+# stops importing (a Python upgrade under it) fails the jobs and silences their alerts.
+#
+# Usage: session-ops-alert-fallback.sh <failed unit>
+set -u
+
+unit=${1:?usage: session-ops-alert-fallback.sh <unit>}
+# One post per window: a broken venv fails every run of every job, each a new alert.
+WINDOW_SECONDS=${WINDOW_SECONDS:-21600}
+marker="${STATE_DIRECTORY:?}/last-post"
+
+if [ -z "${ALERT_DISCORD_WEBHOOK_URL:-}" ]; then
+    echo "No ALERT_DISCORD_WEBHOOK_URL in alerts.env, so $unit is not reported."
+    exit 0
+fi
+
+now=$(date +%s)
+last=$(cat "$marker" 2>/dev/null || echo 0)
+case $last in '' | *[!0-9]*) last=0 ;; esac
+if [ $((now - last)) -lt "$WINDOW_SECONDS" ]; then
+    echo "Posted at $last, under ${WINDOW_SECONDS}s ago, so $unit is not reported again."
+    exit 0
+fi
+
+# JSON string content: backslashes and quotes escaped, control characters dropped.
+json() {
+    printf '%s' "$1" | tr -d '\000-\037' | sed 's/\\/\\\\/g; s/"/\\"/g'
+}
+
+line=$(journalctl "_SYSTEMD_UNIT=$unit" -n 1 -q -o cat --no-pager 2>/dev/null | cut -c1-400)
+host=$(hostname)
+text="🚨 **$(json "$unit")** failed on \`$(json "$host")\`, so failures of session-ops units there are not being reported."
+[ -n "$line" ] && text="$text\\n> $(json "$line")"
+text="$text\\nA ModuleNotFoundError means the venv no longer imports, usually after a Python upgrade: run \`/opt/session-ops/deploy/install.sh\` as root."
+text="$text\\n\`journalctl -u $(json "$unit") -n 50 --no-pager\`"
+
+payload="{\"content\": \"$text\", \"allowed_mentions\": {\"parse\": []}}"
+if ! printf '%s' "$payload" | curl -fsS --max-time 30 -H 'Content-Type: application/json' \
+    --data-binary @- "$ALERT_DISCORD_WEBHOOK_URL" >/dev/null; then
+    echo "Could not post the failure of $unit to Discord." >&2
+    exit 1
+fi
+printf '%s\n' "$now" >"$marker"

deploy/session-ops-alert-fallback@.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Report a failed session-ops-alert to Discord, without the venv
+Documentation=https://github.com/session-foundation/session-shared-scripts
+
+[Service]
+Type=oneshot
+User=sessionops
+Group=sessionops
+EnvironmentFile=/etc/session-ops/alerts.env
+ExecStart=/opt/session-ops/deploy/session-ops-alert-fallback.sh %i
+# Holds the time of the last post, which keeps a broken venv to one post per window.
+StateDirectory=session-ops-alert-fallback
+SupplementaryGroups=systemd-journal
+
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes

deploy/session-ops-alert@.service

@@ -1,6 +1,7 @@
 [Unit]
 Description=Report a failed unit to Discord, unless it already did
 Documentation=https://github.com/session-foundation/session-shared-scripts
+OnFailure=session-ops-alert-fallback@%n.service
 
 [Service]
 Type=oneshot

docs/jobs/session-ops-silence.md

@@ -29,3 +29,9 @@ the first check that found it missing, so a job left disabled alerts once that i
 than its `max_age_hours`: every scheduled job is meant to run.
 
 A job in `jobs.toml` with a `schedule`, or in its `[queue]`, is watched; the `session-ops@` template stamps it.
+
+**A venv that no longer imports.** The jobs, `session-ops-alert` and this checker all
+run from `/opt/session-ops/.venv`, which a Python upgrade under it breaks until
+`install.sh` rebuilds it. `session-ops-alert` failing starts
+`session-ops-alert-fallback`, a shell script that posts the failed alert, its last
+line and that fix to `ALERT_DISCORD_WEBHOOK_URL`, once per 6 hours while it lasts.

tests/ops/test_deploy.py

@@ -6,6 +6,7 @@
 """
 import glob
 import importlib
+import json
 import os
 import re
 import stat
@@ -50,11 +51,24 @@ def test_the_template_stamps_each_success_for_the_silence_checker(self):
     def test_every_unit_reports_its_failure(self):
         for path in glob.glob(os.path.join(ROOT, "deploy", "*.service")):
             name = os.path.basename(path)
-            if name == "session-ops-alert@.service":
+            if name in ("session-ops-alert@.service", "session-ops-alert-fallback@.service"):
                 continue
             with self.subTest(unit=name):
                 self.assertIn("OnFailure=session-ops-alert@%n.service", unit_text(name))
 
+    def test_a_failed_alert_is_reported_by_a_unit_that_runs_nothing_from_the_venv(self):
+        self.assertIn("\nOnFailure=session-ops-alert-fallback@%n.service\n",
+                      unit_text("session-ops-alert@.service"))
+        fallback = unit_text("session-ops-alert-fallback@.service")
+        self.assertNotIn(VENV_BIN, fallback)
+        self.assertNotIn("OnFailure=", fallback)
+        self.assertIn("\nExecStart=/opt/session-ops/deploy/session-ops-alert-fallback.sh %i\n",
+                      fallback)
+        script = os.path.join(ROOT, "deploy", "session-ops-alert-fallback.sh")
+        self.assertTrue(os.stat(script).st_mode & stat.S_IXUSR)
+        with open(script, encoding="utf-8") as handle:
+            self.assertNotIn("python", handle.read())
+
     def test_a_scheduled_job_gets_a_timer_and_an_unscheduled_one_does_not(self):
         files = units.dropins(registry.load(), registry.load_queue())
         for job in registry.load():
@@ -127,5 +141,67 @@ def test_a_clone_anywhere_but_opt_session_ops_is_refused_before_anything_runs(se
         self.assertIn(f"run the clone at /opt/session-ops, not {ROOT}", done.stderr)
 
 
+class TestAlertFallback(unittest.TestCase):
+    UNIT = "session-ops-alert@session-ops@store-versions.service.service"
+
+    def setUp(self):
+        self.stubs = tempfile.mkdtemp()
+        self.state = tempfile.mkdtemp()
+        self.posted = os.path.join(self.stubs, "posted")
+        self.stub("curl", f'cat > "{self.posted}"\nexit "${{CURL_EXIT:-0}}"\n')
+        self.stub("journalctl", "echo \"ModuleNotFoundError: No module named 'session_ops'\"\n")
+        self.stub("hostname", "echo angus\n")
+
+    def stub(self, name, body):
+        path = os.path.join(self.stubs, name)
+        with open(path, "w", encoding="utf-8") as handle:
+            handle.write("#!/bin/sh\n" + body)
+        os.chmod(path, stat.S_IRWXU)
+
+    def run_script(self, **env):
+        if os.path.exists(self.posted):
+            os.remove(self.posted)
+        return subprocess.run(
+            ["sh", os.path.join(ROOT, "deploy", "session-ops-alert-fallback.sh"), self.UNIT],
+            capture_output=True, text=True, check=False,
+            env={"PATH": f"{self.stubs}:{os.environ['PATH']}", "STATE_DIRECTORY": self.state,
+                 "ALERT_DISCORD_WEBHOOK_URL": "https://discord.invalid/hook", **env})
+
+    def test_it_posts_the_unit_host_and_last_line_as_json(self):
+        done = self.run_script()
+        self.assertEqual(done.returncode, 0, done.stderr)
+        with open(self.posted, encoding="utf-8") as handle:
+            payload = json.load(handle)
+        self.assertEqual(payload["allowed_mentions"], {"parse": []})
+        self.assertIn(f"**{self.UNIT}** failed on `angus`", payload["content"])
+        self.assertIn("\n> ModuleNotFoundError: No module named 'session_ops'\n",
+                      payload["content"])
+
+    def test_it_posts_once_per_window(self):
+        self.assertEqual(self.run_script().returncode, 0)
+        again = self.run_script()
+        self.assertEqual(again.returncode, 0)
+        self.assertFalse(os.path.exists(self.posted))
+        self.assertEqual(self.run_script(WINDOW_SECONDS="0").returncode, 0)
+        self.assertTrue(os.path.exists(self.posted))
+
+    def test_a_failed_post_fails_the_unit_and_is_retried_next_time(self):
+        self.assertEqual(self.run_script(CURL_EXIT="22").returncode, 1)
+        self.assertEqual(self.run_script().returncode, 0)
+        self.assertTrue(os.path.exists(self.posted))
+
+    def test_quotes_and_backslashes_in_the_line_stay_valid_json(self):
+        self.stub("journalctl", "printf '%s\\n' 'said \"no\" at C:\\path'\n")
+        self.assertEqual(self.run_script().returncode, 0)
+        with open(self.posted, encoding="utf-8") as handle:
+            self.assertIn('> said "no" at C:\\path', json.load(handle)["content"])
+
+    def test_without_a_webhook_it_says_so_and_succeeds(self):
+        done = self.run_script(ALERT_DISCORD_WEBHOOK_URL="")
+        self.assertEqual(done.returncode, 0)
+        self.assertIn("No ALERT_DISCORD_WEBHOOK_URL", done.stdout)
+        self.assertFalse(os.path.exists(self.posted))
+
+
 if __name__ == "__main__":
     unittest.main()