From b95bf9bb1395616314b83439e2d97b3bd8794771 Mon Sep 17 00:00:00 2001
From: andrewthomaslee <andrew@external.systems>
Date: Fri, 13 Nov 2026 16:07:14 -0600
Subject: [PATCH] Add hostSpec.system.crashDiagnostics: evidence for unexpected
 crashes

A desktop (kamrui-h1) reset itself with a journal that just stopped: no
shutdown, no panic, nothing in pstore, only the AMD firmware's
"software wrote 0x6 to reset control register 0xCF9" at the next boot.
This module makes the next one explain itself: the kernel log tail goes
to pstore on every reboot and panic, a hung kernel panics and reboots,
systemd feeds the hardware watchdog, journald syncs every 10 s, a
sampler logs load/PSI/memory/temperature, and crash-report writes how
the previous boot ended at every boot.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 documentation/docs/crash-diagnostics.md       |  69 +++++++
 documentation/flake-module.nix                |   1 +
 .../nixosModules/_crash-diagnostics.sh        | 150 ++++++++++++++
 .../nixosModules/crash-diagnostics.nix        | 195 ++++++++++++++++++
 tests/fixtures/nixos.nix                      |   5 +
 vm-tests/crash-diagnostics.nix                |  60 ++++++
 6 files changed, 480 insertions(+)
 create mode 100644 documentation/docs/crash-diagnostics.md
 create mode 100644 flake-parts/nixosModules/_crash-diagnostics.sh
 create mode 100644 flake-parts/nixosModules/crash-diagnostics.nix
 create mode 100644 vm-tests/crash-diagnostics.nix

diff --git a/documentation/docs/crash-diagnostics.md b/documentation/docs/crash-diagnostics.md
new file mode 100644
index 0000000..89104fb
--- /dev/null
+++ b/documentation/docs/crash-diagnostics.md
@@ -0,0 +1,69 @@
+# Crash diagnostics
+
+`hostSpec.system.crashDiagnostics` makes a machine that crashes, resets
+or is killed without a shutdown leave enough evidence to say why. It
+was written after a desktop reset itself with a journal that just
+stopped: no shutdown, no panic, nothing in pstore, only the AMD
+firmware's `software wrote 0x6 to reset control register 0xCF9` at the
+next boot.
+
+```nix
+hostSpec.system.crashDiagnostics.enable = true;
+```
+
+## What it changes
+
+| Setting | Default | Effect |
+|---|---|---|
+| `dmesgOnReboot` | `true` | `printk.always_kmsg_dump=1`: the kernel log tail goes to pstore (EFI variables) on every reboot and power-off the kernel performs, not only on a panic. systemd-pstore moves it to `/var/lib/systemd/pstore` at the next boot. |
+| `panicOnHang` | `true` | `nmi_watchdog=panic softlockup_panic=1`: a hung kernel panics, so its log reaches pstore. |
+| `panicRebootSeconds` | `10` | `kernel.panic`: a panic reboots after 10 s instead of freezing. |
+| `panicOnOops` | `false` | `kernel.panic_on_oops`. |
+| `watchdogTimeout` | `"60s"` | systemd feeds the hardware watchdog; if it stops for this long, the watchdog resets the machine (AMD firmware then reports a watchdog reset). `null` leaves the watchdog alone. |
+| `journalSyncInterval` | `"10s"` | journald `SyncIntervalSec` (5 min by default): less of the journal is lost in a hard reset. |
+| `sampler.enable`, `sampler.interval` | `true`, `"30s"` | `crash-sampler` writes load, pressure (PSI), free memory, swap, the hottest sensor and the biggest process to the journal. |
+| `keepReports` | `50` | Boot reports kept in `/var/log/crash-diagnostics`. |
+
+All kernel and journald values are `mkDefault`, so a machine can still
+set them itself.
+
+## Reading the evidence
+
+At every boot `crash-diagnostics-report.service` writes how the previous
+boot ended to `/var/log/crash-diagnostics/<time>-<boot id>.txt` and logs
+one line (a warning if it ended uncleanly):
+
+```console
+$ journalctl -b -t crash-report
+previous boot ended uncleanly; report: /var/log/crash-diagnostics/...; reset    x86/amd: Previous system reset reason [0x00080800]: ...
+$ crash-report          # the previous boot, on demand
+$ crash-report -3       # any boot still in the journal
+$ crash-report --list   # saved reports
+```
+
+A report has the boot's times and the gap to the next boot, a verdict
+(clean if logind or systemd logged a requested shutdown), the firmware's
+reset reason (AMD CPUs only), kernel messages counted by class (panic,
+lockup, hardware, OOM, thermal, GPU, storage, filesystem, ACPI,
+watchdog), the last samples, the last journal lines, and the pstore
+tail.
+
+## Reading a verdict
+
+- **Clean, nothing else:** someone or something asked for the shutdown;
+  the journal line before it says who. On AMD the reset reason of a
+  normal power-off reads `ACPI power state transition occurred`.
+- **Unclean, pstore has a panic:** a kernel crash; the trace is there.
+- **Unclean, pstore ends in `reboot: Restarting system`:** the kernel
+  rebooted on purpose without systemd (`reboot -ff`, a kernel restart
+  path); the lines before it show what ran last.
+- **Unclean, a watchdog reset reason:** userspace or the kernel hung for
+  longer than `watchdogTimeout`.
+- **Unclean, nothing in pstore, no watchdog:** the kernel never got to
+  write anything: firmware, power or a hardware fault. Check the last
+  samples for temperature and load.
+
+## Not included
+
+kdump (`boot.crashDump`) boots a second kernel that waits in a rescue
+shell, which an unattended desktop cannot use.
diff --git a/documentation/flake-module.nix b/documentation/flake-module.nix
index 97222da..d4f4aa7 100644
--- a/documentation/flake-module.nix
+++ b/documentation/flake-module.nix
@@ -8,6 +8,7 @@
       site_name = "modules";
       nav = [
         {Home = "index.md";}
+        {"Crash diagnostics" = "crash-diagnostics.md";}
         {
           Odoo = [
             {"CEO: Odoo" = "ceo-odoo.md";}
diff --git a/flake-parts/nixosModules/_crash-diagnostics.sh b/flake-parts/nixosModules/_crash-diagnostics.sh
new file mode 100644
index 0000000..3d7c223
--- /dev/null
+++ b/flake-parts/nixosModules/_crash-diagnostics.sh
@@ -0,0 +1,150 @@
+# shellcheck shell=bash
+# crash-report and crash-sampler (crash-diagnostics.nix). Both only read:
+# the journal, /proc, /sys and the pstore archive.
+
+die() {
+  printf 'crash-report: %s\n' "$*" >&2
+  exit 1
+}
+
+# No matching entries is an answer here, not an error.
+j() { journalctl --no-pager -q "$@" || true; }
+
+# logind logs this message id (SD_MESSAGE_SHUTDOWN) when a shutdown or
+# reboot is requested; systemd (PID 1) then reaches its shutdown targets.
+# A boot with neither ended without one: a crash, reset or power loss.
+shutdown_requested() {
+  j -b "$1" MESSAGE_ID=98268866d1d54a499c4e98921d93bc40 -o cat -n 1
+  j -b "$1" _PID=1 -o cat -g '^Reached target (System )?(Power-Off|Reboot|Halt|Kexec)' -n 1
+}
+
+# Kernel log classes worth counting: label|extended regex.
+patterns=(
+  "panic or oops|Kernel panic|Oops|BUG:|general protection fault|RIP: "
+  "lockup or hung task|soft lockup|hard LOCKUP|blocked for more than|rcu_sched self-detected stall|rcu: INFO"
+  "hardware error|Machine check|mce: \[|[Hh]ardware [Ee]rror|EDAC .*(CE|UE|error)"
+  "out of memory|Out of memory|oom-kill|Killed process"
+  "thermal|critical temperature|temperature above threshold|thermal.*shutdown|Package temperature"
+  "GPU|amdgpu.*(timeout|GPU reset|hang|page fault)|GPU reset|i915.*(hang|reset)|NVRM: Xid"
+  "storage|I/O error|nvme.*(timeout|reset|abort)|ata[0-9].*(failed|error)|blk_update_request"
+  "filesystem|BTRFS (error|warning|critical)|EXT4-fs error|XFS.*(error|corrupt)|FILE CORRUPTED|csum failed"
+  "power or ACPI|ACPI (Error|BIOS Error)|PM: .*(fail|error)"
+  "watchdog|watchdog did not stop|[Ww]atchdog (timeout|expired)|Watchdog detected hard LOCKUP"
+)
+
+# report OFFSET: how boot OFFSET (-1 = the previous one) ended.
+report() {
+  local offset=$1 boots boot next first last next_first marker line label regex count
+  boots=$(journalctl --list-boots -o json --no-pager)
+  boot=$(jq -c --argjson o "$offset" '.[] | select(.index == $o)' <<<"$boots")
+  [ -n "$boot" ] || die "boot $offset is not in the journal"
+  next=$((offset + 1))
+  first=$(jq -r '.first_entry / 1000000 | floor' <<<"$boot")
+  last=$(jq -r '.last_entry / 1000000 | floor' <<<"$boot")
+  next_first=$(jq -r --argjson o "$next" '.[] | select(.index == $o) | .first_entry / 1000000 | floor' <<<"$boots")
+
+  printf '== boot %s (%s)\n' "$offset" "$(jq -r .boot_id <<<"$boot")"
+  printf 'started   %s\n' "$(date -d "@$first" '+%F %T %Z')"
+  printf 'last log  %s\n' "$(date -d "@$last" '+%F %T %Z')"
+  if [ -n "$next_first" ]; then
+    printf 'next boot %s (%s s later)\n' "$(date -d "@$next_first" '+%F %T %Z')" "$((next_first - last))"
+  fi
+
+  marker=$(shutdown_requested "$offset" | head -n 1)
+  if [ -n "$marker" ]; then
+    printf 'verdict   clean: %s\n' "$marker"
+  else
+    printf 'verdict   UNCLEAN: the journal stops without a requested shutdown (crash, reset, power loss or a forced reboot)\n'
+  fi
+
+  if [ -n "$next_first" ]; then
+    # AMD firmware records why the machine last reset; the next boot's
+    # kernel prints it (x86/amd: Previous system reset reason ...).
+    line=$(j -b "$next" -k -o cat -g 'reset reason' | head -n 1)
+    printf 'reset     %s\n' "${line:-not reported (only AMD CPUs report a reset reason)}"
+  fi
+
+  printf '\n== kernel messages by class (boot %s)\n' "$offset"
+  local klog
+  klog=$(j -b "$offset" -k -o short-iso)
+  for line in "${patterns[@]}"; do
+    label=${line%%|*}
+    regex=${line#*|}
+    count=$(grep -cE -- "$regex" <<<"$klog" || true)
+    [ "$count" -eq 0 ] && continue
+    printf '%-20s %5d  last: %s\n' "$label" "$count" "$(grep -E -- "$regex" <<<"$klog" | tail -n 1 | cut -c1-160)"
+  done
+  count=$(j -b "$offset" -o cat -t systemd-oomd -g 'Killed' | wc -l)
+  [ "$count" -gt 0 ] && printf '%-20s %5d\n' "systemd-oomd kills" "$count"
+
+  printf '\n== last samples (crash-sampler)\n'
+  j -b "$offset" -t crash-sampler -o short-iso -n 6 || true
+
+  printf '\n== last 30 journal lines\n'
+  j -b "$offset" -o short-iso -n 30 | cut -c1-220
+
+  # systemd-pstore moves what the kernel left in pstore (the kernel log
+  # tail of a panic, or of every reboot with printk.always_kmsg_dump)
+  # into /var/lib/systemd/pstore at the next boot.
+  printf '\n== pstore (kernel log tail saved by the firmware)\n'
+  local found=
+  if [ -n "$next_first" ] && [ -d /var/lib/systemd/pstore ]; then
+    while IFS= read -r -d '' f; do
+      found=1
+      printf -- '-- %s\n' "$f"
+      tail -n 60 "$f"
+    done < <(find /var/lib/systemd/pstore -type f -newermt "@$((next_first - 60))" -print0)
+  fi
+  [ -n "$found" ] || printf 'nothing archived for this boot\n'
+}
+
+# At boot: write the previous boot's report to the log directory, say
+# in the journal whether it ended cleanly, and keep the newest reports.
+boot_report() {
+  local dir=$1 keep=$2 id file verdict
+  journalctl --list-boots -o json --no-pager | jq -e 'any(.[]; .index == -1)' >/dev/null || exit 0
+  id=$(journalctl --list-boots -o json --no-pager | jq -r '.[] | select(.index == -1) | .boot_id')
+  install -d -m 0750 "$dir"
+  file="$dir/$(date '+%Y%m%d-%H%M%S')-$id.txt"
+  report -1 >"$file"
+  verdict=$(grep -m1 '^verdict' "$file" || true)
+  if [[ $verdict == *UNCLEAN* ]]; then
+    printf '<4>previous boot ended uncleanly; report: %s; %s\n' "$file" "$(grep -m1 '^reset' "$file" || true)"
+  else
+    printf 'previous boot ended cleanly; report: %s\n' "$file"
+  fi
+  find "$dir" -maxdepth 1 -name '*.txt' -printf '%T@ %p\n' | sort -rn | tail -n "+$((keep + 1))" | cut -d' ' -f2- | xargs -r rm -f --
+}
+
+# One line of machine health for the journal, so the minutes before a
+# crash show load, pressure, memory and temperature.
+sample() {
+  local load psi="" r avail swap_total swap_free t max=0 max_name="" top
+  read -r load _ </proc/loadavg
+  for r in cpu memory io; do
+    [ -r "/proc/pressure/$r" ] || continue
+    psi+=" $r=$(awk '/^some/ { sub("avg10=", "", $2); print $2 }' "/proc/pressure/$r")"
+  done
+  avail=$(awk '/^MemAvailable/ { printf "%.1fG", $2 / 1048576 }' /proc/meminfo)
+  swap_total=$(awk '/^SwapTotal/ { print $2 }' /proc/meminfo)
+  swap_free=$(awk '/^SwapFree/ { print $2 }' /proc/meminfo)
+  for t in /sys/class/hwmon/hwmon*/temp*_input; do
+    [ -r "$t" ] || continue
+    r=$(cat "$t" 2>/dev/null) || continue
+    if [ "$r" -gt "$max" ]; then
+      max=$r
+      max_name=$(cat "${t%/*}/name" 2>/dev/null || echo "?")
+    fi
+  done
+  top=$(ps -eo rss=,comm= --sort=-rss | awk 'NR == 1 { printf "%s:%.1fG", $2, $1 / 1048576 }')
+  printf 'load=%s psi(some avg10):%s mem_avail=%s swap_used=%.1fG temp_max=%dC(%s) top_rss=%s\n' \
+    "$load" "${psi:- n/a}" "$avail" "$(awk -v t="$swap_total" -v f="$swap_free" 'BEGIN { print (t - f) / 1048576 }')" \
+    "$((max / 1000))" "${max_name:-none}" "$top"
+}
+
+usage() {
+  cat <<'EOF'
+usage: crash-report [OFFSET]   how boot OFFSET ended (default -1, the previous boot)
+       crash-report --list     saved boot reports, newest first
+EOF
+}
diff --git a/flake-parts/nixosModules/crash-diagnostics.nix b/flake-parts/nixosModules/crash-diagnostics.nix
new file mode 100644
index 0000000..36247de
--- /dev/null
+++ b/flake-parts/nixosModules/crash-diagnostics.nix
@@ -0,0 +1,195 @@
+# Evidence for unexpected crashes, resets and machine kills. A desktop
+# that resets on its own usually leaves a journal that just stops: no
+# shutdown, no panic, nothing in pstore. This module makes the next one
+# explain itself:
+#
+# - The kernel log tail goes to pstore (EFI variables) on every panic
+#   and, with dmesgOnReboot, on every reboot the kernel performs, so a
+#   reset that never reached the journal still leaves its last kernel
+#   lines. systemd-pstore moves them to /var/lib/systemd/pstore at the
+#   next boot.
+# - A hung kernel panics instead of freezing (panicOnHang), and a panic
+#   reboots after panicRebootSeconds, so the hang reaches pstore and the
+#   machine comes back.
+# - systemd feeds the hardware watchdog (watchdogTimeout): a machine
+#   whose userspace stops is reset by it, and AMD firmware reports that
+#   as a watchdog reset at the next boot.
+# - journald syncs every journalSyncInterval instead of every 5 minutes,
+#   so the minutes before a hard reset are on disk.
+# - crash-sampler writes one line of load, pressure, memory and
+#   temperature to the journal every sampler.interval.
+# - At every boot, crash-report writes how the previous boot ended to
+#   /var/log/crash-diagnostics: clean or not, the firmware's reset
+#   reason, kernel errors by class, the last samples and log lines and
+#   the pstore tail. `crash-report [OFFSET]` reads any earlier boot.
+#
+# Not used: boot.crashDump (kdump) boots a second kernel that waits in a
+# rescue shell, which is wrong for an unattended desktop.
+{
+  flake.nixosModules.crash-diagnostics = {
+    config,
+    lib,
+    pkgs,
+    ...
+  }: let
+    cfg = config.hostSpec.system.crashDiagnostics;
+    reportDir = "/var/log/crash-diagnostics";
+
+    crashApp = name: main:
+      pkgs.writeShellApplication {
+        inherit name;
+        runtimeInputs = with pkgs; [coreutils findutils gawk gnugrep jq procps systemd];
+        text = builtins.readFile ./_crash-diagnostics.sh + "\n${main}\n";
+      };
+
+    crashReport = crashApp "crash-report" ''
+      case "''${1:--1}" in
+        -h | --help) usage ;;
+        --list) ls -1t ${reportDir} 2>/dev/null || true ;;
+        --boot) boot_report ${reportDir} ${toString cfg.keepReports} ;;
+        -[0-9]* | 0) report "''${1:--1}" ;;
+        *) usage >&2; exit 2 ;;
+      esac
+    '';
+    crashSampler = crashApp "crash-sampler" "sample";
+  in {
+    options.hostSpec.system.crashDiagnostics = {
+      enable = lib.mkEnableOption ''
+        evidence for unexpected crashes and resets (pstore, panic on hang,
+        hardware watchdog, a boot report of how the previous boot ended);
+        see this file's header
+      '';
+      panicRebootSeconds = lib.mkOption {
+        type = lib.types.ints.unsigned;
+        default = 10;
+        description = ''
+          Seconds a panicked kernel waits before rebooting (kernel.panic).
+          0 leaves it frozen on screen, as the kernel default does.
+        '';
+      };
+      panicOnHang = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Panic on a hard lockup (NMI watchdog) or soft lockup, so a hung
+          kernel writes its log to pstore and reboots instead of freezing
+          with nothing on disk.
+        '';
+      };
+      panicOnOops = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Panic on any kernel oops (kernel.panic_on_oops). Off by default:
+          after most oopses the machine keeps running and the journal
+          already has the trace.
+        '';
+      };
+      dmesgOnReboot = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Save the kernel log tail to pstore on every reboot and power-off
+          the kernel performs, not just on panics
+          (printk.always_kmsg_dump). This is what explains a reboot that
+          no log saw. Each one writes a few EFI variables, which
+          systemd-pstore deletes again at the next boot.
+        '';
+      };
+      watchdogTimeout = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = "60s";
+        example = "2min";
+        description = ''
+          Hardware watchdog timeout systemd keeps fed
+          (systemd.watchdog.runtimeTime). If systemd stops responding for
+          this long, the watchdog resets the machine. null leaves the
+          watchdog alone.
+        '';
+      };
+      journalSyncInterval = lib.mkOption {
+        type = lib.types.str;
+        default = "10s";
+        description = ''
+          How often journald syncs the journal to disk (SyncIntervalSec,
+          5min by default). Lines not synced are lost in a hard reset.
+        '';
+      };
+      sampler = {
+        enable =
+          lib.mkEnableOption "a periodic health line in the journal (crash-sampler)"
+          // {default = true;};
+        interval = lib.mkOption {
+          type = lib.types.str;
+          default = "30s";
+          description = "How often crash-sampler writes a line.";
+        };
+      };
+      keepReports = lib.mkOption {
+        type = lib.types.ints.positive;
+        default = 50;
+        description = "Boot reports to keep in ${reportDir}.";
+      };
+    };
+
+    config = lib.mkIf cfg.enable {
+      environment.systemPackages = [crashReport];
+
+      boot = {
+        kernelParams =
+          lib.optional cfg.dmesgOnReboot "printk.always_kmsg_dump=1"
+          ++ lib.optionals cfg.panicOnHang ["nmi_watchdog=panic" "softlockup_panic=1"];
+        kernel.sysctl = {
+          "kernel.panic" = lib.mkDefault cfg.panicRebootSeconds;
+          "kernel.panic_on_oops" = lib.mkDefault (
+            if cfg.panicOnOops
+            then 1
+            else 0
+          );
+        };
+      };
+
+      services.journald.settings.Journal.SyncIntervalSec = lib.mkDefault cfg.journalSyncInterval;
+
+      systemd = {
+        watchdog.runtimeTime = lib.mkIf (cfg.watchdogTimeout != null) (lib.mkDefault cfg.watchdogTimeout);
+
+        services = {
+          crash-diagnostics-report = {
+            description = "Report how the previous boot ended";
+            wantedBy = ["multi-user.target"];
+            after = ["systemd-journal-flush.service" "systemd-pstore.service"];
+            wants = ["systemd-pstore.service"];
+            serviceConfig = {
+              Type = "oneshot";
+              # Stays active, so `systemctl status` shows it ran this boot.
+              RemainAfterExit = true;
+              ExecStart = "${lib.getExe crashReport} --boot";
+              SyslogIdentifier = "crash-report";
+            };
+          };
+
+          crash-sampler = lib.mkIf cfg.sampler.enable {
+            description = "Write one line of machine health to the journal";
+            serviceConfig = {
+              Type = "oneshot";
+              ExecStart = lib.getExe crashSampler;
+              SyslogIdentifier = "crash-sampler";
+              Nice = 19;
+              IOSchedulingClass = "idle";
+            };
+          };
+        };
+
+        timers.crash-sampler = lib.mkIf cfg.sampler.enable {
+          wantedBy = ["timers.target"];
+          timerConfig = {
+            OnBootSec = cfg.sampler.interval;
+            OnUnitActiveSec = cfg.sampler.interval;
+            AccuracySec = "1s";
+          };
+        };
+      };
+    };
+  };
+}
diff --git a/tests/fixtures/nixos.nix b/tests/fixtures/nixos.nix
index 12fd01b..d0e61f6 100644
--- a/tests/fixtures/nixos.nix
+++ b/tests/fixtures/nixos.nix
@@ -108,6 +108,11 @@
         enable = true;
         filesystems.root.spec = "LABEL=root";
       };
+      crashDiagnostics = {
+        enable = true;
+        panicOnOops = true;
+        sampler.interval = "1min";
+      };
       jovian = {
         enable = true;
         gamingMode = true;
diff --git a/vm-tests/crash-diagnostics.nix b/vm-tests/crash-diagnostics.nix
new file mode 100644
index 0000000..a827d9f
--- /dev/null
+++ b/vm-tests/crash-diagnostics.nix
@@ -0,0 +1,60 @@
+# hostSpec.system.crashDiagnostics: the kernel, journald and the
+# watchdog are set up to leave evidence, crash-sampler writes health
+# lines, and crash-report tells a clean shutdown from a crash at the next
+# boot. The crash is a hard VM reset (machine.crash()), the closest to a
+# machine killed without a shutdown.
+_: {
+  nodes.machine = {
+    virtualisation.qemu.options = ["-device" "i6300esb"];
+    hostSpec.system.crashDiagnostics = {
+      enable = true;
+      sampler.interval = "5s";
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("multi-user.target")
+
+    with subtest("kernel, journald and watchdog settings"):
+        cmdline = machine.succeed("cat /proc/cmdline")
+        for param in ["printk.always_kmsg_dump=1", "nmi_watchdog=panic", "softlockup_panic=1"]:
+            assert param in cmdline, cmdline
+        assert machine.succeed("sysctl -n kernel.panic").strip() == "10"
+        assert machine.succeed("sysctl -n kernel.panic_on_oops").strip() == "0"
+        machine.succeed("grep -qx 'SyncIntervalSec=10s' /etc/systemd/journald.conf")
+        assert machine.succeed("systemctl show -P RuntimeWatchdogUSec").strip() == "1min"
+
+    with subtest("first boot: nothing to report"):
+        machine.wait_for_unit("crash-diagnostics-report.service")
+        machine.succeed("test -z \"$(ls -A /var/log/crash-diagnostics 2>/dev/null)\"")
+
+    with subtest("crash-sampler writes health lines"):
+        machine.wait_until_succeeds("journalctl -t crash-sampler -o cat | grep -q '^load=.*mem_avail=.*temp_max='", timeout=60)
+
+    with subtest("a clean shutdown is reported clean"):
+        machine.shutdown()
+        machine.start()
+        machine.wait_for_unit("crash-diagnostics-report.service")
+        report = machine.succeed("cat /var/log/crash-diagnostics/*.txt")
+        assert "verdict   clean" in report, report
+        machine.succeed("journalctl -b -t crash-report -o cat | grep -q 'ended cleanly'")
+
+    with subtest("a hard reset is reported unclean"):
+        machine.wait_until_succeeds("journalctl -b -t crash-sampler -o cat | grep -q '^load='", timeout=60)
+        # Only what reached the disk survives a reset; sync, as the
+        # interval would within 10 seconds.
+        machine.succeed("journalctl --sync")
+        machine.crash()
+        machine.start()
+        machine.wait_for_unit("crash-diagnostics-report.service")
+        machine.succeed("journalctl -b -t crash-report -o cat | grep -q 'ended uncleanly'")
+        latest = machine.succeed("ls -1t /var/log/crash-diagnostics | head -n 1").strip()
+        report = machine.succeed(f"cat /var/log/crash-diagnostics/{latest}")
+        assert "verdict   UNCLEAN" in report, report
+        assert "crash-sampler" in report, report
+        # crash-report reads any boot on demand.
+        assert "verdict   UNCLEAN" in machine.succeed("crash-report -1")
+        assert "verdict   clean" in machine.succeed("crash-report -2")
+        assert machine.succeed("crash-report --list").count(".txt") == 2
+  '';
+}