From d201550de72edfaf6ebcb0955a2c1b9b04f17ef6 Mon Sep 17 00:00:00 2001
From: andrewthomaslee <andrew@external.systems>
Date: Thu, 12 Nov 2026 13:35:31 -0600
Subject: [PATCH] odoo: reset copies, snapshots, Mailpit, apps, restarts and
 memory caps

Testing usability and stability for the Odoo test instances:

1. Restart=on-failure (10 s, at most 4 in 15 min); the scripts clear the
   start limit before deliberate starts.
2. MemoryMax per instance (default 4G) and Odoo's per-worker memory and
   time limits, max_cron_threads = 1.
3. Reset copies: a newly created database is saved as a PostgreSQL
   template (odoo-<name>@golden) plus its files; odoo-reset restores it
   in seconds and rebuilds it from scratch (--clean, or when the
   package, addons, demo data, apps or language changed).
4. odoo-snapshot save/restore/list/rm: named states, same mechanism.
5. `modules` (apps at creation, installed when added later) and
   `language`.
6. Mailpit at https://mail.<baseDomain> (and on the LAN): neutralize
   every new database, replace neutralize's blocking mail server with
   one for Mailpit, keep the mail queue on, and give mail without a
   sender <name>@<baseDomain>.
7. A new package or addons runs `-u all` on the next start.
8. LAN vhosts listen on every address: no lan.addresses (breaking), no
   ip_nonlocal_bind; http://<IP>/hosts.txt (nginx fills in the address)
   and /ca.crt bootstrap a client; odoo-hosts reads the current address.
   The allow-list for `return` locations goes through a geo map (return
   runs before allow/deny).

Scripts move to flake-parts/nixosModules/_odoo/*.sh. New VM test
odoo-lifecycle; odoo, odoo-reset and odoo-lan extended. Bumps org-pkgs
to external-systems/pkgs#30 (run summary) with sync-locks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 README.md                                     |   5 +-
 devenv.lock                                   |   8 +-
 devenv.yaml                                   |   2 +-
 documentation/docs/odoo-access.md             |  53 +-
 documentation/docs/odoo.md                    | 181 +++-
 flake-parts/nixosModules/_odoo/golden-save.sh |  26 +
 flake-parts/nixosModules/_odoo/lib.sh         | 127 +++
 flake-parts/nixosModules/_odoo/purge.sh       |  80 ++
 flake-parts/nixosModules/_odoo/reset.sh       | 105 ++
 flake-parts/nixosModules/_odoo/snapshot.sh    | 122 +++
 flake-parts/nixosModules/odoo.nix             | 930 ++++++++++--------
 flake.lock                                    |   6 +-
 tests/fixtures/nixos.nix                      |   9 +-
 vm-tests/_odoo-probe.py                       |  52 +-
 vm-tests/odoo-lan.nix                         |  54 +-
 vm-tests/odoo-lifecycle.nix                   | 104 ++
 vm-tests/odoo-public.nix                      |   3 +
 vm-tests/odoo-reset.nix                       | 119 ++-
 vm-tests/odoo.nix                             |  23 +-
 19 files changed, 1463 insertions(+), 546 deletions(-)
 create mode 100644 flake-parts/nixosModules/_odoo/golden-save.sh
 create mode 100644 flake-parts/nixosModules/_odoo/lib.sh
 create mode 100644 flake-parts/nixosModules/_odoo/purge.sh
 create mode 100644 flake-parts/nixosModules/_odoo/reset.sh
 create mode 100644 flake-parts/nixosModules/_odoo/snapshot.sh
 create mode 100644 vm-tests/odoo-lifecycle.nix

diff --git a/README.md b/README.md
index 74a5785..dd2efa3 100644
--- a/README.md
+++ b/README.md
@@ -52,8 +52,9 @@ e.g. `nixosModules.tailscale` or `homeModules.starship`.
   as fixed by external-systems/pkgs `overlays.odoo` (in its
   `overlays.default`): nixpkgs' own odoo19 does not evaluate. It runs
   0..n local Odoo instances for testing at `https://<name>.odoo.localhost`,
-  reset with `odoo-reset`, each optionally exposed to the LAN or (with
-  Let's Encrypt) the internet; see
+  reset in seconds with `odoo-reset`, saved with `odoo-snapshot`, mail
+  caught by Mailpit, each optionally exposed to the LAN or (with Let's
+  Encrypt) the internet; see
   [the Odoo page](https://modules.external.systems/odoo.html).
 - **plasma-manager.** `homeModules.plasma-manager` sets `programs.plasma`;
   add `inputs.plasma-manager.homeModules.plasma-manager` to
diff --git a/devenv.lock b/devenv.lock
index 6cdc35b..f5c6038 100644
--- a/devenv.lock
+++ b/devenv.lock
@@ -199,17 +199,17 @@
         "zen-browser": "zen-browser"
       },
       "locked": {
-        "lastModified": 1794230460,
-        "narHash": "sha256-zQ11S2rIUURH4yuk4vqxFzJ7Uiv58aQxmsud3weDZGI=",
+        "lastModified": 1794509535,
+        "narHash": "sha256-js5L/es+CXbtZn05wil3ReJVXUF9D8cROJaBJQ/thcI=",
         "owner": "external-systems",
         "repo": "pkgs",
-        "rev": "1188419cc1a2f1e9c22cbe3a62e9858b8a92f557",
+        "rev": "5c80ae9badefcbb2633c4d3a93063546eed63b80",
         "type": "github"
       },
       "original": {
         "owner": "external-systems",
         "repo": "pkgs",
-        "rev": "1188419cc1a2f1e9c22cbe3a62e9858b8a92f557",
+        "rev": "5c80ae9badefcbb2633c4d3a93063546eed63b80",
         "type": "github"
       }
     },
diff --git a/devenv.yaml b/devenv.yaml
index a904b7c..cfd56f1 100644
--- a/devenv.yaml
+++ b/devenv.yaml
@@ -6,7 +6,7 @@
 # pkgs ("attribute 'bash' missing").
 inputs:
   org-pkgs:
-    url: github:external-systems/pkgs/1188419cc1a2f1e9c22cbe3a62e9858b8a92f557
+    url: github:external-systems/pkgs/5c80ae9badefcbb2633c4d3a93063546eed63b80
   nixpkgs:
     follows: org-pkgs/nixpkgs
 imports:
diff --git a/documentation/docs/odoo-access.md b/documentation/docs/odoo-access.md
index 7014c2d..ec14fbc 100644
--- a/documentation/docs/odoo-access.md
+++ b/documentation/docs/odoo-access.md
@@ -14,9 +14,8 @@ Each instance can add two more ways in, independently of the others:
 hostSpec.services.odoo = {
   enable = true;
   lan = {
-    addresses = [ /* this machine's LAN IPs, from the values flake */ ];
     interfaces = [ /* e.g. the Wi-Fi or Ethernet interface */ ];
-    allowedClients = [ /* the LAN's CIDRs */ ];
+    allowedClients = [ /* the LAN's CIDRs, from the values flake */ ];
   };
   acme.email = /* only with expose.public */;
   instances = {
@@ -38,8 +37,12 @@ hostSpec.services.odoo = {
 ```
 
 Nothing listens beyond loopback until an instance asks for it. Odoo's
-own ports (18069 and up) stay on 127.0.0.1 in every mode; only nginx
-faces the network.
+own ports (18069 and up) and Mailpit's stay on 127.0.0.1 in every mode;
+only nginx faces the network. It listens on every address, so a new DHCP
+address needs no rebuild or restart; the firewall (`lan.interfaces`) and
+`lan.allowedClients` decide who gets in. With LAN instances, Mailpit is
+on the LAN too (`https://mail.odoo.<host>.home.arpa`;
+`mail.expose.lan = false` keeps it local).
 
 ## Why not `*.odoo.localhost` from the LAN
 
@@ -57,20 +60,21 @@ its line in the hosts file.
 
 ### 1. Get the hosts-file lines
 
-On the Odoo machine:
-
-```sh
-odoo-hosts
-```
+All you need is the Odoo machine's LAN IP address. Open
+`http://<that IP>/hosts.txt` in a browser on the laptop:
 
 ```text
 198.51.100.230  odoo.beelink.home.arpa
-198.51.100.230  blank.odoo.beelink.home.arpa
 198.51.100.230  demo.odoo.beelink.home.arpa
+198.51.100.230  blank.odoo.beelink.home.arpa
+198.51.100.230  mail.odoo.beelink.home.arpa
 ```
 
-The same lines are on the LAN index page. Hosts files have no
-wildcards: one line per name.
+nginx fills in the address you reached it on, so the lines are right
+whatever DHCP handed out. On the Odoo machine, `odoo-hosts` prints the
+same lines for the address the LAN interface has now. Hosts files have
+no wildcards: one line per name. When the machine's address changes,
+fetch the lines again.
 
 ### 2. Add them to the Windows hosts file
 
@@ -95,7 +99,7 @@ instead, for every device on the network at once.
 
 The certificate comes from the machine's own CA. Download it over plain
 HTTP (it is public, and HTTPS is not trusted yet):
-`http://odoo.beelink.home.arpa/ca.crt`. Then, in PowerShell as
+`http://<that IP>/ca.crt` or `http://odoo.beelink.home.arpa/ca.crt`. Then, in PowerShell as
 administrator:
 
 ```powershell
@@ -120,7 +124,9 @@ the machine that ran `clan vars generate`.
 
 ### 4. Open it
 
-`https://odoo.beelink.home.arpa` lists the LAN instances. Log in as
+`https://odoo.beelink.home.arpa` lists the LAN instances and links
+Mailpit (`https://mail.odoo.beelink.home.arpa`), which shows every mail
+the instances sent. Log in as
 `admin` with `clan vars get beelink odoo-passwords/admin-password`.
 
 ### Removing it again
@@ -138,11 +144,16 @@ The same lines go into `/etc/hosts`. Trust the CA with
 
 ### What the LAN side enforces
 
-- nginx listens on `lan.addresses` only, and the firewall opens 80 and
-  443 on `lan.interfaces` only.
-- Clients outside `lan.allowedClients` get 403 for every LAN name.
+- The firewall opens 80 and 443 on `lan.interfaces` only. nginx
+  listens on every address, so set `lan.allowedClients`: without it, any
+  client that reaches the port (over a VPN's trusted interface, say)
+  gets in.
+- Clients outside `lan.allowedClients` get 403 for every LAN name,
+  Mailpit, `hosts.txt` and `ca.crt`.
 - A name that is not a LAN-exposed instance (a local-only one, a
-  `*.odoo.localhost` name, a bare IP) has its TLS handshake refused.
+  `*.odoo.localhost` name, a bare IP over HTTPS) has its TLS handshake
+  refused; plain HTTP to the bare IP answers only `/hosts.txt` and
+  `/ca.crt`.
 - When the CA's names change (first LAN exposure, a new `lan.domain`),
   `clan vars generate` makes a new CA: import the new `ca.crt` on each
   client and delete the old one.
@@ -164,6 +175,10 @@ Before enabling it:
    `security.acme.defaults.email`), from the values flake.
 4. Put the domain in the values flake, not in a module or machine file.
 
+With instances both on the LAN and public, `lan.allowedClients` is
+required: a router forwards internet traffic to the LAN address, and
+the LAN names must not answer it.
+
 What the module adds for a public instance:
 
 - the firewall opens 80 and 443 on every interface;
@@ -192,5 +207,5 @@ odoo-hosts          # LAN hosts-file lines
 
 | Test | Covers |
 | --- | --- |
-| `odoo-lan` | a client pastes the `odoo-hosts` lines, downloads `ca.crt` over HTTP and uses two LAN instances (logins, demo data, websocket, index); a local-only instance is unreachable under any name; Odoo ports closed; a client outside `allowedClients` gets 403; the CA rejects a certificate for another domain |
+| `odoo-lan` | a client that knows only the server's IP fetches `hosts.txt` and `ca.crt` over HTTP and uses two LAN instances and Mailpit (logins, demo data, websocket, index); a local-only instance is unreachable under any name; Odoo ports closed; a client outside `allowedClients` gets 403; the CA rejects a certificate for another domain; a new DHCP address works without a rebuild or restart |
 | `odoo-public` | Let's Encrypt-style issuance over HTTP-01 from pebble; an internet client logs in at the public domain; 404 for the database manager, HSTS, frozen `web.base.url`, login rate limit; a local-only instance is unreachable |
diff --git a/documentation/docs/odoo.md b/documentation/docs/odoo.md
index 60afc4b..04b92d8 100644
--- a/documentation/docs/odoo.md
+++ b/documentation/docs/odoo.md
@@ -1,11 +1,12 @@
 # Odoo for local testing
 
 `hostSpec.services.odoo` runs any number of Odoo instances on one
-machine, for testing only. Each instance has its own PostgreSQL database
-and state, is served over trusted HTTPS at
-`https://<name>.odoo.localhost`, and can be reset to a freshly created
-database (with or without demo data) by one command. Nothing is
-reachable from other machines unless an instance asks for it: see
+machine, for testing only (never production). Each instance has its own
+PostgreSQL database and state, is served over trusted HTTPS at
+`https://<name>.odoo.localhost`, comes with the apps you list, sends its
+mail to a local Mailpit instead of the world, and goes back to a freshly
+created state in seconds. Nothing is reachable from other machines
+unless an instance asks for it: see
 [Reaching Odoo from other machines](odoo-access.md) for the LAN (a
 Windows laptop, say) and the internet (a real domain with Let's
 Encrypt).
@@ -16,37 +17,48 @@ Encrypt).
 hostSpec.services.odoo = {
   enable = true;
   instances = {
-    demo = {};                # https://demo.odoo.localhost, demo data
-    blank.demoData = false;   # https://blank.odoo.localhost, empty
+    demo.modules = [ "sale_management" "crm" ];   # demo data + apps
+    blank.demoData = false;                        # empty database
   };
 };
 ```
 
 Run `clan vars generate <machine>` once after enabling, then deploy.
 Adding or removing an instance afterwards needs only a deploy: every
-instance shares one certificate (for `odoo.localhost` and
-`*.odoo.localhost`) and one pair of passwords.
+instance shares one certificate and one pair of passwords.
 
-`https://odoo.localhost` is an index page linking every instance.
-`instances = {}` is valid: then no Odoo or nginx runs, only PostgreSQL
-(so `odoo-purge` can still clean up).
+| Address | What |
+| --- | --- |
+| `https://odoo.localhost` | index of the instances |
+| `https://<name>.odoo.localhost` | an instance |
+| `https://mail.odoo.localhost` | Mailpit: every mail the instances sent |
+
+`instances = {}` is valid: then no Odoo, nginx or Mailpit runs, only
+PostgreSQL (so `odoo-purge` can still clean up).
 
 | Option (per instance) | Default | Meaning |
 | --- | --- | --- |
-| `demoData` | `true` | load Odoo's demo data when the database is created (first start and every reset) |
+| `demoData` | `true` | Odoo's demo data in the database (at creation and every reset) |
+| `modules` | `[]` | apps installed with the database, e.g. `[ "sale_management" "crm" "stock" "l10n_us" ]`; added later, they are installed on the next start |
+| `language` | `null` | a language loaded at creation, e.g. `"de_DE"` |
+| `neutralize` | `true` | `odoo neutralize` the database (see Mail) |
 | `workers` | `2` | HTTP worker processes; at least 1, so the websocket service (Discuss, live updates) runs |
+| `memoryMax` | `"4G"` | systemd `MemoryMax` for the whole instance; `null` for none |
+| `limits.memorySoft`, `.memoryHard` | `1536`, `2048` | Odoo's per-worker memory limits, MiB |
+| `limits.timeCpu`, `.timeReal` | `600`, `1200` | Odoo's per-request time limits, seconds (generous for imports and reports) |
 | `addons` | `[]` | extra addon packages |
 | `package` | `hostSpec.services.odoo.package` (`pkgs.odoo19`) | Odoo for this instance |
 | `port` | automatic | HTTP port on 127.0.0.1; the websocket port is `port + 1` |
 | `expose.lan` | `false` | also at `https://<name>.odoo.<host>.home.arpa` for the LAN ([details](odoo-access.md)) |
 | `expose.public.enable`, `.domain` | off | also at `https://<domain>` for the internet, Let's Encrypt ([details](odoo-access.md)) |
 
-Instance names are DNS labels: lowercase letters, digits and hyphens,
-starting with a letter. Automatic ports follow the name order from
-`basePort` (18069): with `blank` and `demo`, blank gets 18069/18070 and
-demo 18071/18072. Other machine-wide options: `baseDomain`
-(`odoo.localhost`), `package`, `trustCertificate`, `lan.*` and
-`acme.email` for exposure, and the secret options below.
+Instance names are DNS labels of at most 24 characters: lowercase
+letters, digits and hyphens, starting with a letter, and not `mail`.
+Automatic ports follow the name order from `basePort` (18069): with
+`blank` and `demo`, blank gets 18069/18070 and demo 18071/18072. Other
+machine-wide options: `baseDomain` (`odoo.localhost`), `package`,
+`mail.*`, `trustCertificate`, `lan.*` and `acme.email` for exposure, and
+the secret options below.
 
 ## What each instance gets
 
@@ -58,10 +70,12 @@ For an instance `<name>`, everything is called `odoo-<name>`:
 | system user, PostgreSQL role and database | `odoo-<name>` (peer auth over the unix socket, no database password) |
 | state: filestore, sessions, markers | `/var/lib/private/odoo-<name>/data` |
 | config with the master password (0600) | `/run/odoo-<name>/odoo.cfg` |
+| reset copy | database `odoo-<name>@golden`, files in `/var/lib/odoo-golden/<name>` |
+| snapshots | databases `odoo-<name>@snap-<tag>`, files in `/var/lib/odoo-snapshots/<name>` |
 | URL | `https://<name>.odoo.localhost` |
 
-Odoo and nginx bind loopback only, no firewall port is opened and
-PostgreSQL has no TCP listener. Odoo's database manager is off.
+Odoo, nginx and Mailpit bind loopback only, no firewall port is opened
+and PostgreSQL has no TCP listener. Odoo's database manager is off.
 
 ## Logging in
 
@@ -76,51 +90,97 @@ The master password (`odoo-passwords/master-password`) never reaches the
 Nix store. A password changed in the Odoo UI survives restarts and
 reboots; a reset sets the provisioned one again.
 
-## HTTPS in Firefox
+## Mail: Mailpit and neutralized databases
+
+Test data has real-looking addresses, and a test instance must never
+mail them. Two things make sure of it:
+
+- **Every instance sends to Mailpit** (`127.0.0.1:1025`), which keeps the
+  mail and shows it at `https://mail.odoo.localhost`: open it to read
+  what Odoo sent (invoices, password resets, follow-ups).
+- **Every new database is neutralized** (`odoo neutralize`): mail
+  servers configured in the database, fetchmail, payment providers,
+  outgoing integrations and scheduled actions are switched off, and Odoo
+  shows a "neutralized database" banner. Neutralize also adds a mail
+  server with the host `invalid` that blocks all mail; the module
+  replaces it with one named `Mailpit` and turns the mail queue back on,
+  so mail is sent, to Mailpit only. This runs again after new apps are
+  installed. Mail without a sender gets `<name>@odoo.localhost`.
+
+`mail.enable = false` drops Mailpit and keeps neutralize's blocking mail
+server: mail goes nowhere.
 
-A CA made by clan vars (`odoo-local-ca`) signs the certificate. It is
-name-constrained to `odoo.localhost`, the LAN domain
-(`odoo.<host>.home.arpa`) and their subdomains, so trusting it cannot
-vouch for any other site, and its private key never leaves the
-machine that ran `clan vars generate`. With `trustCertificate = true`
-(the default) the CA is trusted system wide and installed into every
-Firefox profile through the enterprise policy
-`/etc/firefox/policies/policies.json`, including a home-manager Firefox.
-Firefox resolves `*.localhost` to loopback itself; `networking.hosts`
-does it for everything else. Other browsers need the CA imported by
-hand: `ca.crt` is in `clan vars get <machine> odoo-local-ca/ca.crt`.
+## Starting, crashing, updating
+
+Each start runs these steps before Odoo serves:
+
+1. The first start (and a reset from scratch) creates the database:
+   base, the `modules`, demo data if asked, the `language`; then
+   neutralizes it and saves it as the instance's **reset copy**.
+2. When the Odoo package or addons changed since the last start (a
+   nixpkgs bump), every module is updated (`odoo -u all`), keeping the
+   data. When apps were added to `modules`, they are installed. Removing
+   an app from `modules` does not uninstall it; a reset does.
+3. The provisioned admin password is applied when the secret changed.
+
+A crashed or memory-killed instance restarts after 10 s; after four
+failures in 15 minutes it stays down (`systemctl status odoo-<name>`,
+`journalctl -u odoo-<name>`). `memoryMax` caps the whole instance, so a
+runaway import or report cannot starve the machine or PostgreSQL.
 
 ## Resetting an instance: odoo-reset
 
 `odoo-reset` puts an instance back to how it was when it was first
-enabled: a freshly created database, with demo data or without it as the
-instance is configured, no uploaded files, and the provisioned admin
-password.
+created: every record, file and setting made since is gone, the demo
+data and apps are back as configured, and the provisioned admin
+password applies.
 
 ```sh
-odoo-reset --list               # name, URL, demo or not, unit state
+odoo-reset --list               # name, URL, data, state, reset copy, exposure
 sudo odoo-reset demo            # asks, then resets demo
 sudo odoo-reset demo blank      # several
 sudo odoo-reset --all           # every instance
 sudo odoo-reset --yes demo      # no question (scripts, no terminal)
+sudo odoo-reset --clean demo    # create the database from scratch
 ```
 
-For each instance it stops the unit, drops and recreates the PostgreSQL
-database, deletes the state (filestore, sessions and the markers that
-record the first start and the admin password), starts the unit again
-and waits until Odoo answers. Creating a database with demo data takes a
-few minutes. The certificate, the CA and both passwords do not change.
-An unknown name, `--all` with names, a missing `--yes` without a
-terminal, or running it without root changes nothing.
+A reset restores the **reset copy** (a PostgreSQL template copy of the
+database as first created, plus its files): seconds. The copy is made
+automatically whenever a database is created from scratch. It is
+`stale` in `--list` once the package, addons, demo data, apps or
+language changed; a reset then creates the database from scratch
+(minutes with demo data) and makes a new copy. `--clean` does that on
+purpose. The certificate, the CA and both passwords never change.
+Snapshots survive a reset. An unknown name, `--all` with names, a
+missing `--yes` without a terminal, or running it without root changes
+nothing.
+
+## Saving a state: odoo-snapshot
+
+A snapshot keeps an instance as it is now, to come back to after a
+test: "the state just before step X".
+
+```sh
+sudo odoo-snapshot save demo before-import     # stops demo for a few seconds
+sudo odoo-snapshot list [demo]
+sudo odoo-snapshot restore demo before-import  # asks, then puts it back
+sudo odoo-snapshot rm demo before-import
+```
+
+A restore brings back everything as saved, the admin password included.
+After a package change, the next start updates the restored database
+(`-u all`) as for any other. Tags: lowercase letters, digits, `.`, `_`
+and `-`, at most 24.
 
 ## Removing an instance: odoo-purge
 
 Removing an instance from `instances` and deploying stops it, but its
-database, role and state stay. `odoo-purge` lists and deletes them:
+database, role, state, reset copy and snapshots stay. `odoo-purge` lists
+and deletes them:
 
 ```sh
 sudo odoo-purge                 # list what removed instances left behind
-sudo odoo-purge old-demo        # asks, then deletes database, role and state
+sudo odoo-purge old-demo        # asks, then deletes all of it
 sudo odoo-purge --yes old-demo  # no question
 ```
 
@@ -129,6 +189,18 @@ those) and needs PostgreSQL, which keeps running while the module is
 enabled, even with no instances. After `enable = false`, PostgreSQL is
 no longer started by this module: purge first.
 
+## Troubleshooting
+
+| Symptom | Look at |
+| --- | --- |
+| an instance does not come up | `journalctl -u odoo-<name> -b`: the `pre-start:` lines say which step ran (creating, updating, installing apps, reset copy, admin password) |
+| it stays down after crashes | `systemctl status odoo-<name>` (start limit hit), then `systemctl reset-failed odoo-<name>` and start it |
+| it was killed | `journalctl -u odoo-<name> | grep -i oom`; raise `memoryMax` or lower `workers` |
+| a request times out | Odoo's `limit_time_*` (`limits.timeCpu`, `.timeReal`) |
+| reset takes minutes | `odoo-reset --list` shows the copy `stale` or `none`; the next reset rebuilds it |
+| no mail arrives in Mailpit | `systemctl status mailpit-odoo`; the mail queue cron in Odoo (Settings, Technical, Scheduled Actions) |
+| an app is missing after adding it | the next start installs it: `systemctl restart odoo-<name>` |
+
 ## One-time migration from the single instance
 
 Before this layout the module ran one instance: unit `odoo.service`,
@@ -156,15 +228,17 @@ example on beelink):
     ```
 
 4. Check: `sudo -u postgres psql -l` lists only `odoo-<name>` databases
-   (plus PostgreSQL's own), `odoo-reset --list` shows the new instances
-   as active, and `https://odoo.localhost` lists them.
+   and their `@golden` copies (plus PostgreSQL's own), `odoo-reset --list`
+   shows the new instances as active with a `ready` copy, and
+   `https://odoo.localhost` lists them.
 
 ## Secrets without clan vars
 
 `provisionSecrets = false` turns the clan vars generators off; then set
-`caFile`, `tlsCertFile`, `tlsKeyFile` (a certificate for `baseDomain`
-and `*.baseDomain`), `adminPasswordFile` and `masterPasswordFile`. The VM
-tests do this with build-time fakes.
+`caFile`, `tlsCertFile`, `tlsKeyFile` (a certificate for `baseDomain`,
+`lan.domain` and their wildcards, the key readable by nginx),
+`adminPasswordFile` and `masterPasswordFile`. The VM tests do this with
+build-time fakes.
 
 ## VM tests
 
@@ -173,7 +247,8 @@ builder):
 
 | Test | Covers |
 | --- | --- |
-| `odoo` | two instances (demo and no demo) and a machine with none: trusted HTTPS, index page, logins, isolation between instances, websocket, master password outside the store, loopback only, password changes surviving restarts, reboots not re-initialising |
-| `odoo-reset` | `odoo-reset` for one and all instances (records, files and UI password changes gone, demo data as configured, other instances untouched), `odoo-purge`, and every refusal |
+| `odoo` | two instances (demo and no demo) and a machine with none: trusted HTTPS, index page, logins, isolation between instances, websocket, nginx reloads, master password outside the store, loopback only, a killed instance restarting, memory caps, password changes surviving restarts, reboots not re-initialising |
+| `odoo-reset` | the reset copy (made at first start, restored in seconds, `--clean`, stale), resets of one and all instances, snapshots, `odoo-purge` (copies and snapshots too), and every refusal |
+| `odoo-lifecycle` | apps from `modules`, neutralize, mail landing in Mailpit, then a package bump and an added app: `-u all` and the install keep the data |
 | `odoo-lan`, `odoo-public` | exposure: see [Reaching Odoo from other machines](odoo-access.md) |
 | `odoo-firefox` | a home-manager Firefox loads an instance without a certificate error, and does not get past TLS on an untrusted control site |
diff --git a/flake-parts/nixosModules/_odoo/golden-save.sh b/flake-parts/nixosModules/_odoo/golden-save.sh
new file mode 100644
index 0000000..bab3924
--- /dev/null
+++ b/flake-parts/nixosModules/_odoo/golden-save.sh
@@ -0,0 +1,26 @@
+# shellcheck shell=bash disable=SC2034,SC2154
+# A fragment: hostSpec.services.odoo (odoo.nix, mkScript) sets prog, instances,
+# golden_root and snapshot_root and puts lib.sh first; the built script is
+# linted whole again by writeShellApplication.
+# odoo-golden-save NAME: runs as root in odoo-NAME's pre-start, right
+# after its database was created (the init leaves .golden-pending). Saves
+# that fresh database and its files as the reset copy, so odoo-reset can
+# restore it in seconds instead of creating it again.
+yes=false
+name=$1
+data=$(field "$name" dataDir)
+[ -e "$data/.golden-pending" ] || exit 0
+
+unit=$(field "$name" unit)
+dir="$golden_root/$name"
+echo "pre-start: saving the new database as the reset copy"
+mkdir -p "$dir"
+chmod 700 "$golden_root" "$dir"
+drop_db "$unit@golden"
+# The init has stopped: nothing is connected to the database.
+copy_db "$unit" "$unit@golden" postgres
+freeze_db "$unit@golden"
+rm -f "$data/.golden-pending"
+# The admin password and base URL are applied again after a restore.
+save_data "$data" "$dir/data.tar" --exclude=./.admin-password.sha256 --exclude=./.base-url
+field "$name" goldenKey > "$dir/key"
diff --git a/flake-parts/nixosModules/_odoo/lib.sh b/flake-parts/nixosModules/_odoo/lib.sh
new file mode 100644
index 0000000..0d75a6f
--- /dev/null
+++ b/flake-parts/nixosModules/_odoo/lib.sh
@@ -0,0 +1,127 @@
+# shellcheck shell=bash disable=SC2034,SC2154
+# A fragment: hostSpec.services.odoo (odoo.nix, mkScript) sets prog, instances,
+# golden_root and snapshot_root and puts lib.sh first; the built script is
+# linted whole again by writeShellApplication.
+# Shared by odoo-reset, odoo-snapshot, odoo-purge and odoo-golden-save
+# (hostSpec.services.odoo). The wrapper sets, before this file:
+#   prog           the command name, for messages
+#   instances      JSON list of the configured instances (name, unit,
+#                  stateDir, dataDir, httpPort, url, demoData, goldenKey)
+#   golden_root    where reset copies live: <golden_root>/<name>/{data.tar,key}
+#   snapshot_root  where snapshots live: <snapshot_root>/<name>/<tag>.tar
+# Database names: odoo-<name> (the instance), odoo-<name>@golden (its
+# reset copy), odoo-<name>@snap-<tag> (snapshots). Instance names cannot
+# contain @, so the suffixes never collide with an instance.
+
+die() {
+  echo "$prog: $*" >&2
+  exit 1
+}
+
+need_root() {
+  [ "$(id -u)" -eq 0 ] || die "run as root (sudo $prog ...)"
+}
+
+sql() {
+  runuser -u postgres -- psql -v ON_ERROR_STOP=1 -qtA -d postgres "$@"
+}
+
+configured() {
+  jq -e --arg n "$1" 'any(.[]; .name == $n)' "$instances" >/dev/null
+}
+
+field() {
+  jq -r --arg n "$1" --arg f "$2" '.[] | select(.name == $n) | .[$f]' "$instances"
+}
+
+instance_names() {
+  jq -r '.[].name' "$instances"
+}
+
+db_exists() {
+  [ "$(sql -c "SELECT count(*) FROM pg_database WHERE datname = '$1'")" = 1 ]
+}
+
+# Drops a database, also one frozen as a template.
+drop_db() {
+  if db_exists "$1"; then
+    sql -c "ALTER DATABASE \"$1\" WITH IS_TEMPLATE false ALLOW_CONNECTIONS true" \
+      -c "DROP DATABASE \"$1\" WITH (FORCE)"
+  fi
+}
+
+# A copy nobody connects to and that only serves as a template.
+freeze_db() {
+  sql -c "ALTER DATABASE \"$1\" WITH IS_TEMPLATE true ALLOW_CONNECTIONS false"
+}
+
+# copy_db SOURCE TARGET OWNER: a file-level copy (CREATE DATABASE
+# TEMPLATE), seconds even with demo data. SOURCE must have no connections.
+copy_db() {
+  sql -c "CREATE DATABASE \"$2\" OWNER \"$3\" TEMPLATE \"$1\""
+}
+
+# save_data DATADIR ARCHIVE [tar --exclude ...]: the instance's files
+# (filestore, markers) without sessions.
+save_data() {
+  local data=$1 archive=$2
+  shift 2
+  tar -C "$data" --exclude=./sessions "$@" -cf "$archive.tmp" .
+  mv "$archive.tmp" "$archive"
+}
+
+# restore_data NAME ARCHIVE: replaces the instance's state with ARCHIVE.
+restore_data() {
+  local name=$1 archive=$2 state data unit
+  state=$(field "$name" stateDir)
+  data=$(field "$name" dataDir)
+  unit=$(field "$name" unit)
+  if [ -d "$state" ]; then
+    find "$state" -mindepth 1 -delete
+  fi
+  mkdir -p "$data"
+  tar -C "$data" -xf "$archive"
+  chown -R "$unit:$unit" "$state"
+}
+
+# Starts the unit and waits until Odoo answers on its port.
+start_and_wait() {
+  local name=$1 unit port
+  unit=$(field "$name" unit)
+  port=$(field "$name" httpPort)
+  # A deliberate start: clear the crash-loop limit (StartLimitBurst counts
+  # every start, so repeated resets would otherwise lock the unit out).
+  systemctl reset-failed "$unit.service" 2>/dev/null || true
+  systemctl start "$unit.service"
+  for _ in $(seq 300); do
+    if curl -sf -o /dev/null "http://127.0.0.1:$port/web/login"; then
+      return 0
+    fi
+    sleep 1
+  done
+  die "$name did not answer on port $port; see journalctl -u $unit"
+}
+
+# confirm MESSAGE: asks for "yes" unless $yes is true.
+confirm() {
+  if ! $yes; then
+    [ -t 0 ] || die "not a terminal: pass --yes"
+    echo "$1"
+    read -r -p "Type yes to continue: " answer
+    [ "$answer" = yes ] || die "aborted, nothing changed"
+  fi
+}
+
+# The reset copy of NAME is usable when it exists and was made from the
+# current configuration (package, addons, demo data, apps, language).
+golden_valid() {
+  local name=$1 dir="$golden_root/$1"
+  [ -f "$dir/data.tar" ] && [ -f "$dir/key" ] &&
+    [ "$(cat "$dir/key")" = "$(field "$name" goldenKey)" ] &&
+    db_exists "$(field "$name" unit)@golden"
+}
+
+drop_golden() {
+  drop_db "$(field "$1" unit)@golden"
+  rm -rf "${golden_root:?}/$1"
+}
diff --git a/flake-parts/nixosModules/_odoo/purge.sh b/flake-parts/nixosModules/_odoo/purge.sh
new file mode 100644
index 0000000..9087a06
--- /dev/null
+++ b/flake-parts/nixosModules/_odoo/purge.sh
@@ -0,0 +1,80 @@
+# shellcheck shell=bash disable=SC2034,SC2154
+# A fragment: hostSpec.services.odoo (odoo.nix, mkScript) sets prog, instances,
+# golden_root and snapshot_root and puts lib.sh first; the built script is
+# linted whole again by writeShellApplication.
+usage() {
+  cat <<'EOF'
+Usage: odoo-purge
+       odoo-purge [--yes] NAME...
+
+Without names: lists what removed Odoo instances left behind (a database
+or role odoo-NAME, its reset copy and snapshots, or state under
+/var/lib/private/odoo-NAME, /var/lib/odoo-golden/NAME or
+/var/lib/odoo-snapshots/NAME, for a NAME no longer configured). With
+names: deletes all of it. A configured instance is refused: odoo-reset
+empties those. Runs as root.
+
+  --yes    do not ask for confirmation (needed without a terminal)
+EOF
+}
+
+leftovers() {
+  {
+    # odoo-NAME, odoo-NAME@golden, odoo-NAME@snap-TAG.
+    sql -c "SELECT datname FROM pg_database WHERE datname LIKE 'odoo-%'" | sed -E 's/@.*$//'
+    sql -c "SELECT rolname FROM pg_roles WHERE rolname LIKE 'odoo-%'"
+    for dir in /var/lib/private/odoo-*; do
+      if [ -e "$dir" ]; then basename "$dir"; fi
+    done
+    for dir in "$golden_root"/* "$snapshot_root"/*; do
+      if [ -e "$dir" ]; then echo "odoo-$(basename "$dir")"; fi
+    done
+  } | sed -n 's/^odoo-//p' | sort -u | while read -r name; do
+    configured "$name" || echo "$name"
+  done
+}
+
+yes=false
+selected=()
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --yes | -y) yes=true ;;
+    -h | --help) usage; exit 0 ;;
+    -*) usage >&2; die "unknown option $1" ;;
+    *) selected+=("$1") ;;
+  esac
+  shift
+done
+
+need_root
+sql -c "SELECT 1" >/dev/null || die "PostgreSQL is not reachable"
+
+if [ ${#selected[@]} -eq 0 ]; then
+  found=$(leftovers)
+  if [ -z "$found" ]; then
+    echo "Nothing left behind by removed Odoo instances."
+  else
+    echo "Left behind by removed instances (odoo-purge NAME deletes it):"
+    echo "$found"
+  fi
+  exit 0
+fi
+
+for name in "${selected[@]}"; do
+  [[ "$name" =~ ^[a-z][a-z0-9-]*$ ]] || die "'$name' is not an instance name"
+  if configured "$name"; then
+    die "'$name' is still configured; remove it from hostSpec.services.odoo.instances and deploy first, or use odoo-reset"
+  fi
+done
+
+confirm "This deletes the database, reset copy, snapshots, role and state of: ${selected[*]}"
+
+for name in "${selected[@]}"; do
+  unit="odoo-$name"
+  echo "==> $name: dropping its databases and role $unit"
+  sql -c "SELECT datname FROM pg_database WHERE datname = '$unit' OR datname LIKE '$unit@%'" |
+    while read -r db; do drop_db "$db"; done
+  sql -c "DROP ROLE IF EXISTS \"$unit\""
+  echo "==> $name: deleting its state, reset copy and snapshots"
+  rm -rf "/var/lib/private/$unit" "/var/lib/$unit" "${golden_root:?}/$name" "${snapshot_root:?}/$name"
+done
diff --git a/flake-parts/nixosModules/_odoo/reset.sh b/flake-parts/nixosModules/_odoo/reset.sh
new file mode 100644
index 0000000..1f39ce5
--- /dev/null
+++ b/flake-parts/nixosModules/_odoo/reset.sh
@@ -0,0 +1,105 @@
+# shellcheck shell=bash disable=SC2034,SC2154
+# A fragment: hostSpec.services.odoo (odoo.nix, mkScript) sets prog, instances,
+# golden_root and snapshot_root and puts lib.sh first; the built script is
+# linted whole again by writeShellApplication.
+usage() {
+  cat <<'EOF'
+Usage: odoo-reset --list
+       odoo-reset [--yes] [--clean] NAME...
+       odoo-reset [--yes] [--clean] --all
+
+Resets Odoo instances to how they were when first created: every record,
+file and setting made since is gone, demo data and apps are back as the
+instance is configured, and the provisioned admin password applies.
+
+A reset restores the instance's reset copy (made automatically when its
+database was first created): seconds. Without a valid copy (none yet, or
+the package, addons, demo data, apps or language changed since), or with
+--clean, the database is created from scratch (minutes) and a new copy
+is made. Snapshots (odoo-snapshot) are kept. Runs as root.
+
+  --list   show the configured instances
+  --all    reset every configured instance
+  --clean  ignore the reset copy and create the database from scratch
+  --yes    do not ask for confirmation (needed without a terminal)
+EOF
+}
+
+list() {
+  if [ "$(jq length "$instances")" -eq 0 ]; then
+    echo "No Odoo instances configured."
+    return
+  fi
+  printf '%-16s %-40s %-9s %-10s %-11s %s\n' NAME URL DATA STATE RESET-COPY EXPOSED
+  jq -r '.[] | [.name, .url, (if .demoData then "demo" else "no-demo" end), .unit,
+      ([if .lan then "lan" else empty end, (.publicUrl // empty)] | if length == 0 then "local only" else join(" ") end)] | @tsv' "$instances" |
+    while IFS=$'\t' read -r name url data unit exposed; do
+      state=$(systemctl is-active "$unit.service" || true)
+      copy=none
+      if [ -f "$golden_root/$name/key" ]; then
+        if [ "$(cat "$golden_root/$name/key")" = "$(field "$name" goldenKey)" ]; then copy=ready; else copy=stale; fi
+      fi
+      printf '%-16s %-40s %-9s %-10s %-11s %s\n' "$name" "$url" "$data" "$state" "$copy" "$exposed"
+    done
+}
+
+all=false
+yes=false
+clean=false
+selected=()
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --list) list; exit 0 ;;
+    --all) all=true ;;
+    --clean) clean=true ;;
+    --yes | -y) yes=true ;;
+    -h | --help) usage; exit 0 ;;
+    -*) usage >&2; die "unknown option $1" ;;
+    *) selected+=("$1") ;;
+  esac
+  shift
+done
+
+if $all; then
+  [ ${#selected[@]} -eq 0 ] || die "--all takes no names"
+  mapfile -t selected < <(instance_names)
+  if [ ${#selected[@]} -eq 0 ]; then
+    echo "No Odoo instances configured."
+    exit 0
+  fi
+fi
+if [ ${#selected[@]} -eq 0 ]; then
+  usage >&2
+  exit 1
+fi
+for name in "${selected[@]}"; do
+  configured "$name" || die "no instance named '$name' (odoo-reset --list shows them)"
+done
+need_root
+confirm "This deletes every record and file of: ${selected[*]}"
+
+for name in "${selected[@]}"; do
+  unit=$(field "$name" unit)
+  state=$(field "$name" stateDir)
+  url=$(field "$name" url)
+  $clean && drop_golden "$name"
+
+  echo "==> $name: stopping $unit"
+  systemctl stop "$unit.service"
+  drop_db "$unit"
+  if golden_valid "$name"; then
+    echo "==> $name: restoring the reset copy"
+    copy_db "$unit@golden" "$unit" "$unit"
+    restore_data "$name" "$golden_root/$name/data.tar"
+    echo "==> $name: starting"
+  else
+    echo "==> $name: no valid reset copy; creating the database from scratch (minutes)"
+    drop_golden "$name"
+    sql -c "CREATE DATABASE \"$unit\" OWNER \"$unit\""
+    if [ -d "$state" ]; then
+      find "$state" -mindepth 1 -delete
+    fi
+  fi
+  start_and_wait "$name"
+  echo "==> $name: fresh at $url"
+done
diff --git a/flake-parts/nixosModules/_odoo/snapshot.sh b/flake-parts/nixosModules/_odoo/snapshot.sh
new file mode 100644
index 0000000..d51f6ac
--- /dev/null
+++ b/flake-parts/nixosModules/_odoo/snapshot.sh
@@ -0,0 +1,122 @@
+# shellcheck shell=bash disable=SC2034,SC2154
+# A fragment: hostSpec.services.odoo (odoo.nix, mkScript) sets prog, instances,
+# golden_root and snapshot_root and puts lib.sh first; the built script is
+# linted whole again by writeShellApplication.
+usage() {
+  cat <<'EOF'
+Usage: odoo-snapshot list [NAME]
+       odoo-snapshot save NAME TAG
+       odoo-snapshot restore [--yes] NAME TAG
+       odoo-snapshot rm [--yes] NAME TAG
+
+Saves an instance as it is now (database and files) under a tag, and puts
+it back later: "the state just before step X". The instance stops for a
+few seconds while it is saved or restored. A restore keeps everything as
+saved, the admin password included. Snapshots survive odoo-reset;
+odoo-purge removes them with their instance. TAG: lowercase letters,
+digits, ., _ and -, at most 24. Runs as root (list too).
+
+  --yes    do not ask for confirmation (needed without a terminal)
+EOF
+}
+
+yes=false
+args=()
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --yes | -y) yes=true ;;
+    -h | --help) usage; exit 0 ;;
+    -*) usage >&2; die "unknown option $1" ;;
+    *) args+=("$1") ;;
+  esac
+  shift
+done
+[ ${#args[@]} -gt 0 ] || { usage >&2; exit 1; }
+action=${args[0]}
+name=${args[1]:-}
+tag=${args[2]:-}
+
+need_root
+
+check_name() {
+  [ -n "$name" ] || die "$action needs an instance name"
+  configured "$name" || die "no instance named '$name' (odoo-reset --list shows them)"
+}
+check_tag() {
+  [ -n "$tag" ] || die "$action needs a tag"
+  [[ "$tag" =~ ^[a-z0-9][a-z0-9._-]{0,23}$ ]] || die "'$tag' is not a tag (lowercase letters, digits, . _ -, at most 24)"
+}
+
+case "$action" in
+  list)
+    if [ -n "$name" ]; then
+      check_name
+      names=("$name")
+    else
+      mapfile -t names < <(instance_names)
+    fi
+    printf '%-16s %-26s %-20s %s\n' NAME TAG SAVED FILES
+    for n in "${names[@]}"; do
+      dir="$snapshot_root/$n"
+      [ -d "$dir" ] || continue
+      for archive in "$dir"/*.tar; do
+        [ -e "$archive" ] || continue
+        t=$(basename "$archive" .tar)
+        saved=$(date -r "$archive" '+%Y-%m-%d %H:%M')
+        size=$(du -h "$archive" | cut -f1)
+        printf '%-16s %-26s %-20s %s\n' "$n" "$t" "$saved" "$size"
+      done
+    done
+    ;;
+
+  save)
+    check_name
+    check_tag
+    unit=$(field "$name" unit)
+    snap="$unit@snap-$tag"
+    db_exists "$snap" && die "snapshot $tag of $name exists; odoo-snapshot rm $name $tag first"
+    mkdir -p "$snapshot_root/$name"
+    chmod 700 "$snapshot_root" "$snapshot_root/$name"
+    echo "==> $name: stopping $unit"
+    systemctl stop "$unit.service"
+    copy_db "$unit" "$snap" postgres
+    freeze_db "$snap"
+    save_data "$(field "$name" dataDir)" "$snapshot_root/$name/$tag.tar"
+    start_and_wait "$name"
+    echo "==> $name: saved as $tag"
+    ;;
+
+  restore)
+    check_name
+    check_tag
+    unit=$(field "$name" unit)
+    snap="$unit@snap-$tag"
+    archive="$snapshot_root/$name/$tag.tar"
+    { db_exists "$snap" && [ -f "$archive" ]; } || die "no snapshot $tag of $name (odoo-snapshot list $name)"
+    confirm "This replaces every record and file of $name with snapshot $tag"
+    echo "==> $name: stopping $unit"
+    systemctl stop "$unit.service"
+    drop_db "$unit"
+    copy_db "$snap" "$unit" "$unit"
+    restore_data "$name" "$archive"
+    start_and_wait "$name"
+    echo "==> $name: restored $tag at $(field "$name" url)"
+    ;;
+
+  rm)
+    check_name
+    check_tag
+    unit=$(field "$name" unit)
+    archive="$snapshot_root/$name/$tag.tar"
+    { db_exists "$unit@snap-$tag" || [ -f "$archive" ]; } || die "no snapshot $tag of $name"
+    confirm "This deletes snapshot $tag of $name"
+    drop_db "$unit@snap-$tag"
+    rm -f "$archive"
+    echo "==> $name: deleted snapshot $tag"
+    ;;
+
+  *)
+    usage >&2
+    die "unknown action $action"
+    ;;
+esac
diff --git a/flake-parts/nixosModules/odoo.nix b/flake-parts/nixosModules/odoo.nix
index 0926a9a..b25a388 100644
--- a/flake-parts/nixosModules/odoo.nix
+++ b/flake-parts/nixosModules/odoo.nix
@@ -1,25 +1,26 @@
 _: {
   # ------ NixOS Modules ------ #
-  # Local Odoo instances for testing: 0..n independent Odoo servers on one
-  # machine, each with its own PostgreSQL database and state, behind nginx
-  # with trusted HTTPS at https://<name>.<baseDomain> (default
-  # https://<name>.odoo.localhost), reachable from this machine only.
-  # https://<baseDomain> is an index page linking every instance.
+  # Odoo instances for testing (never production): 0..n independent Odoo
+  # servers on one machine, each with its own PostgreSQL database and
+  # state, behind nginx with trusted HTTPS at https://<name>.<baseDomain>
+  # (default https://<name>.odoo.localhost). https://<baseDomain> is an
+  # index page; https://mail.<baseDomain> is Mailpit, which catches every
+  # mail the instances send.
   #
   # Local by default: every Odoo binds 127.0.0.1 (HTTP port and gevent
   # websocket port = port + 1), nginx listens on loopback only, no firewall
   # port is opened and PostgreSQL keeps its default (unix socket, peer
   # auth, no TCP listener, no database password). Each instance can add:
-  #   expose.lan     https://<name>.<lan.domain> (odoo.<host>.home.arpa) on
-  #                  lan.addresses, 80/443 open on lan.interfaces, nginx
-  #                  allow-list lan.allowedClients, the local CA's
-  #                  certificate; clients use hosts-file lines from
-  #                  odoo-hosts and import http://<lan.domain>/ca.crt
+  #   expose.lan     https://<name>.<lan.domain> (odoo.<host>.home.arpa),
+  #                  80/443 open on lan.interfaces, nginx allow-list
+  #                  lan.allowedClients, the local CA's certificate; a client
+  #                  gets its hosts-file lines and the CA over plain HTTP at
+  #                  http://<this machine's LAN IP>/hosts.txt and /ca.crt
   #   expose.public  https://<domain> for the internet, Let's Encrypt
   #                  (HTTP-01), 80/443 open, database manager 404, login
   #                  rate limit, HSTS, web.base.url frozen at the domain
   # Names nginx does not serve on a LAN or public address get their TLS
-  # handshake refused (catch-all default servers).
+  # handshake refused (one catch-all default server).
   #
   # Not the nixpkgs services.odoo: that module runs exactly one instance
   # (one unit, one database named odoo, ports 8069/8072 fixed in its nginx
@@ -31,11 +32,24 @@ _: {
   #   unit, system user, PostgreSQL role and database   odoo-<name>
   #   state (data_dir: filestore, sessions, markers)     /var/lib/private/odoo-<name>
   #   runtime config (0600, holds the master password)   /run/odoo-<name>/odoo.cfg
+  #   reset copy (database odoo-<name>@golden)           /var/lib/odoo-golden/<name>
+  #   snapshots (databases odoo-<name>@snap-<tag>)       /var/lib/odoo-snapshots/<name>
   #
-  # odoo-reset NAME... | --all   back to a freshly created database (with
-  #                              or without demo data, per instance)
-  # odoo-purge [NAME...]         list or delete the database, role and
-  #                              state of instances no longer configured
+  # Start sequence of odoo-<name> (ExecStartPre, each step idempotent):
+  #   1. runtime config with the master password
+  #   2. first start (no .odoo.initialized): create the database with base
+  #      + `modules` (+ demo data, + language), neutralize it, mark it
+  #      for the reset copy
+  #   3. the package or addons changed: `-u all`; `modules` changed:
+  #      install the new apps
+  #   4. as root: save a newly created database as the reset copy
+  #   5. admin password (when the secret changed), web.base.url (public)
+  # Restart=on-failure, MemoryMax per instance.
+  #
+  # odoo-reset NAME... | --all   back to the freshly created state: from the
+  #                              reset copy in seconds, else from scratch
+  # odoo-snapshot save|restore|list|rm NAME TAG   named states to go back to
+  # odoo-purge [NAME...]         list or delete what removed instances left
   # odoo-hosts                   hosts-file lines for LAN clients
   #
   # Package: pkgs.odoo19 as fixed by external-systems/pkgs overlays.odoo
@@ -48,7 +62,7 @@ _: {
   #                   lan.domain and their subdomains, ca.key never
   #                   deployed, ca.crt trusted system wide and by Firefox
   #   odoo-tls        one nginx certificate for baseDomain, lan.domain and
-  #                   their wildcards
+  #                   their wildcards; key root:nginx 0440
   #   odoo-passwords  admin-password (web login of user "admin") and
   #                   master-password (Odoo's admin_passwd)
   # Run `clan vars generate <machine>` after enabling. The web login is
@@ -69,6 +83,8 @@ _: {
     cfg = config.hostSpec.services.odoo;
     iniFormat = pkgs.formats.ini {};
     postgresql = config.services.postgresql.package;
+    goldenRoot = "/var/lib/odoo-golden";
+    snapshotRoot = "/var/lib/odoo-snapshots";
 
     # Resolve a secret file: explicit setting wins; otherwise the clan vars
     # generator output (only touched when provisionSecrets = true, so
@@ -93,17 +109,13 @@ _: {
     ];
 
     loopback = ["127.0.0.1"] ++ lib.optional config.networking.enableIPv6 "[::1]";
-    # nginx wants IPv6 listen addresses in brackets.
-    bracket = address:
-      if lib.hasInfix ":" address
-      then "[${address}]"
-      else address;
-    lanListen = map bracket cfg.lan.addresses;
-    # Public vhosts also list the LAN and loopback addresses: nginx picks a
-    # server by the address a connection arrived on first, and a router
-    # forwards to the LAN address.
     wildcard = ["0.0.0.0"] ++ lib.optional config.networking.enableIPv6 "[::]";
-    publicListen = wildcard ++ lanListen ++ loopback;
+    # LAN and public vhosts listen on every address (no dependency on a
+    # DHCP address) plus loopback explicitly: nginx matches the address a
+    # connection arrived on before the name, and this machine reaches its
+    # own LAN names over loopback. Who gets in is the firewall's job
+    # (lan.interfaces) and lan.allowedClients'.
+    exposedListen = wildcard ++ loopback;
 
     # Instances in name order; the index fixes the automatic port.
     instances = lib.imap0 mkInstance (lib.attrNames cfg.instances);
@@ -112,10 +124,13 @@ _: {
     publicInstances = lib.filter (i: i.publicDomain != null) instances;
     hasLan = lanInstances != [];
     hasPublic = publicInstances != [];
+    mailOn = cfg.mail.enable && hasInstances;
+    mailLan = mailOn && cfg.mail.expose.lan && hasLan;
 
-    # LAN vhosts answer allowedClients only (all clients when empty).
+    # LAN vhosts answer allowedClients (and this machine) only; all
+    # clients when the list is empty.
     lanAccess = lib.optionalString (cfg.lan.allowedClients != []) ''
-      ${lib.concatMapStrings (client: "allow ${client};\n") cfg.lan.allowedClients}deny all;
+      ${lib.concatMapStrings (client: "allow ${client};\n") (cfg.lan.allowedClients ++ ["127.0.0.1" "::1"])}deny all;
     '';
 
     proxyTimeouts = ''
@@ -124,9 +139,16 @@ _: {
       proxy_send_timeout 720s;
     '';
 
+    # Every name a LAN client needs in its hosts file.
+    lanNames =
+      [cfg.lan.domain]
+      ++ map (i: i.lanHost) lanInstances
+      ++ lib.optional mailLan "mail.${cfg.lan.domain}";
+
     mkInstance = index: name: let
       icfg = cfg.instances.${name};
       unit = "odoo-${name}";
+      odoo = "${icfg.package}/bin/odoo";
       stateDir = "/var/lib/private/${unit}";
       dataDir = "${stateDir}/data";
       httpPort =
@@ -141,6 +163,16 @@ _: {
         then icfg.expose.public.domain
         else null;
 
+      # What a database is made from. The package key triggers `-u all`
+      # when it changes, the modules key the install of new apps, and the
+      # golden key (all of it) decides whether the reset copy still fits.
+      packageKey = builtins.unsafeDiscardStringContext "${icfg.package} ${toString icfg.addons}";
+      modulesKey = lib.concatStringsSep "," (lib.sort lib.lessThan icfg.modules);
+      goldenKey = builtins.hashString "sha256" (builtins.toJSON {
+        inherit packageKey modulesKey;
+        inherit (icfg) demoData language neutralize;
+      });
+
       storeConfig = iniFormat.generate "${unit}.cfg" {
         options =
           {
@@ -157,6 +189,21 @@ _: {
             list_db = false;
             # Behind nginx: honour X-Forwarded-*.
             proxy_mode = true;
+            # Per worker; MemoryMax caps the whole instance.
+            limit_memory_soft = icfg.limits.memorySoft * 1024 * 1024;
+            limit_memory_hard = icfg.limits.memoryHard * 1024 * 1024;
+            limit_time_cpu = icfg.limits.timeCpu;
+            limit_time_real = icfg.limits.timeReal;
+            max_cron_threads = 1;
+          }
+          // lib.optionalAttrs mailOn {
+            # Every mail goes to Mailpit; the neutralized database has no
+            # mail server of its own.
+            smtp_server = "127.0.0.1";
+            smtp_port = cfg.mail.smtpPort;
+            # A sender for mail that has none (system mail, a mail.mail
+            # created by hand); Odoo refuses to send it otherwise.
+            email_from = "${name}@${cfg.baseDomain}";
           }
           // lib.optionalAttrs (icfg.addons != []) {
             addons_path = lib.concatMapStringsSep "," toString icfg.addons;
@@ -175,16 +222,83 @@ _: {
           ${storeConfig} > ${runtimeConfig}
       '';
 
-      # First start, and the first start after odoo-reset: the marker is
-      # gone, so the (empty) database gets base and, if asked, demo data.
+      # Neutralize: mail servers, payment providers, outgoing integrations
+      # and crons off. Neutralize also adds an active mail server with the
+      # host "invalid", which blocks every mail. With Mailpit, that server
+      # is replaced by one for Mailpit and the mail queue runs again, so
+      # mail is sent but reaches Mailpit only. Idempotent: runs again
+      # after new apps.
+      afterNeutralize = pkgs.writeText "${unit}-after-neutralize.py" (lib.optionalString mailOn ''
+          Server = env["ir.mail_server"].sudo().with_context(active_test=False)
+          mailpit = Server.search([("name", "=", "Mailpit")], limit=1)
+          (Server.search([]) - mailpit).write({"active": False})
+          values = {
+              "name": "Mailpit",
+              "smtp_host": "127.0.0.1",
+              "smtp_port": ${toString cfg.mail.smtpPort},
+              "smtp_encryption": "none",
+              "sequence": 1,
+              "active": True,
+          }
+          if mailpit:
+              mailpit.write(values)
+          else:
+              Server.create(values)
+          cron = env.ref("mail.ir_cron_mail_scheduler_action", raise_if_not_found=False)
+          if cron:
+              cron.active = True
+        ''
+        + ''
+          env.cr.commit()
+        '');
+      neutralizeSteps = lib.optionalString icfg.neutralize ''
+        echo "pre-start: neutralizing ${unit} (test database: nothing reaches the outside)"
+        ${odoo} neutralize --database=${unit}
+        ${odoo} shell --database=${unit} --no-http < ${afterNeutralize}
+      '';
+
+      initModules = lib.concatStringsSep "," (["base"] ++ icfg.modules);
+
+      # First start, and the first start after a reset from scratch: the
+      # marker is gone, so the (empty) database gets base, the apps and,
+      # if asked, demo data.
       initDatabase = pkgs.writeShellScript "${unit}-init" ''
         set -euo pipefail
         mkdir -p ${dataDir}
-        marker=${dataDir}/.odoo.initialized
-        if [ ! -e "$marker" ]; then
-          echo "pre-start: creating database ${unit} (demo data: ${lib.boolToYesNo icfg.demoData})"
-          ${icfg.package}/bin/odoo --init=base --database=${unit} --stop-after-init ${lib.optionalString icfg.demoData "--with-demo"}
-          touch "$marker"
+        if [ ! -e ${dataDir}/.odoo.initialized ]; then
+          echo "pre-start: creating database ${unit} (demo data: ${lib.boolToYesNo icfg.demoData}, apps: ${initModules})"
+          ${odoo} --init=${initModules} --database=${unit} --stop-after-init \
+            ${lib.optionalString icfg.demoData "--with-demo"} \
+            ${lib.optionalString (icfg.language != null) "--load-language=${icfg.language}"}
+          ${neutralizeSteps}
+          echo ${lib.escapeShellArg packageKey} > ${dataDir}/.package
+          echo ${lib.escapeShellArg modulesKey} > ${dataDir}/.modules
+          touch ${dataDir}/.golden-pending
+          touch ${dataDir}/.odoo.initialized
+        fi
+      '';
+
+      # A new package or addons (a nixpkgs bump) leaves the database on
+      # the old module data: update every module. New apps in `modules`
+      # get installed; removed ones stay installed.
+      syncDatabase = pkgs.writeShellScript "${unit}-sync" ''
+        set -euo pipefail
+        package=${dataDir}/.package
+        modules=${dataDir}/.modules
+        [ -e "$package" ] || echo ${lib.escapeShellArg packageKey} > "$package"
+        [ -e "$modules" ] || echo ${lib.escapeShellArg modulesKey} > "$modules"
+        if [ "$(cat "$package")" != ${lib.escapeShellArg packageKey} ]; then
+          echo "pre-start: the Odoo package or addons changed; updating all modules of ${unit}"
+          ${odoo} --update=all --database=${unit} --stop-after-init
+          echo ${lib.escapeShellArg packageKey} > "$package"
+        fi
+        if [ "$(cat "$modules")" != ${lib.escapeShellArg modulesKey} ]; then
+          ${lib.optionalString (icfg.modules != []) ''
+          echo "pre-start: installing apps ${modulesKey} into ${unit}"
+          ${odoo} --init=${lib.concatStringsSep "," icfg.modules} --database=${unit} --stop-after-init
+          ${neutralizeSteps}
+        ''}
+          echo ${lib.escapeShellArg modulesKey} > "$modules"
         fi
       '';
 
@@ -197,22 +311,21 @@ _: {
         env.cr.commit()
       '';
 
-      # After the init. The marker holds the hash of the password last
-      # applied; odoo-reset deletes it with the rest of the state.
+      # The marker holds the hash of the password last applied; a reset
+      # removes it, so the provisioned password applies again.
       applyAdminPassword = pkgs.writeShellScript "${unit}-admin-password" ''
         set -euo pipefail
         marker=${dataDir}/.admin-password.sha256
         want=$(${pkgs.coreutils}/bin/sha256sum < "$CREDENTIALS_DIRECTORY/admin-password" | ${pkgs.coreutils}/bin/cut -d' ' -f1)
         if [ "$(cat "$marker" 2>/dev/null || true)" != "$want" ]; then
           echo "pre-start: setting the admin password"
-          ${icfg.package}/bin/odoo shell --database=${unit} --no-http < ${setAdminPassword}
+          ${odoo} shell --database=${unit} --no-http < ${setAdminPassword}
           echo "$want" > "$marker"
         fi
       '';
 
       # A public instance builds its links (mails, reports, portal) from
-      # the public URL, whichever name the admin logged in through. The
-      # marker holds the URL last applied; odoo-reset deletes it.
+      # the public URL, whichever name the admin logged in through.
       publicUrl = "https://${publicDomain}";
       setBaseUrl = pkgs.writeText "${unit}-set-base-url.py" ''
         params = env["ir.config_parameter"].sudo()
@@ -225,7 +338,7 @@ _: {
         marker=${dataDir}/.base-url
         if [ "$(cat "$marker" 2>/dev/null || true)" != "${publicUrl}" ]; then
           echo "pre-start: freezing web.base.url at ${publicUrl}"
-          ${icfg.package}/bin/odoo shell --database=${unit} --no-http < ${setBaseUrl}
+          ${odoo} shell --database=${unit} --no-http < ${setBaseUrl}
           echo "${publicUrl}" > "$marker"
         fi
       '';
@@ -251,7 +364,7 @@ _: {
         inherit locations;
       };
     in {
-      inherit name unit stateDir dataDir httpPort geventPort host lanHost publicDomain;
+      inherit name unit stateDir dataDir httpPort geventPort host lanHost publicDomain goldenKey;
       inherit (icfg) demoData;
       inherit (icfg.expose) lan;
       url = "https://${host}";
@@ -264,16 +377,30 @@ _: {
       service = {
         description = "Odoo instance ${name} (https://${host})";
         wantedBy = ["multi-user.target"];
-        after = ["network.target" "postgresql.target"];
+        after = ["network.target" "postgresql.target"] ++ lib.optional mailOn "mailpit-odoo.service";
         requires = ["postgresql.target"];
         # pg_dump for Odoo's backups.
         path = [postgresql];
         environment.ODOO_RC = runtimeConfig;
+        # A crashed or OOM-killed instance comes back; a broken one stops
+        # retrying after 4 failures in 15 minutes.
+        startLimitIntervalSec = 900;
+        startLimitBurst = 4;
         serviceConfig = {
-          ExecStart = "${icfg.package}/bin/odoo";
+          ExecStart = odoo;
           ExecStartPre =
-            ["${writeRuntimeConfig}" "${initDatabase}" "${applyAdminPassword}"]
+            [
+              "${writeRuntimeConfig}"
+              "${initDatabase}"
+              "${syncDatabase}"
+              # As root: copies databases as the postgres user.
+              "+${odooGoldenSave}/bin/odoo-golden-save ${name}"
+              "${applyAdminPassword}"
+            ]
             ++ lib.optional (publicDomain != null) "${applyBaseUrl}";
+          Restart = "on-failure";
+          RestartSec = "10s";
+          MemoryMax = lib.mkIf (icfg.memoryMax != null) icfg.memoryMax;
           # A static user (users.users below) with DynamicUser's sandbox,
           # as nixpkgs services.odoo does: state lands in /var/lib/private.
           DynamicUser = true;
@@ -283,9 +410,9 @@ _: {
           RuntimeDirectory = unit;
           RuntimeDirectoryMode = "0700";
           LoadCredential = credentials;
-          # The first start creates the database (with demo data,
-          # minutes); systemd's 90 s default would kill it.
-          TimeoutStartSec = "15min";
+          # Creating the database (with demo data and apps) or updating
+          # every module takes minutes; systemd's 90 s default would kill it.
+          TimeoutStartSec = "20min";
         };
       };
 
@@ -299,7 +426,7 @@ _: {
       lanVirtualHost =
         ownCertificate
         // {
-          listenAddresses = lanListen;
+          listenAddresses = exposedListen;
           extraConfig = proxyTimeouts + lanAccess;
         };
 
@@ -308,7 +435,7 @@ _: {
       publicVirtualHost = {
         forceSSL = true;
         enableACME = true;
-        listenAddresses = publicListen;
+        listenAddresses = exposedListen;
         extraConfig =
           proxyTimeouts
           + ''
@@ -331,321 +458,174 @@ _: {
       };
     };
 
-    # What odoo-reset and odoo-purge know about the configured instances.
+    # What the scripts know about the configured instances.
     instancesJson = pkgs.writeText "odoo-instances.json" (builtins.toJSON (map (i: {
-        inherit (i) name unit stateDir httpPort url demoData lan publicUrl;
+        inherit (i) name unit stateDir dataDir httpPort url demoData lan publicUrl goldenKey;
       })
       instances));
 
-    # The hosts-file lines a LAN client needs: one per exposed name and
-    # LAN address (hosts files have no wildcards).
-    hostsLines = lib.concatMapStrings (address:
-      lib.concatMapStrings (name: "${address}  ${name}\n")
-      ([cfg.lan.domain] ++ map (i: i.lanHost) lanInstances))
-    cfg.lan.addresses;
-
-    odooHosts = pkgs.writeShellScriptBin "odoo-hosts" ''
-      # The lines to add to a LAN client's hosts file (Windows:
-      # C:\Windows\System32\drivers\etc\hosts) to reach the LAN-exposed
-      # Odoo instances of this machine.
-      cat <<'EOF'
-      ${
-        if hasLan
-        then hostsLines
-        else "# No Odoo instance on this machine is exposed to the LAN.\n"
-      }EOF
+    commonTools = [pkgs.coreutils pkgs.curl pkgs.findutils pkgs.gnused pkgs.gnutar pkgs.jq pkgs.systemd pkgs.util-linux postgresql];
+
+    # The scripts share _odoo/lib.sh; shellcheck runs on each at build.
+    mkScript = name: file:
+      pkgs.writeShellApplication {
+        inherit name;
+        runtimeInputs = commonTools;
+        text =
+          ''
+            export prog=${name} instances=${instancesJson} golden_root=${goldenRoot} snapshot_root=${snapshotRoot}
+          ''
+          + builtins.readFile ./_odoo/lib.sh
+          + builtins.readFile file;
+      };
+
+    odooReset = mkScript "odoo-reset" ./_odoo/reset.sh;
+    odooSnapshot = mkScript "odoo-snapshot" ./_odoo/snapshot.sh;
+    odooPurge = mkScript "odoo-purge" ./_odoo/purge.sh;
+    odooGoldenSave = mkScript "odoo-golden-save" ./_odoo/golden-save.sh;
+
+    odooHosts = pkgs.writeShellApplication {
+      name = "odoo-hosts";
+      runtimeInputs = [pkgs.iproute2 pkgs.gawk];
+      text = ''
+        # The lines a LAN client (Windows: C:\Windows\System32\drivers\etc\hosts)
+        # needs for this machine's LAN-exposed Odoo, with the IPv4 addresses
+        # lan.interfaces have right now.
+        names=(${lib.escapeShellArgs lanNames})
+        ${
+          if hasLan
+          then ''
+            interfaces=(${lib.escapeShellArgs cfg.lan.interfaces})
+            found=0
+            for interface in "''${interfaces[@]}"; do
+              while read -r address; do
+                found=1
+                for name in "''${names[@]}"; do
+                  printf '%s  %s\n' "$address" "$name"
+                done
+              done < <(ip -o -4 addr show dev "$interface" scope global 2>/dev/null | awk '{split($4, a, "/"); print a[1]}')
+            done
+            if [ "$found" = 0 ]; then
+              echo "odoo-hosts: no IPv4 address on ${toString cfg.lan.interfaces} right now" >&2
+              exit 1
+            fi
+          ''
+          else ''
+            echo "# No Odoo instance on this machine is exposed to the LAN."
+            : "''${names[@]}"
+          ''
+        }
+      '';
+    };
+
+    statusText = i:
+      (
+        if i.demoData
+        then "demo data"
+        else "no demo data"
+      )
+      + lib.optionalString i.lan ", LAN: ${i.lanUrl}"
+      + lib.optionalString (i.publicUrl != null) ", public: ${i.publicUrl}";
+
+    pageStyle = ''
+      <style>
+        body { font-family: Georgia, serif; background: #1f1e1d; color: #faf9f5; max-width: 44rem; margin: 3rem auto; padding: 0 1rem; }
+        a { color: #e8a48a; }
+        li { margin: 0.5rem 0; }
+        pre { background: #2b2a27; padding: 0.75rem; overflow-x: auto; }
+        .muted { color: #b0aea5; }
+      </style>
     '';
 
-    lanIndexPage = pkgs.writeTextDir "index.html" ''
+    indexPage = pkgs.writeTextDir "index.html" ''
       <!doctype html>
       <html lang="en">
         <head>
           <meta charset="utf-8">
-          <title>Odoo on ${config.networking.hostName}</title>
-          <style>
-            body { font-family: Georgia, serif; background: #1f1e1d; color: #faf9f5; max-width: 44rem; margin: 3rem auto; padding: 0 1rem; }
-            a { color: #e8a48a; }
-            li { margin: 0.5rem 0; }
-            pre { background: #2b2a27; padding: 0.75rem; overflow-x: auto; }
-            .muted { color: #b0aea5; }
-          </style>
+          <title>Odoo instances</title>
+          ${pageStyle}
         </head>
         <body>
-          <h1>Odoo on ${config.networking.hostName}</h1>
+          <h1>Odoo instances</h1>
           <ul>
-            ${lib.concatMapStringsSep "\n" (i: ''<li><a href="${i.lanUrl}/">${i.name}</a> <span class="muted">${
-          if i.demoData
-          then "demo data"
-          else "no demo data"
-        }</span></li>'')
-      lanInstances}
+            ${lib.concatMapStringsSep "\n" (i: ''<li><a href="${i.url}/">${i.name}</a> <span class="muted">${statusText i}</span></li>'') instances}
           </ul>
-          <h2>Setting up a client</h2>
-          <p>1. Add these lines to the hosts file (Windows, as administrator:
-          <code>C:\Windows\System32\drivers\etc\hosts</code>, then
-          <code>ipconfig /flushdns</code>):</p>
-          <pre>${hostsLines}</pre>
-          <p>2. Trust the CA once: <a href="/ca.crt">ca.crt</a> (download it over
-          <code>http://${cfg.lan.domain}/ca.crt</code>). Windows, PowerShell as
-          administrator: <code>Import-Certificate -FilePath ca.crt -CertStoreLocation Cert:\LocalMachine\Root</code>.
-          It is only valid for names under ${cfg.baseDomain} and ${cfg.lan.domain}.</p>
+          ${lib.optionalString mailOn ''<p>Mail sent by the instances: <a href="https://mail.${cfg.baseDomain}/">Mailpit</a>.</p>''}
+          <p class="muted"><code>odoo-reset --list</code>, <code>sudo odoo-reset NAME</code>,
+          <code>sudo odoo-snapshot save NAME TAG</code>.</p>
         </body>
       </html>
     '';
 
-    indexPage = pkgs.writeTextDir "index.html" ''
+    lanIndexPage = pkgs.writeTextDir "index.html" ''
       <!doctype html>
       <html lang="en">
         <head>
           <meta charset="utf-8">
-          <title>Odoo instances</title>
-          <style>
-            body { font-family: Georgia, serif; background: #1f1e1d; color: #faf9f5; max-width: 40rem; margin: 3rem auto; padding: 0 1rem; }
-            a { color: #e8a48a; }
-            li { margin: 0.5rem 0; }
-            .muted { color: #b0aea5; }
-          </style>
+          <title>Odoo on ${config.networking.hostName}</title>
+          ${pageStyle}
         </head>
         <body>
-          <h1>Odoo instances</h1>
-          ${
-        if hasInstances
-        then ''
+          <h1>Odoo on ${config.networking.hostName}</h1>
           <ul>
-            ${lib.concatMapStringsSep "\n" (i: ''<li><a href="${i.url}/">${i.name}</a> <span class="muted">${
-              if i.demoData
-              then "demo data"
-              else "no demo data"
-            }${lib.optionalString i.lan ", LAN: ${i.lanUrl}"}${lib.optionalString (i.publicUrl != null) ", public: ${i.publicUrl}"}</span></li>'')
-          instances}
-          </ul>''
-        else "<p>No instances configured.</p>"
-      }
-          <p class="muted">Reset one: <code>sudo odoo-reset NAME</code>. List: <code>odoo-reset --list</code>.</p>
+            ${lib.concatMapStringsSep "\n" (i: ''<li><a href="${i.lanUrl}/">${i.name}</a> <span class="muted">${statusText i}</span></li>'') lanInstances}
+          </ul>
+          ${lib.optionalString mailLan ''<p>Mail sent by the instances: <a href="https://mail.${cfg.lan.domain}/">Mailpit</a>.</p>''}
+          <h2>Setting up a client</h2>
+          <p>1. Add <a href="/hosts.txt">these lines</a> to the hosts file (Windows, as
+          administrator: <code>C:\Windows\System32\drivers\etc\hosts</code>, then
+          <code>ipconfig /flushdns</code>).</p>
+          <p>2. Trust the CA once: <a href="/ca.crt">ca.crt</a>. Windows, PowerShell as
+          administrator: <code>Import-Certificate -FilePath ca.crt -CertStoreLocation Cert:\LocalMachine\Root</code>.
+          It is only valid for names under ${cfg.baseDomain} and ${cfg.lan.domain}.</p>
+          <p class="muted">Both also at <code>http://&lt;this machine's IP&gt;/hosts.txt</code> and
+          <code>/ca.crt</code>, before the hosts file knows any name.</p>
         </body>
       </html>
     '';
 
-    commonTools = [pkgs.coreutils pkgs.findutils pkgs.gnused pkgs.jq pkgs.util-linux postgresql];
-
-    odooReset = pkgs.writeShellApplication {
-      name = "odoo-reset";
-      runtimeInputs = commonTools ++ [pkgs.curl pkgs.systemd];
-      text = ''
-        instances=${instancesJson}
-
-        usage() {
-          cat <<'EOF'
-        Usage: odoo-reset --list
-               odoo-reset [--yes] NAME...
-               odoo-reset [--yes] --all
-
-        Resets Odoo instances to how they were when first enabled: stops the
-        instance, drops and recreates its PostgreSQL database, empties its state
-        (filestore, sessions, markers) and starts it again, which creates the
-        database afresh with demo data or without it, as the instance is
-        configured, and sets the provisioned admin password. Runs as root.
-
-          --list   show the configured instances
-          --all    reset every configured instance
-          --yes    do not ask for confirmation (needed without a terminal)
-        EOF
-        }
-
-        die() {
-          echo "odoo-reset: $*" >&2
-          exit 1
-        }
-
-        list() {
-          if [ "$(jq length "$instances")" -eq 0 ]; then
-            echo "No Odoo instances configured."
-            return
-          fi
-          printf '%-16s %-40s %-9s %-10s %s\n' NAME URL DATA STATE EXPOSED
-          jq -r '.[] | [.name, .url, (if .demoData then "demo" else "no-demo" end), .unit,
-              ([if .lan then "lan" else empty end, (.publicUrl // empty)] | if length == 0 then "local only" else join(" ") end)] | @tsv' "$instances" |
-            while IFS=$'\t' read -r name url data unit exposed; do
-              state=$(systemctl is-active "$unit.service" || true)
-              printf '%-16s %-40s %-9s %-10s %s\n' "$name" "$url" "$data" "$state" "$exposed"
-            done
-        }
-
-        all=false
-        yes=false
-        selected=()
-        while [ $# -gt 0 ]; do
-          case "$1" in
-            --list) list; exit 0 ;;
-            --all) all=true ;;
-            --yes | -y) yes=true ;;
-            -h | --help) usage; exit 0 ;;
-            -*) usage >&2; die "unknown option $1" ;;
-            *) selected+=("$1") ;;
-          esac
-          shift
-        done
-
-        if $all; then
-          [ ''${#selected[@]} -eq 0 ] || die "--all takes no names"
-          mapfile -t selected < <(jq -r '.[].name' "$instances")
-          if [ ''${#selected[@]} -eq 0 ]; then
-            echo "No Odoo instances configured."
-            exit 0
-          fi
-        fi
-        if [ ''${#selected[@]} -eq 0 ]; then
-          usage >&2
-          exit 1
-        fi
-        for name in "''${selected[@]}"; do
-          jq -e --arg n "$name" 'any(.[]; .name == $n)' "$instances" >/dev/null ||
-            die "no instance named '$name' (odoo-reset --list shows them)"
-        done
-        [ "$(id -u)" -eq 0 ] || die "run as root (sudo odoo-reset ...)"
-
-        if ! $yes; then
-          [ -t 0 ] || die "not a terminal: pass --yes to reset without asking"
-          echo "This deletes every record and file of: ''${selected[*]}"
-          read -r -p "Type yes to reset: " answer
-          [ "$answer" = yes ] || die "aborted, nothing changed"
-        fi
-
-        for name in "''${selected[@]}"; do
-          info=$(jq -c --arg n "$name" '.[] | select(.name == $n)' "$instances")
-          unit=$(jq -r .unit <<<"$info")
-          stateDir=$(jq -r .stateDir <<<"$info")
-          port=$(jq -r .httpPort <<<"$info")
-          url=$(jq -r .url <<<"$info")
-          demo=$(jq -r 'if .demoData then "with demo data" else "without demo data" end' <<<"$info")
-
-          echo "==> $name: stopping $unit"
-          systemctl stop "$unit.service"
-          echo "==> $name: recreating database $unit"
-          runuser -u postgres -- psql -v ON_ERROR_STOP=1 -q -d postgres \
-            -c "DROP DATABASE IF EXISTS \"$unit\" WITH (FORCE)" \
-            -c "CREATE DATABASE \"$unit\" OWNER \"$unit\""
-          echo "==> $name: emptying $stateDir"
-          if [ -d "$stateDir" ]; then
-            find "$stateDir" -mindepth 1 -delete
-          fi
-          echo "==> $name: starting, creating the database $demo (takes minutes)"
-          systemctl start "$unit.service"
-          ready=false
-          for _ in $(seq 300); do
-            if curl -sf -o /dev/null "http://127.0.0.1:$port/web/login"; then
-              ready=true
-              break
-            fi
-            sleep 1
-          done
-          $ready || die "$name did not answer on port $port; see journalctl -u $unit"
-          echo "==> $name: fresh at $url"
-        done
+    # The client's view of this machine: nginx fills in the address the
+    # request arrived on, so the lines are right whatever DHCP handed out.
+    # `return` runs in nginx's rewrite phase, before allow/deny (access
+    # phase), so the allow-list is checked here with the geo map.
+    hostsTxtLocation = {
+      extraConfig = ''
+        default_type text/plain;
+        ${lib.optionalString (cfg.lan.allowedClients != []) ''
+          if ($odoo_lan_client = 0) {
+            return 403;
+          }
+        ''}
+        return 200 "${lib.concatMapStrings (name: "$server_addr  ${name}\\n") lanNames}";
       '';
     };
-
-    odooPurge = pkgs.writeShellApplication {
-      name = "odoo-purge";
-      runtimeInputs = commonTools;
-      text = ''
-        instances=${instancesJson}
-
-        usage() {
-          cat <<'EOF'
-        Usage: odoo-purge
-               odoo-purge [--yes] NAME...
-
-        Without names: lists what removed Odoo instances left behind (a
-        database or role odoo-NAME, or a state folder /var/lib/private/odoo-NAME,
-        for a NAME no longer configured). With names: deletes it. A configured
-        instance is refused: odoo-reset empties those. Runs as root.
-
-          --yes    do not ask for confirmation (needed without a terminal)
-        EOF
-        }
-
-        die() {
-          echo "odoo-purge: $*" >&2
-          exit 1
-        }
-
-        sql() {
-          runuser -u postgres -- psql -v ON_ERROR_STOP=1 -qtA -d postgres "$@"
-        }
-
-        configured() {
-          jq -e --arg n "$1" 'any(.[]; .name == $n)' "$instances" >/dev/null
-        }
-
-        leftovers() {
-          {
-            sql -c "SELECT datname FROM pg_database WHERE datname LIKE 'odoo-%'"
-            sql -c "SELECT rolname FROM pg_roles WHERE rolname LIKE 'odoo-%'"
-            for dir in /var/lib/private/odoo-*; do
-              if [ -e "$dir" ]; then basename "$dir"; fi
-            done
-          } | sed -n 's/^odoo-//p' | sort -u | while read -r name; do
-            configured "$name" || echo "$name"
-          done
-        }
-
-        yes=false
-        selected=()
-        while [ $# -gt 0 ]; do
-          case "$1" in
-            --yes | -y) yes=true ;;
-            -h | --help) usage; exit 0 ;;
-            -*) usage >&2; die "unknown option $1" ;;
-            *) selected+=("$1") ;;
-          esac
-          shift
-        done
-
-        [ "$(id -u)" -eq 0 ] || die "run as root (sudo odoo-purge ...)"
-        sql -c "SELECT 1" >/dev/null || die "PostgreSQL is not reachable"
-
-        if [ ''${#selected[@]} -eq 0 ]; then
-          found=$(leftovers)
-          if [ -z "$found" ]; then
-            echo "Nothing left behind by removed Odoo instances."
-          else
-            echo "Left behind by removed instances (odoo-purge NAME deletes it):"
-            echo "$found"
-          fi
-          exit 0
-        fi
-
-        for name in "''${selected[@]}"; do
-          [[ "$name" =~ ^[a-z][a-z0-9-]*$ ]] || die "'$name' is not an instance name"
-          if configured "$name"; then
-            die "'$name' is still configured; remove it from hostSpec.services.odoo.instances and deploy first, or use odoo-reset"
-          fi
-        done
-
-        if ! $yes; then
-          [ -t 0 ] || die "not a terminal: pass --yes to purge without asking"
-          echo "This deletes the database, role and state of: ''${selected[*]}"
-          read -r -p "Type yes to purge: " answer
-          [ "$answer" = yes ] || die "aborted, nothing changed"
-        fi
-
-        for name in "''${selected[@]}"; do
-          unit="odoo-$name"
-          echo "==> $name: dropping database and role $unit"
-          sql -c "DROP DATABASE IF EXISTS \"$unit\" WITH (FORCE)" -c "DROP ROLE IF EXISTS \"$unit\""
-          echo "==> $name: deleting /var/lib/private/$unit"
-          rm -rf "/var/lib/private/$unit" "/var/lib/$unit"
-        done
+    caCrtLocation = {
+      alias = caFile;
+      extraConfig = ''
+        ${lanAccess}
+        default_type application/x-x509-ca-cert;
       '';
     };
+
+    mailVirtualHost = listen: access: {
+      forceSSL = true;
+      listenAddresses = listen;
+      sslCertificate = tlsCertFile;
+      sslCertificateKey = tlsKeyFile;
+      extraConfig = access;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString cfg.mail.uiPort}";
+        proxyWebsockets = true;
+        recommendedProxySettings = true;
+      };
+    };
   in {
     options.hostSpec.services.odoo = {
       enable = lib.mkEnableOption ''
-        local Odoo instances for testing: PostgreSQL, the HTTPS CA and
-        certificate, odoo-reset and odoo-purge. The instances themselves
-        come from `instances`; with none, only PostgreSQL runs (so
-        odoo-purge can still clean up)'';
+        Odoo instances for testing: PostgreSQL, the HTTPS CA and
+        certificate, Mailpit, odoo-reset, odoo-snapshot, odoo-purge and
+        odoo-hosts. The instances themselves come from `instances`; with
+        none, only PostgreSQL runs (so odoo-purge can still clean up)'';
 
       package = lib.mkOption {
         type = lib.types.package;
@@ -663,10 +643,10 @@ _: {
         default = "odoo.localhost";
         description = ''
           Instance NAME is served at https://NAME.<baseDomain>;
-          https://<baseDomain> lists the instances. Every name resolves to
-          loopback through networking.hosts (Firefox resolves *.localhost
-          itself). The CA is name-constrained to this domain and its
-          subdomains.
+          https://<baseDomain> lists the instances, https://mail.<baseDomain>
+          is Mailpit. Every name resolves to loopback through
+          networking.hosts (Firefox resolves *.localhost itself). The CA is
+          name-constrained to this domain, lan.domain and their subdomains.
         '';
       };
 
@@ -681,6 +661,34 @@ _: {
         '';
       };
 
+      mail = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Run Mailpit and send every instance's mail to it, at
+            https://mail.<baseDomain>: nothing reaches a real mailbox, and
+            every mail can be read. Off, mail goes nowhere (no SMTP server
+            on localhost:25).
+          '';
+        };
+        expose.lan = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Also serve Mailpit at https://mail.<lan.domain> while an instance is LAN-exposed.";
+        };
+        uiPort = lib.mkOption {
+          type = lib.types.port;
+          default = 8025;
+          description = "Mailpit's web UI port on 127.0.0.1 (nginx proxies it).";
+        };
+        smtpPort = lib.mkOption {
+          type = lib.types.port;
+          default = 1025;
+          description = "Mailpit's SMTP port on 127.0.0.1, where every instance sends its mail.";
+        };
+      };
+
       lan = {
         domain = lib.mkOption {
           type = lib.types.str;
@@ -688,33 +696,29 @@ _: {
           defaultText = lib.literalExpression ''"odoo.''${config.networking.hostName}.home.arpa"'';
           description = ''
             LAN-exposed instance NAME is served at https://NAME.<lan.domain>;
-            https://<lan.domain> lists them with the hosts-file lines and the
-            CA (http://<lan.domain>/ca.crt). home.arpa is reserved for home
+            https://<lan.domain> lists them. home.arpa is reserved for home
             networks (RFC 8375). *.localhost cannot work from other
             machines: browsers send it to the client itself.
           '';
         };
-        addresses = lib.mkOption {
+        interfaces = lib.mkOption {
           type = lib.types.listOf lib.types.str;
           default = [];
           description = ''
-            This machine's LAN addresses (IPv4 or IPv6, no prefix length)
-            nginx listens on for LAN-exposed instances. From the values
-            flake, never a literal here.
+            Interfaces the firewall opens 80 and 443 on while an instance
+            sets expose.lan; odoo-hosts reads their addresses. nginx listens
+            on every address, so a DHCP address change needs no rebuild.
           '';
         };
-        interfaces = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          default = [];
-          description = "Interfaces the firewall opens 80 and 443 on while an instance sets expose.lan.";
-        };
         allowedClients = lib.mkOption {
           type = lib.types.listOf lib.types.str;
           default = [];
           description = ''
-            Addresses or CIDRs (nginx allow) that may use the LAN names;
-            everyone else gets 403. Empty allows every client that reaches
-            the interfaces.
+            Addresses or CIDRs (nginx allow) that may use the LAN names
+            (this machine always may); everyone else gets 403. Empty allows
+            every client that reaches the port, including over trusted
+            interfaces such as a VPN. Required when an instance is also
+            public.
           '';
         };
       };
@@ -732,15 +736,16 @@ _: {
         default = {};
         example = lib.literalExpression ''
           {
-            demo = {};                     # https://demo.odoo.localhost, demo data
-            blank.demoData = false;        # https://blank.odoo.localhost, empty
+            demo.modules = ["sale_management" "crm"];   # demo data + apps
+            blank.demoData = false;                      # empty database
           }
         '';
         description = ''
-          Odoo instances, by name (a DNS label: lowercase letters, digits
-          and hyphens, starting with a letter). Each gets its own unit,
-          user, database and state, all named odoo-NAME. Removing one
-          leaves its database and state behind: odoo-purge deletes them.
+          Odoo instances, by name (a DNS label of at most 24 characters:
+          lowercase letters, digits and hyphens, starting with a letter;
+          not "mail"). Each gets its own unit, user, database and state, all
+          named odoo-NAME. Removing one leaves its database and state
+          behind: odoo-purge deletes them.
         '';
         type = lib.types.attrsOf (lib.types.submodule {
           options = {
@@ -749,7 +754,37 @@ _: {
               default = true;
               description = ''
                 Load Odoo's demo data when the database is created: at the
-                first start and after every odoo-reset.
+                first start and on every reset.
+              '';
+            };
+
+            modules = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [];
+              example = ["sale_management" "crm" "stock" "l10n_us"];
+              description = ''
+                Apps (Odoo module names) installed when the database is
+                created, and installed on the next start when added later.
+                Removing one does not uninstall it; a reset does.
+              '';
+            };
+
+            language = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = null;
+              example = "de_DE";
+              description = "Language loaded when the database is created (--load-language).";
+            };
+
+            neutralize = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Run `odoo neutralize` on every new database and after new
+                apps: mail servers, payment providers, outgoing integrations
+                and scheduled actions off (the mail queue stays on, and its
+                mail reaches Mailpit only). Odoo shows a "neutralized"
+                banner.
               '';
             };
 
@@ -757,7 +792,10 @@ _: {
               type = lib.types.package;
               default = cfg.package;
               defaultText = lib.literalExpression "config.hostSpec.services.odoo.package";
-              description = "Odoo package of this instance.";
+              description = ''
+                Odoo package of this instance. When it changes, the next
+                start updates every module (`-u all`).
+              '';
             };
 
             workers = lib.mkOption {
@@ -770,6 +808,40 @@ _: {
               '';
             };
 
+            memoryMax = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = "4G";
+              description = ''
+                systemd MemoryMax for the whole instance (workers, cron,
+                websocket, database creation). An instance over it is
+                killed and restarted, instead of starving the machine.
+                null: no cap.
+              '';
+            };
+
+            limits = {
+              memorySoft = lib.mkOption {
+                type = lib.types.ints.positive;
+                default = 1536;
+                description = "Odoo limit_memory_soft per worker, MiB: a worker over it is recycled after its request.";
+              };
+              memoryHard = lib.mkOption {
+                type = lib.types.ints.positive;
+                default = 2048;
+                description = "Odoo limit_memory_hard per worker, MiB: a worker over it is killed at once.";
+              };
+              timeCpu = lib.mkOption {
+                type = lib.types.ints.positive;
+                default = 600;
+                description = "Odoo limit_time_cpu per request, seconds (generous for imports and reports).";
+              };
+              timeReal = lib.mkOption {
+                type = lib.types.ints.positive;
+                default = 1200;
+                description = "Odoo limit_time_real per request, seconds.";
+              };
+            };
+
             addons = lib.mkOption {
               type = lib.types.listOf lib.types.package;
               default = [];
@@ -788,8 +860,8 @@ _: {
             expose = {
               lan = lib.mkEnableOption ''
                 https://NAME.<lan.domain> for other machines on the LAN
-                (lan.addresses, lan.interfaces, lan.allowedClients), with
-                the local CA's certificate'';
+                (lan.interfaces, lan.allowedClients), with the local CA's
+                certificate'';
 
               public = {
                 enable = lib.mkEnableOption ''
@@ -839,7 +911,7 @@ _: {
       tlsCertFile = lib.mkOption {
         type = lib.types.nullOr lib.types.str;
         default = null;
-        description = "External certificate PEM for baseDomain and *.baseDomain. Required when provisionSecrets = false.";
+        description = "External certificate PEM for baseDomain, lan.domain and their wildcards. Required when provisionSecrets = false.";
       };
       tlsKeyFile = lib.mkOption {
         type = lib.types.nullOr lib.types.str;
@@ -861,13 +933,18 @@ _: {
     config = lib.mkIf cfg.enable (lib.mkMerge [
       {
         assertions = let
+          names = lib.attrNames cfg.instances;
           ports = lib.concatMap (i: [i.httpPort i.geventPort]) instances;
           publicDomains = map (i: i.publicDomain) publicInstances;
           ownDomain = domain: domain == cfg.baseDomain || lib.hasSuffix ".${cfg.baseDomain}" domain || domain == cfg.lan.domain || lib.hasSuffix ".${cfg.lan.domain}" domain;
         in [
           {
-            assertion = lib.all (name: builtins.match "[a-z][a-z0-9-]*" name != null) (lib.attrNames cfg.instances);
-            message = "hostSpec.services.odoo.instances: names must be DNS labels (lowercase letters, digits, hyphens, starting with a letter)";
+            assertion = lib.all (name: builtins.match "[a-z][a-z0-9-]{0,23}" name != null) names;
+            message = "hostSpec.services.odoo.instances: names must be DNS labels of at most 24 characters (lowercase letters, digits, hyphens, starting with a letter)";
+          }
+          {
+            assertion = !(cfg.mail.enable && lib.elem "mail" names);
+            message = "hostSpec.services.odoo.instances: \"mail\" is Mailpit's name (https://mail.${cfg.baseDomain})";
           }
           {
             assertion = lib.length ports == lib.length (lib.unique ports);
@@ -878,8 +955,16 @@ _: {
             message = "hostSpec.services.odoo.instances: port + 1 must be a valid port";
           }
           {
-            assertion = !hasLan || (cfg.lan.addresses != [] && cfg.lan.interfaces != []);
-            message = "hostSpec.services.odoo.lan: addresses and interfaces must be set when an instance sets expose.lan";
+            assertion = !mailOn || !(lib.elem cfg.mail.uiPort ports || lib.elem cfg.mail.smtpPort ports);
+            message = "hostSpec.services.odoo.mail: uiPort and smtpPort must not be an instance's port";
+          }
+          {
+            assertion = !hasLan || cfg.lan.interfaces != [];
+            message = "hostSpec.services.odoo.lan.interfaces must be set when an instance sets expose.lan";
+          }
+          {
+            assertion = !(hasLan && hasPublic) || cfg.lan.allowedClients != [];
+            message = "hostSpec.services.odoo.lan.allowedClients must be set when instances are both LAN-exposed and public: the LAN names would otherwise answer the internet";
           }
           {
             assertion = lib.all (i: i.publicDomain != null) (lib.filter (i: cfg.instances.${i.name}.expose.public.enable) instances);
@@ -966,7 +1051,13 @@ _: {
           };
         };
 
-        environment.systemPackages = [odooReset odooPurge odooHosts];
+        environment.systemPackages = [odooReset odooSnapshot odooPurge odooHosts];
+
+        # Reset copies and snapshots: whole databases' files, root only.
+        systemd.tmpfiles.rules = [
+          "d ${goldenRoot} 0700 root root -"
+          "d ${snapshotRoot} 0700 root root -"
+        ];
 
         # On even with no instances, so odoo-purge can drop what removed
         # instances left behind.
@@ -1023,58 +1114,87 @@ _: {
         };
 
         networking.hosts = let
-          names = [cfg.baseDomain] ++ map (i: i.host) instances;
+          names =
+            [cfg.baseDomain]
+            ++ map (i: i.host) instances
+            ++ lib.optional mailOn "mail.${cfg.baseDomain}"
+            # This machine reaches its LAN names over loopback.
+            ++ lib.optionals hasLan lanNames;
         in
           {"127.0.0.1" = names;}
           // lib.optionalAttrs config.networking.enableIPv6 {"::1" = names;};
       })
 
-      # LAN: the exposed instances, an index with the CA, and a catch-all
-      # that refuses every other name, on the LAN addresses only; 80/443
+      # Mailpit: every mail the instances send, readable in a browser.
+      (lib.mkIf mailOn {
+        services = {
+          mailpit.instances.odoo = {
+            listen = "127.0.0.1:${toString cfg.mail.uiPort}";
+            smtp = "127.0.0.1:${toString cfg.mail.smtpPort}";
+            database = "odoo.db";
+            max = 2000;
+          };
+          nginx.virtualHosts."mail.${cfg.baseDomain}" = mailVirtualHost loopback "";
+        };
+      })
+
+      # LAN: the exposed instances, an index, hosts.txt and the CA; 80/443
       # open on the LAN interfaces only.
       (lib.mkIf hasLan {
-        services.nginx.virtualHosts =
-          lib.listToAttrs (map (i: lib.nameValuePair i.lanHost i.lanVirtualHost) lanInstances)
-          // {
-            ${cfg.lan.domain} = {
-              # HTTP too: a new client fetches /ca.crt before it trusts HTTPS.
-              addSSL = true;
-              listenAddresses = lanListen;
-              sslCertificate = tlsCertFile;
-              sslCertificateKey = tlsKeyFile;
-              root = lanIndexPage;
-              extraConfig = lanAccess;
-              locations."= /ca.crt" = {
-                alias = caFile;
-                extraConfig = "default_type application/x-x509-ca-cert;";
+        services.nginx = {
+          virtualHosts =
+            lib.listToAttrs (map (i: lib.nameValuePair i.lanHost i.lanVirtualHost) lanInstances)
+            // {
+              ${cfg.lan.domain} = {
+                # HTTP too: a new client fetches /ca.crt before it trusts HTTPS.
+                addSSL = true;
+                listenAddresses = exposedListen;
+                sslCertificate = tlsCertFile;
+                sslCertificateKey = tlsKeyFile;
+                root = lanIndexPage;
+                extraConfig = lanAccess;
+                locations = {
+                  "= /ca.crt" = caCrtLocation;
+                  "= /hosts.txt" = hostsTxtLocation;
+                };
               };
+            }
+            // lib.optionalAttrs mailLan {
+              "mail.${cfg.lan.domain}" = mailVirtualHost exposedListen lanAccess;
             };
-            odoo-lan-default = {
-              serverName = "_";
-              default = true;
-              rejectSSL = true;
-              listenAddresses = lanListen;
-              locations."/".return = "444";
-            };
-          };
 
-        # nginx binds lan.addresses, which DHCP (NetworkManager) may not
-        # have assigned yet at boot or after a reconnect; without this
-        # nginx fails to start and takes every vhost down with it.
-        boot.kernel.sysctl = {
-          "net.ipv4.ip_nonlocal_bind" = 1;
-          "net.ipv6.ip_nonlocal_bind" = 1;
+          # lan.allowedClients for locations that answer with `return`.
+          appendHttpConfig = lib.mkIf (cfg.lan.allowedClients != []) ''
+            geo $odoo_lan_client {
+              default 0;
+              ${lib.concatMapStrings (client: "${client} 1;\n") (cfg.lan.allowedClients ++ ["127.0.0.1" "::1"])}}
+          '';
         };
 
-        networking = {
-          firewall.interfaces = lib.genAttrs cfg.lan.interfaces (_: {allowedTCPPorts = [80 443];});
-          # This machine reaches its LAN names too.
-          hosts = lib.genAttrs cfg.lan.addresses (_: [cfg.lan.domain] ++ map (i: i.lanHost) lanInstances);
+        networking.firewall.interfaces = lib.genAttrs cfg.lan.interfaces (_: {allowedTCPPorts = [80 443];});
+      })
+
+      # One catch-all for every exposed address: unknown names (local-only
+      # instances, *.localhost, a bare IP) get their TLS handshake refused.
+      # On a LAN, plain http://<IP>/hosts.txt and /ca.crt bootstrap a client
+      # that knows no name yet.
+      (lib.mkIf (hasLan || hasPublic) {
+        services.nginx.virtualHosts.odoo-default = {
+          serverName = "_";
+          default = true;
+          rejectSSL = true;
+          listenAddresses = wildcard;
+          locations =
+            {"/".return = "444";}
+            // lib.optionalAttrs hasLan {
+              "= /ca.crt" = caCrtLocation;
+              "= /hosts.txt" = hostsTxtLocation;
+            };
         };
       })
 
       # Internet: Let's Encrypt over HTTP-01 (port 80 must reach this
-      # machine), a catch-all that refuses unknown names, 80/443 open.
+      # machine), 80/443 open.
       (lib.mkIf hasPublic {
         security.acme = {
           acceptTerms = true;
@@ -1085,17 +1205,7 @@ _: {
           appendHttpConfig = ''
             limit_req_zone $binary_remote_addr zone=odoo_login:10m rate=10r/m;
           '';
-          virtualHosts =
-            lib.listToAttrs (map (i: lib.nameValuePair i.publicDomain i.publicVirtualHost) publicInstances)
-            // {
-              odoo-public-default = {
-                serverName = "_";
-                default = true;
-                rejectSSL = true;
-                listenAddresses = wildcard;
-                locations."/".return = "444";
-              };
-            };
+          virtualHosts = lib.listToAttrs (map (i: lib.nameValuePair i.publicDomain i.publicVirtualHost) publicInstances);
         };
 
         networking.firewall.allowedTCPPorts = [80 443];
diff --git a/flake.lock b/flake.lock
index f484464..0170ed2 100644
--- a/flake.lock
+++ b/flake.lock
@@ -322,11 +322,11 @@
         "zen-browser": "zen-browser"
       },
       "locked": {
-        "lastModified": 1794230460,
-        "narHash": "sha256-zQ11S2rIUURH4yuk4vqxFzJ7Uiv58aQxmsud3weDZGI=",
+        "lastModified": 1794509535,
+        "narHash": "sha256-js5L/es+CXbtZn05wil3ReJVXUF9D8cROJaBJQ/thcI=",
         "owner": "external-systems",
         "repo": "pkgs",
-        "rev": "1188419cc1a2f1e9c22cbe3a62e9858b8a92f557",
+        "rev": "5c80ae9badefcbb2633c4d3a93063546eed63b80",
         "type": "github"
       },
       "original": {
diff --git a/tests/fixtures/nixos.nix b/tests/fixtures/nixos.nix
index a532242..12fd01b 100644
--- a/tests/fixtures/nixos.nix
+++ b/tests/fixtures/nixos.nix
@@ -61,15 +61,20 @@
         enable = true;
         # Documentation-range placeholders (RFC 5737, RFC 2606).
         lan = {
-          addresses = ["192.0.2.10"];
           interfaces = ["eth0"];
           allowedClients = ["192.0.2.0/24"];
         };
         acme.email = "hostmaster@example.com";
         instances = {
-          demo.expose.lan = true;
+          demo = {
+            modules = ["crm"];
+            language = "de_DE";
+            expose.lan = true;
+          };
           blank = {
             demoData = false;
+            neutralize = false;
+            memoryMax = null;
             port = 19069;
           };
           shop.expose = {
diff --git a/vm-tests/_odoo-probe.py b/vm-tests/_odoo-probe.py
index 4c2e79e..cc7c2a0 100644
--- a/vm-tests/_odoo-probe.py
+++ b/vm-tests/_odoo-probe.py
@@ -16,6 +16,10 @@
   INSTANCE db-list                     exit 0 if the database service lists databases
   INSTANCE db-drop MASTER_PASSWORD     exit 0 if the database service drops the database
   INSTANCE base-url PASSWORD URL       exit 0 if web.base.url is URL and frozen
+  INSTANCE send-mail PASSWORD TO SUBJ  queue a mail and run the mail queue cron
+  INSTANCE mail-state PASSWORD SUBJ    print the queued mail's state and failure
+  INSTANCE installed PASSWORD MODULE   exit 0 if MODULE is installed
+  INSTANCE neutralized PASSWORD        exit 0 if the database is neutralized
 """
 
 import base64
@@ -53,7 +57,8 @@ def call(self, service, method, *args):
             reply = json.load(response)
         if "error" in reply:
             raise SystemExit(f"odoo error: {json.dumps(reply['error'])[:2000]}")
-        return reply["result"]
+        # A method returning None leaves "result" out.
+        return reply.get("result")
 
     def login(self, password):
         uid = self.call("common", "login", self.db, "admin", password)
@@ -121,6 +126,51 @@ def main():
         }
         assert params.get("web.base.url") == url, params
         assert params.get("web.base.url.freeze") == "True", params
+    elif action == "send-mail":
+        password, to, subject = args
+        # Queued and kept after sending, as Odoo's own mails are; the queue
+        # cron (kept on by the module after neutralize) sends it.
+        odoo.execute(
+            password,
+            "mail.mail",
+            "create",
+            {"subject": subject, "email_to": to, "body_html": "<p>vm test</p>", "auto_delete": False},
+        )
+        [cron] = odoo.execute(
+            password,
+            "ir.model.data",
+            "search_read",
+            [["module", "=", "mail"], ["name", "=", "ir_cron_mail_scheduler_action"]],
+            fields=["res_id"],
+        )
+        odoo.execute(password, "ir.cron", "method_direct_trigger", [cron["res_id"]])
+    elif action == "mail-state":
+        password, subject = args
+        print(odoo.execute(
+            password,
+            "mail.mail",
+            "search_read",
+            [["subject", "=", subject]],
+            fields=["state", "failure_reason", "email_from", "email_to"],
+        ))
+    elif action == "installed":
+        password, module = args
+        found = odoo.execute(
+            password,
+            "ir.module.module",
+            "search_count",
+            [["name", "=", module], ["state", "=", "installed"]],
+        )
+        assert found, f"{module} is not installed"
+    elif action == "neutralized":
+        value = odoo.execute(
+            args[0],
+            "ir.config_parameter",
+            "search_read",
+            [["key", "=", "database.is_neutralized"]],
+            fields=["value"],
+        )
+        assert value and value[0]["value"] in ("True", "true", "1"), value
     elif action == "db-list":
         odoo.call("db", "list")
     elif action == "db-drop":
diff --git a/vm-tests/odoo-lan.nix b/vm-tests/odoo-lan.nix
index 83a66ad..a140ea4 100644
--- a/vm-tests/odoo-lan.nix
+++ b/vm-tests/odoo-lan.nix
@@ -1,11 +1,12 @@
 # expose.lan, the Windows-laptop path on a Linux client: the server
 # exposes demo (demo data) and blank (none) to the LAN and keeps private
-# local only. The client does what a Windows user does: pastes the
-# odoo-hosts lines into its hosts file, downloads the CA over HTTP and
-# trusts it, then uses both instances. private stays unreachable under
-# any name, the Odoo ports stay closed, a stranger outside allowedClients
-# gets 403, the CA cannot vouch for names outside its constraints, and
-# nginx still starts while the LAN address is missing.
+# local only. The client does what a Windows user does: knowing only the
+# server's IP, it fetches http://<IP>/hosts.txt into its hosts file and
+# http://<IP>/ca.crt into its trust store, then uses both instances and
+# Mailpit. private stays unreachable under any name, the Odoo ports stay
+# closed, a stranger outside allowedClients gets 403, the CA cannot vouch
+# for names outside its constraints, and nginx does not depend on the LAN
+# address (DHCP may change it).
 {inputs, ...}: let
   common = import ./_odoo-common.nix {inherit inputs;};
 in {
@@ -14,7 +15,6 @@ in {
       imports = [common.module];
       hostSpec.services.odoo = {
         lan = {
-          addresses = [nodes.server.networking.primaryIPAddress];
           interfaces = ["eth1"];
           allowedClients = map (n: "${nodes.${n}.networking.primaryIPAddress}/32") ["server" "client"];
         };
@@ -52,32 +52,40 @@ in {
 
     server_ip = "${nodes.server.networking.primaryIPAddress}"
     lan = "odoo.server.home.arpa"
+    names = (lan, f"demo.{lan}", f"blank.{lan}", f"mail.{lan}")
     ca = "/tmp/odoo-ca.crt"
 
     start_all()
     wait_ready(server, ["demo", "blank", "private"])
     server.wait_for_open_port(443, server_ip, timeout=300)
 
-    with subtest("odoo-hosts prints one line per LAN name"):
+    with subtest("odoo-hosts prints the current address for every LAN name"):
         hosts = server.succeed("odoo-hosts")
-        for name in (lan, f"demo.{lan}", f"blank.{lan}"):
+        for name in names:
             assert f"{server_ip}  {name}\n" in hosts, hosts
         assert "private" not in hosts, hosts
         listing = server.succeed("odoo-reset --list")
         assert "local only" in listing and "lan" in listing, listing
 
-    with subtest("a client adds the lines and downloads the CA over HTTP"):
+    with subtest("a client knowing only the IP bootstraps itself over HTTP"):
+        served = client.succeed(f"curl -sf http://{server_ip}/hosts.txt")
+        assert served == hosts, f"hosts.txt {served!r} != odoo-hosts {hosts!r}"
         for node in (client, stranger):
             # /etc/hosts is a store symlink; a user edits a real file.
             node.succeed('cp --remove-destination "$(readlink -f /etc/hosts)" /etc/hosts')
             node.succeed(f"cat >> /etc/hosts <<'EOF'\n{hosts}{server_ip}  private.{lan}\nEOF")
         # Untrusted until the CA is imported.
         client.fail(f"curl -sf https://demo.{lan}/web/login")
-        client.succeed(f"curl -sf http://{lan}/ca.crt -o {ca}")
-        served = client.succeed(f"sha256sum < {ca}")
-        assert served == server.succeed("sha256sum < /etc/odoo-test-tls/ca.crt"), "served CA differs"
+        client.succeed(f"curl -sf http://{server_ip}/ca.crt -o {ca}")
+        assert client.succeed(f"sha256sum < {ca}") == server.succeed("sha256sum < /etc/odoo-test-tls/ca.crt"), "served CA differs"
+        assert client.succeed(f"curl -sf http://{lan}/ca.crt | sha256sum") == client.succeed(f"sha256sum < {ca}")
         content_type = client.succeed(f"curl -sf -o /dev/null -w '%{{content_type}}' http://{lan}/ca.crt")
         assert content_type == "application/x-x509-ca-cert", content_type
+        # Anything else on the bare IP: closed.
+        client.fail(f"curl -sf http://{server_ip}/")
+
+    with subtest("Mailpit is on the LAN too"):
+        client.succeed(f"curl -sf --cacert {ca} https://mail.{lan}/ >/dev/null")
 
     with subtest("with the CA, both LAN instances work from the client"):
         for name in ("demo", "blank"):
@@ -111,7 +119,10 @@ in {
         for url in (f"https://demo.{lan}/web/login", f"https://{lan}/"):
             code = stranger.succeed(f"curl -sk -o /dev/null -w '%{{http_code}}' {url}").strip()
             assert code == "403", f"{url}: {code}"
-        code = stranger.succeed(f"curl -s -o /dev/null -w '%{{http_code}}' http://{lan}/ca.crt").strip()
+        for url in (f"http://{lan}/ca.crt", f"http://{server_ip}/hosts.txt", f"http://{server_ip}/ca.crt"):
+            code = stranger.succeed(f"curl -s -o /dev/null -w '%{{http_code}}' {url}").strip()
+            assert code == "403", f"{url}: {code}"
+        code = stranger.succeed(f"curl -sk -o /dev/null -w '%{{http_code}}' https://mail.{lan}/").strip()
         assert code == "403", code
 
     with subtest("the CA cannot vouch for names outside its constraints"):
@@ -122,14 +133,21 @@ in {
         server.succeed("curl -sf https://private.odoo.localhost/web/login >/dev/null")
         server.succeed(f"curl -sf https://demo.{lan}/web/login >/dev/null")
 
-    with subtest("nginx starts while the LAN address is missing (DHCP not there yet)"):
+    with subtest("a new DHCP address needs no rebuild or restart"):
         prefix = server.succeed(f"ip -o -4 addr show dev eth1 | grep -o '{server_ip}/[0-9]*'").strip()
-        server.succeed(f"ip addr del {prefix} dev eth1")
+        length = prefix.split("/")[1]
+        new_ip = server_ip.rsplit(".", 1)[0] + ".99"
+        server.succeed(f"ip addr del {prefix} dev eth1 && ip addr add {new_ip}/{length} dev eth1")
+        assert f"{new_ip}  demo.{lan}\n" in server.succeed("odoo-hosts")
+        assert server_ip not in server.succeed("odoo-hosts")
+        client.succeed(f"curl -sf --cacert {ca} --resolve demo.{lan}:443:{new_ip} https://demo.{lan}/web/login >/dev/null")
+        assert client.succeed(f"curl -sf http://{new_ip}/hosts.txt").startswith(f"{new_ip}  ")
+        # nginx starts whatever the address (or none).
+        server.succeed(f"ip addr del {new_ip}/{length} dev eth1")
         server.succeed("systemctl restart nginx.service")
         # nginx is Type=simple: restart returns before it listens.
         server.wait_until_succeeds("curl -sf https://demo.odoo.localhost/web/login >/dev/null", timeout=60)
-        server.succeed("systemctl is-active nginx.service")
-        server.fail("journalctl -u nginx.service -b | grep 'Cannot assign requested address' >/dev/null")
+        server.fail("odoo-hosts")
         server.succeed(f"ip addr add {prefix} dev eth1")
         client.wait_until_succeeds(f"curl -sf --cacert {ca} https://demo.{lan}/web/login >/dev/null", timeout=60)
   '';
diff --git a/vm-tests/odoo-lifecycle.nix b/vm-tests/odoo-lifecycle.nix
new file mode 100644
index 0000000..ebe1e66
--- /dev/null
+++ b/vm-tests/odoo-lifecycle.nix
@@ -0,0 +1,104 @@
+# An instance's life after creation: apps from `modules` installed with
+# the database; the database neutralized, so every mail lands in Mailpit
+# (https://mail.odoo.localhost) and nothing leaves the machine; then a
+# new Odoo package and a new app arrive (the "changed" specialisation, as
+# after a nixpkgs bump plus a config edit) and the next start updates
+# every module and installs the app, keeping the data, while the reset
+# copy turns stale.
+{inputs, ...}: let
+  common = import ./_odoo-common.nix {inherit inputs;};
+in {
+  nodes.machine = {
+    lib,
+    pkgs,
+    ...
+  }: let
+    odoo = inputs.org-pkgs.packages.${pkgs.stdenv.hostPlatform.system}.odoo19;
+  in {
+    imports = [common.module];
+    hostSpec.services.odoo.instances.app = {
+      demoData = false;
+      modules = ["mail"];
+      workers = 1;
+    };
+    # Same Odoo under another store path: what a package bump looks like.
+    specialisation.changed.configuration.hostSpec.services.odoo.instances.app = {
+      package = lib.mkForce (pkgs.symlinkJoin {
+        name = "odoo19-bumped";
+        paths = [odoo];
+      });
+      modules = lib.mkForce ["mail" "contacts"];
+    };
+    environment.systemPackages = [pkgs.jq];
+    virtualisation = {
+      memorySize = 4096;
+      cores = 4;
+    };
+  };
+
+  # Create with mail, then -u all and an install.
+  globalTimeout = 3600;
+
+  testScript = ''
+    ${common.pythonHelpers}
+
+    start_all()
+    wait_ready(machine, ["app"])
+
+    with subtest("apps from `modules` come with the database"):
+        probe(machine, "app", "installed", admin_password, "mail")
+        probe_fails(machine, "app", "installed", admin_password, "contacts")
+
+    with subtest("the database is neutralized, the mail queue still runs"):
+        probe(machine, "app", "neutralized", admin_password)
+        active = machine.succeed(
+            "runuser -u postgres -- psql -qtA -d odoo-app -c "
+            "\"SELECT c.active FROM ir_cron c JOIN ir_model_data d ON d.res_id = c.id AND d.model = 'ir.cron' "
+            "WHERE d.module = 'mail' AND d.name = 'ir_cron_mail_scheduler_action'\""
+        ).strip()
+        assert active == "t", active
+
+    with subtest("mail lands in Mailpit, readable over HTTPS"):
+        machine.wait_for_unit("mailpit-odoo.service", timeout=120)
+        probe(machine, "app", "send-mail", admin_password, "someone@example.com", "vm-test-mail-subject")
+        try:
+            machine.wait_until_succeeds(
+                "curl -sf http://127.0.0.1:8025/api/v1/messages | jq -e '.messages | any(.Subject == \"vm-test-mail-subject\")'",
+                timeout=60,
+            )
+        except Exception:
+            print(probe(machine, "app", "mail-state", admin_password, "vm-test-mail-subject"))
+            print(machine.execute("journalctl -u odoo-app -u mailpit-odoo -b --no-pager | grep -iE 'mail|smtp' | tail -40")[1])
+            raise
+        assert "'sent'" in probe(machine, "app", "mail-state", admin_password, "vm-test-mail-subject")
+        # Sent from the instance's default sender (email_from).
+        machine.succeed(
+            "curl -sf http://127.0.0.1:8025/api/v1/messages | "
+            f"jq -e '.messages | any(.From.Address == \"app@{base_domain}\")'"
+        )
+        machine.succeed(f"curl -sf https://mail.{base_domain}/ >/dev/null")
+        machine.succeed("grep -x 'smtp_port=1025' /run/odoo-app/odoo.cfg >/dev/null")
+
+    with subtest("a new package and a new app: -u all and the install, data kept"):
+        probe(machine, "app", "partner-create", admin_password, "kept-across-the-bump")
+        assert "ready" in machine.succeed("odoo-reset --list")
+        machine.succeed("/run/current-system/specialisation/changed/bin/switch-to-configuration test >&2")
+        machine.wait_for_unit("odoo-app.service", timeout=1500)
+        machine.wait_until_succeeds(f"curl -sf https://app.{base_domain}/web/login >/dev/null", timeout=300)
+        journal = machine.succeed("journalctl -u odoo-app.service -b")
+        assert "the Odoo package or addons changed; updating all modules" in journal, journal[-3000:]
+        assert "installing apps contacts,mail" in journal, journal[-3000:]
+        probe(machine, "app", "installed", admin_password, "contacts")
+        probe(machine, "app", "partner-exists", admin_password, "kept-across-the-bump")
+        probe(machine, "app", "neutralized", admin_password)
+        # The copy was made from the old package and apps.
+        assert "stale" in machine.succeed("odoo-reset --list")
+
+    with subtest("the next start changes nothing"):
+        machine.succeed("systemctl restart odoo-app.service")
+        machine.wait_until_succeeds(f"curl -sf https://app.{base_domain}/web/login >/dev/null", timeout=300)
+        since = machine.succeed("systemctl show -p ExecMainStartTimestamp --value odoo-app.service").strip()
+        journal = machine.succeed(f"journalctl -u odoo-app.service --since {shlex.quote(since)}")
+        assert "updating all modules" not in journal and "installing apps" not in journal, journal[-2000:]
+  '';
+}
diff --git a/vm-tests/odoo-public.nix b/vm-tests/odoo-public.nix
index a9f940d..89db77b 100644
--- a/vm-tests/odoo-public.nix
+++ b/vm-tests/odoo-public.nix
@@ -101,6 +101,9 @@ in {
         internet.fail(f"curl -sk --resolve shop.odoo.localhost:443:{server_ip} https://shop.odoo.localhost/web/login")
         # No SNI or an unknown one: the handshake is refused.
         internet.fail(f"curl -sk https://{server_ip}/")
+        # No LAN exposure: the bare IP offers no hosts.txt or CA.
+        internet.fail(f"curl -sf http://{server_ip}/hosts.txt")
+        internet.fail(f"curl -sf http://{server_ip}/ca.crt")
         for port in range(18069, 18073):
             internet.fail(f"curl -s --connect-timeout 3 http://{server_ip}:{port}/")
         server.succeed("curl -sf https://intern.odoo.localhost/web/login >/dev/null")
diff --git a/vm-tests/odoo-reset.nix b/vm-tests/odoo-reset.nix
index 96ff666..ff956bd 100644
--- a/vm-tests/odoo-reset.nix
+++ b/vm-tests/odoo-reset.nix
@@ -1,10 +1,11 @@
-# odoo-reset and odoo-purge on two instances: demo (demo data) and blank
-# (none). Records, an attachment in the filestore and a password changed
-# in the UI disappear on reset; the instance comes back as freshly
-# created, with or without demo data as configured, and the other
-# instance is untouched. odoo-purge removes what a removed instance left
-# behind (database, role, state) and never touches a configured one.
-# Refusals change nothing.
+# odoo-reset, odoo-snapshot and odoo-purge on two instances: demo (demo
+# data) and blank (none). The first start saves each new database as its
+# reset copy; a reset restores it in seconds, and records, files and a
+# password changed in the UI are gone. --clean and a stale copy fall back
+# to creating the database from scratch, which makes a new copy.
+# Snapshots save a state and bring it back, password included. odoo-purge
+# removes what a removed instance left (database, role, state, reset copy,
+# snapshots) and never touches a configured one. Refusals change nothing.
 {inputs, ...}: let
   common = import ./_odoo-common.nix {inherit inputs;};
 in {
@@ -23,10 +24,12 @@ in {
     };
   };
 
-  # Five database creations: two at boot, one reset, then both again.
+  # Four databases created from scratch: two at boot, --clean, a stale copy.
   globalTimeout = 3600;
 
   testScript = ''
+    import time
+
     ${common.pythonHelpers}
 
     def marker(name):
@@ -35,9 +38,27 @@ in {
     def sql(query):
         return machine.succeed(f"runuser -u postgres -- psql -qtA -d postgres -c {shlex.quote(query)}").strip()
 
+    def copy_state(name):
+        line = [l for l in machine.succeed("odoo-reset --list").splitlines() if l.startswith(name + " ")]
+        assert line, f"{name} not listed"
+        return line[0].split()[4]
+
+    def reset(args):
+        start = time.monotonic()
+        out = machine.succeed(f"odoo-reset --yes {args} 2>&1")
+        return out, time.monotonic() - start
+
     start_all()
     wait_ready(machine, ["demo", "blank"])
 
+    with subtest("the first start saves a reset copy of each new database"):
+        for name in ("demo", "blank"):
+            assert copy_state(name) == "ready", machine.succeed("odoo-reset --list")
+            machine.succeed(f"test -s /var/lib/odoo-golden/{name}/data.tar")
+            assert sql(f"SELECT datistemplate AND NOT datallowconn FROM pg_database WHERE datname = 'odoo-{name}@golden'") == "t"
+            machine.fail(f"test -e /var/lib/private/odoo-{name}/data/.golden-pending")
+        assert machine.succeed("stat -c '%a %U' /var/lib/odoo-golden").strip() == "700 root"
+
     with subtest("seed both instances"):
         for name in ("demo", "blank"):
             probe(machine, name, "partner-create", admin_password, f"seeded-{name}")
@@ -47,67 +68,103 @@ in {
         demo_marker = marker("demo")
 
     with subtest("refusals change nothing"):
-        # Without a terminal and without --yes.
         machine.fail("odoo-reset demo < /dev/null")
-        # An unknown name, even next to a known one.
         machine.fail("odoo-reset --yes demo nope")
-        # --all with names.
         machine.fail("odoo-reset --yes --all demo")
-        # Not root.
         machine.fail("runuser -u nobody -- odoo-reset --yes demo")
         machine.succeed("systemctl is-active odoo-demo.service")
         probe(machine, "demo", "partner-exists", "changed-in-the-ui", "seeded-demo")
         machine.succeed(f"test -f {attachment}")
         assert marker("demo") == demo_marker
 
-    with subtest("odoo-reset demo: fresh demo database, blank untouched"):
-        machine.succeed("odoo-reset --yes demo >&2")
+    with subtest("odoo-reset demo restores the reset copy in seconds"):
+        out, took = reset("demo")
+        assert "restoring the reset copy" in out, out
+        assert took < 120, f"reset took {took:.0f} s"
         machine.succeed("systemctl is-active odoo-demo.service")
-        assert marker("demo") != demo_marker
-        # The provisioned admin password is back; the UI change is gone.
         probe(machine, "demo", "login", admin_password)
         probe_fails(machine, "demo", "login", "changed-in-the-ui")
         probe(machine, "demo", "demo", admin_password)
         probe_fails(machine, "demo", "partner-exists", admin_password, "seeded-demo")
         machine.fail(f"test -e {attachment}")
-        # blank kept its record.
         probe(machine, "blank", "partner-exists", admin_password, "seeded-blank")
+        assert copy_state("demo") == "ready"
+
+    with subtest("--clean creates the database from scratch and a new copy"):
+        out, _ = reset("--clean demo")
+        assert "from scratch" in out, out
+        assert marker("demo") != demo_marker
+        probe(machine, "demo", "demo", admin_password)
+        assert copy_state("demo") == "ready"
+
+    with subtest("a stale copy is not used"):
+        machine.succeed("echo stale > /var/lib/odoo-golden/blank/key")
+        assert copy_state("blank") == "stale"
+        out, _ = reset("blank")
+        assert "no valid reset copy" in out, out
+        probe_fails(machine, "blank", "partner-exists", admin_password, "seeded-blank")
+        probe(machine, "blank", "no-demo", admin_password)
+        assert copy_state("blank") == "ready"
 
     with subtest("odoo-reset --all: each instance comes back as configured"):
         probe(machine, "demo", "partner-create", admin_password, "seeded-again")
-        machine.succeed("odoo-reset --yes --all >&2")
-        for name in ("demo", "blank"):
-            machine.succeed(f"systemctl is-active odoo-{name}.service")
-            probe(machine, name, "login", admin_password)
+        out, _ = reset("--all")
+        assert out.count("restoring the reset copy") == 2, out
         probe_fails(machine, "demo", "partner-exists", admin_password, "seeded-again")
-        probe_fails(machine, "blank", "partner-exists", admin_password, "seeded-blank")
         probe(machine, "demo", "demo", admin_password)
         probe(machine, "blank", "no-demo", admin_password)
-        listing = machine.succeed("odoo-reset --list")
-        assert "active" in listing, listing
+
+    with subtest("snapshots save a state and bring it back"):
+        probe(machine, "demo", "partner-create", admin_password, "before-step-x")
+        probe(machine, "demo", "set-password", admin_password, "snapshot-password")
+        machine.succeed("odoo-snapshot save demo before-x >&2")
+        assert "before-x" in machine.succeed("odoo-snapshot list demo")
+        assert sql("SELECT count(*) FROM pg_database WHERE datname = 'odoo-demo@snap-before-x'") == "1"
+        # Step X: more data, then a reset wipes everything.
+        probe(machine, "demo", "partner-create", "snapshot-password", "after-step-x")
+        reset("demo")
+        probe_fails(machine, "demo", "partner-exists", admin_password, "before-step-x")
+        # The snapshot survived the reset and brings back its state.
+        machine.fail("odoo-snapshot restore demo before-x < /dev/null")
+        machine.succeed("odoo-snapshot restore --yes demo before-x >&2")
+        probe(machine, "demo", "partner-exists", "snapshot-password", "before-step-x")
+        probe_fails(machine, "demo", "partner-exists", "snapshot-password", "after-step-x")
+        # Refusals: an existing tag, a bad tag, an unknown instance, not root.
+        machine.fail("odoo-snapshot save demo before-x")
+        machine.fail("odoo-snapshot save demo 'Bad Tag'")
+        machine.fail("odoo-snapshot save nope x")
+        machine.fail("runuser -u nobody -- odoo-snapshot list")
+        machine.succeed("odoo-snapshot rm --yes demo before-x")
+        assert "before-x" not in machine.succeed("odoo-snapshot list")
+        assert sql("SELECT count(*) FROM pg_database WHERE datname LIKE 'odoo-demo@snap-%'") == "0"
+        reset("demo")
 
     with subtest("odoo-purge lists and deletes what a removed instance left"):
         assert "Nothing left behind" in machine.succeed("odoo-purge")
-        # What a removed instance "gone" leaves: role, database, state.
+        # What a removed instance "gone" leaves: role, database, reset copy,
+        # a snapshot, state.
         sql('CREATE ROLE "odoo-gone" LOGIN')
         sql('CREATE DATABASE "odoo-gone" OWNER "odoo-gone"')
-        machine.succeed("mkdir -p /var/lib/private/odoo-gone/data/filestore")
+        sql('CREATE DATABASE "odoo-gone@golden" TEMPLATE "odoo-gone"')
+        sql('ALTER DATABASE "odoo-gone@golden" WITH IS_TEMPLATE true ALLOW_CONNECTIONS false')
+        sql('CREATE DATABASE "odoo-gone@snap-x" TEMPLATE "odoo-gone"')
+        machine.succeed("mkdir -p /var/lib/private/odoo-gone/data /var/lib/odoo-golden/gone /var/lib/odoo-snapshots/gone")
         listing = machine.succeed("odoo-purge")
         assert listing.strip().splitlines()[-1] == "gone", listing
-        # Refusals: configured instances, bad names, no terminal, not root.
         machine.fail("odoo-purge --yes demo")
         machine.fail("odoo-purge --yes 'x; rm -rf /'")
         machine.fail("odoo-purge gone < /dev/null")
         machine.fail("runuser -u nobody -- odoo-purge --yes gone")
-        assert sql("SELECT count(*) FROM pg_database WHERE datname = 'odoo-gone'") == "1"
+        assert sql("SELECT count(*) FROM pg_database WHERE datname LIKE 'odoo-gone%'") == "3"
         machine.succeed("odoo-purge --yes gone >&2")
-        assert sql("SELECT count(*) FROM pg_database WHERE datname = 'odoo-gone'") == "0"
+        assert sql("SELECT count(*) FROM pg_database WHERE datname LIKE 'odoo-gone%'") == "0"
         assert sql("SELECT count(*) FROM pg_roles WHERE rolname = 'odoo-gone'") == "0"
         machine.fail("test -e /var/lib/private/odoo-gone")
+        machine.fail("test -e /var/lib/odoo-golden/gone")
+        machine.fail("test -e /var/lib/odoo-snapshots/gone")
         assert "Nothing left behind" in machine.succeed("odoo-purge")
-        # The configured instances are untouched.
         for name in ("demo", "blank"):
             probe(machine, name, "login", admin_password)
-        assert sql("SELECT count(*) FROM pg_database WHERE datname IN ('odoo-demo', 'odoo-blank')") == "2"
+            assert copy_state(name) == "ready"
   '';
 }
diff --git a/vm-tests/odoo.nix b/vm-tests/odoo.nix
index b5180d0..38c9f92 100644
--- a/vm-tests/odoo.nix
+++ b/vm-tests/odoo.nix
@@ -16,6 +16,7 @@ in {
         blank = {
           demoData = false;
           workers = 1;
+          memoryMax = "2G";
         };
       };
       # Two databases created at once, one with demo data.
@@ -141,14 +142,31 @@ in {
 
     with subtest("everything listens on loopback only"):
         listeners = machine.succeed("ss -ltnH").split("\n")
-        for port in ("80", "443", "18069", "18070", "18071", "18072"):
+        # Odoo, nginx and Mailpit (UI 8025, SMTP 1025).
+        for port in ("80", "443", "18069", "18070", "18071", "18072", "8025", "1025"):
             local = [line.split()[3] for line in listeners if line and line.split()[3].endswith(":" + port)]
             assert local, f"nothing listens on {port}"
             for address in local:
                 assert address.startswith(("127.0.0.1:", "[::1]:")), f"{address} is not loopback"
-        for port in (80, 443, 18069, 18070, 18071, 18072):
+        for port in (80, 443, 18069, 18070, 18071, 18072, 8025, 1025):
             outsider.fail(f"curl -s --connect-timeout 3 http://machine:{port}/")
 
+    with subtest("a crashed instance comes back on its own"):
+        pid = machine.succeed("systemctl show -p MainPID --value odoo-blank.service").strip()
+        machine.succeed(f"kill -9 {pid}")
+        machine.wait_until_succeeds(
+            f"[ \"$(systemctl show -p MainPID --value odoo-blank.service)\" != {pid} ] && systemctl is-active odoo-blank.service",
+            timeout=120,
+        )
+        machine.wait_until_succeeds(f"curl -sf https://blank.{base_domain}/web/login >/dev/null", timeout=300)
+        assert machine.succeed("systemctl show -p NRestarts --value odoo-blank.service").strip() == "1"
+
+    with subtest("memory is capped per instance and per worker"):
+        assert machine.succeed("systemctl show -p MemoryMax --value odoo-demo.service").strip() == str(4 * 1024**3)
+        assert machine.succeed("systemctl show -p MemoryMax --value odoo-blank.service").strip() == str(2 * 1024**3)
+        machine.succeed(f"grep -x 'limit_memory_hard={2048 * 1024 * 1024}' /run/odoo-demo/odoo.cfg >/dev/null")
+        machine.succeed("grep -x 'max_cron_threads=1' /run/odoo-demo/odoo.cfg >/dev/null")
+
     with subtest("a password changed in Odoo survives a restart"):
         probe(machine, "demo", "set-password", admin_password, "changed-in-the-ui")
         machine.succeed("systemctl restart odoo-demo.service")
@@ -175,6 +193,7 @@ in {
         empty.wait_for_unit("postgresql.service", timeout=300)
         empty.fail("systemctl list-units --all --plain --no-legend 'odoo-*' | grep . >/dev/null")
         empty.fail("systemctl is-active nginx.service")
+        empty.fail("systemctl is-active mailpit-odoo.service")
         assert "No Odoo instances configured." in empty.succeed("odoo-reset --list")
         empty.succeed("odoo-reset --all --yes")
         assert "Nothing left behind" in empty.succeed("odoo-purge")