diff --git a/defaults/finbot.env b/defaults/finbot.env index ff6a5b9..2b1bbdb 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -12,7 +12,7 @@ # util/check_config.py /opt/finbot.env # Where state and cache live. Leave unset under systemd with StateDirectory=finbot (it uses $STATE_DIRECTORY, -# /var/lib/finbot). +# /var/lib/finbot, the only directory the shipped unit can write). # DATA_DIR=/var/lib/finbot # The listening socket, for a network in push mode (its webhook URL set below) diff --git a/doc/caching.md b/doc/caching.md index 66facf4..fa23620 100644 --- a/doc/caching.md +++ b/doc/caching.md @@ -17,7 +17,7 @@ Each Sharesight holdings report also leaves `finbot_sharesight_largest.json`, wh Short Man's CSV of ASX short interest, which `/shorts` reads, is cached for `cache_seconds` too (`finbot_shortman.json`). Yahoo's cookie and crumb (for 14 days) and Sharesight's access token (until it expires) are kept whatever `cache` says, and each is renewed at once when its service refuses it. -To flush the cache, so that what it holds is fetched afresh, run `util/flush.py` with the files to delete, by when they were written. With no argument it prints its help: +To flush the cache, so that what it holds is fetched afresh, run `util/flush.py` with the files to delete, by when they were written. With no argument it prints its help (run by hand on a host with finbot.service, it finds the service's data directory: use `sudo`): ``` util/flush.py --before 180s # the files written more than 180 seconds ago util/flush.py --after 24h # the files written in the last 24 hours diff --git a/doc/configuration.md b/doc/configuration.md index bdb1fe6..03f0a02 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -4,7 +4,7 @@ Configuration is split into two files -- finbot.env for startup environment vari Copy the shipped template: ``` -sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env +sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env ``` ## `finbot.env` @@ -86,12 +86,12 @@ The administrators of the chat a person is in are admins there with no setting: A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. -edit `settings.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `util/check_config.py /opt/finbot.env`. Edit `finbot.env`, then restart: a running bot cannot see it change. +edit `settings.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `sudo util/check_config.py /opt/finbot.env` (under systemd it reads the service's `settings.conf`). Edit `finbot.env`, then restart: a running bot cannot see it change. **Upgrading from a version before 16 Oct 2026?** Start commit `ff52f2b` once first, as [Scheduling](scheduling.md#from-an-earlier-version) says: it splits one old `.env` (`util/split_env.py`, which it asks for) and renames `preferences.conf` to `settings.conf`, which later versions no longer do. ## Data directory -All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`); in Docker it is `/data`, the host's `/opt/finbot-data` (see [docker.md](docker.md)). +All state and cache lives in one data directory: `DATA_DIR` if set, else systemd's `$STATE_DIRECTORY` (`/var/lib/finbot` under the shipped unit), else `var/` in the checkout. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`); in Docker it is `/data`, the host's `/opt/finbot-data` (see [docker.md](docker.md)). The tools in `util/` (`sync_defaults.py`, `check_config.py`, `flush.py`), run by hand outside the service, where systemd's `$STATE_DIRECTORY` is not set, also find the service's directory, `/var/lib/private/finbot`, when there is one, so that they act on the directory the service reads; use `sudo`, as it belongs to the service. Reference data (`finbot_adr.json`, `finbot_sws_*.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: ``` util/sync_defaults.py # dry run @@ -102,7 +102,7 @@ util/sync_defaults.py --dest /srv/finbot --create --apply # and make it (mode 7 ``` The reminders that `reminder.json` held are default jobs now (see [Scheduling](scheduling.md#the-default-jobs)), and finbot reads `reminder.json` no more: delete the data directory's copy. A chat adds its own with `/jobs add reminder SCHEDULE "TEXT"`. -With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. +With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`, a link to `/var/lib/private/finbot`), so no setting is needed, and none should be made: under the shipped unit only that directory is writable. To move existing data once, start the service (it makes the directory), stop it, run `sudo cp -a /opt/finbot/var/. /var/lib/private/finbot/` and start it again. An explicit `DATA_DIR` always wins. In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. Each start copies the defaults the data directory lacks (`util/sync_defaults.py --apply`). diff --git a/doc/install.md b/doc/install.md index 323d88a..d0e01e9 100644 --- a/doc/install.md +++ b/doc/install.md @@ -29,19 +29,30 @@ systemd, as below. To run it in a container instead, see [docker.md](docker.md). sudo apt install python3-websocket # Slack's Socket Mode, Discord's gateway API sudo apt install python3-nacl # Discord's push mode ``` -2. The code: +2. The code, in `/opt/finbot` (not under `/home`: the service unit hides it): ``` - git clone https://github.com/robdevops/finbot.git ~/finbot + sudo git clone https://github.com/robdevops/finbot.git /opt/finbot ``` 3. The startup file, from its template; then set each network's keys from its guide, and Sharesight's: ``` - sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env + sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env sudoedit /opt/finbot.env ``` -4. Check it: `~/finbot/util/check_config.py /opt/finbot.env` (it also warns if the file is not `600 root:root`) -5. Run it under systemd, below. The first start makes the data directory (`var/` in the checkout, or - `/var/lib/finbot` with `DynamicUser=`); each chat then has the default scheduled jobs, which `/jobs` changes - ([scheduling.md](scheduling.md)). +4. Check it: `/opt/finbot/util/check_config.py /opt/finbot.env` (it also warns if the file is not `600 root:root`) +5. Run it under systemd ([below](#daemonize-systemd)). The first start makes the data directory, `/var/lib/finbot` + (systemd keeps it in `/var/lib/private/finbot`, where `/var/lib/finbot` points); each chat then has the default + scheduled jobs, which `/jobs` changes ([scheduling.md](scheduling.md)). +6. Optional, to change settings: copy the starter `settings.conf` into the data directory, edit it, check it and reload + (`util/sync_defaults.py` and `util/check_config.py` find the service's data directory when run by hand; run them + with `sudo`, as the directory is the service's): + ``` + sudo /opt/finbot/util/sync_defaults.py --apply + sudoedit /var/lib/finbot/settings.conf + sudo /opt/finbot/util/check_config.py /opt/finbot.env + sudo systemctl reload finbot + ``` + The files are root-owned and readable, so the service reads them; systemd gives the directory to the service's user + again at the next start. Leave `DATA_DIR` unset: under the unit, only the state directory is writable. [configuration.md](configuration.md) has every key and setting. @@ -64,21 +75,25 @@ environment: .venv/bin/python util/check_config.py ~/finbot.env (set -a && . ~/finbot.env && exec .venv/bin/python bot.py) ``` -Under systemd (below), `ExecStart=` runs the venv's Python: `/home//finbot/.venv/bin/python -/home//finbot/bot.py`. +Under systemd (below), with the checkout in `/opt/finbot` and its venv made there, `ExecStart=` runs the venv's Python: +`/opt/finbot/.venv/bin/python /opt/finbot/bot.py`. ## Daemonize (systemd) -`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy -`finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of -[`finbot.env`](configuration.md#finbotenv) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start -it. `systemctl reload finbot` re-reads `settings.conf` without a restart; a change to `finbot.env` needs +`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. It expects the +checkout in `/opt/finbot` and the startup file at `/opt/finbot.env` (see [`finbot.env`](configuration.md#finbotenv)), +as above, and needs no editing then. Copy it to `/etc/systemd/system/`, then enable and start it. +`systemctl reload finbot` re-reads `settings.conf` without a restart; a change to `finbot.env` needs `systemctl restart finbot`: ``` -sudo cp -v ~/finbot/finbot.service /etc/systemd/system/ -sudo sed -i 's/CHANGEME/YOUR LINUX USERNAME/' /etc/systemd/system/finbot.service +sudo cp -v /opt/finbot/finbot.service /etc/systemd/system/ sudo systemctl daemon-reload sudo systemctl enable finbot --now ``` +The unit runs finbot as a throwaway user (`DynamicUser=yes`) with its state in `/var/lib/finbot`, inside a sandbox: a +read-only filesystem but for that directory, no access to `/home`, `/boot`, `/media` or `/mnt`, no privileges, no +executable writable memory, and only the ordinary system calls and network families. Hence the checkout must not be +under `/home`. `systemd-analyze security finbot` scores it. `MemoryMax` and `TasksMax` are starting values: after a few +days, read `systemctl show finbot -p MemoryPeak -p TasksCurrent` and adjust them. The bot's log: ``` journalctl -fu finbot diff --git a/finbot.service b/finbot.service index 3652d1e..e5068ae 100644 --- a/finbot.service +++ b/finbot.service @@ -3,13 +3,15 @@ Description=Finbot interactive chat bot [Service] Type=simple -User=CHANGEME -# Or instead of User=: DynamicUser=yes and StateDirectory=finbot (state in /var/lib/finbot) -ExecStart=/home/CHANGEME/finbot/bot.py +# A throwaway user for each start; its state lives in /var/lib/finbot (StateDirectory=). The checkout is in /opt/finbot, +# not /home: ProtectHome=yes hides /home from the service +DynamicUser=yes +StateDirectory=finbot +ExecStart=/opt/finbot/bot.py # systemctl reload finbot re-reads settings.conf without a restart ExecReload=/bin/kill -HUP $MAINPID # Refuse to start on a typo in finbot.env or settings.conf ("+" runs the check as root) -ExecStartPre=+/home/CHANGEME/finbot/util/check_config.py /opt/finbot.env +ExecStartPre=+/opt/finbot/util/check_config.py /opt/finbot.env Restart=on-failure RestartSec=30s StandardOutput=null @@ -19,5 +21,49 @@ Environment=PYTHONUNBUFFERED=1 # finbot.env: loaded by systemd as root (600 root:root); a change needs a restart EnvironmentFile=/opt/finbot.env +# Filesystem isolation +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +InaccessiblePaths=/boot /media /mnt +UMask=0077 + +# Privilege restrictions +AmbientCapabilities= +CapabilityBoundingSet= +LockPersonality=yes +# No JIT or other code generated at run time: finbot and its libraries need none +MemoryDenyWriteExecute=yes +NoNewPrivileges=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +SystemCallArchitectures=native +SystemCallErrorNumber=EPERM +SystemCallFilter=@system-service + +# Kernel and system protection +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes + +# Process isolation +PrivateUsers=yes +ProtectProc=invisible +RestrictNamespaces=yes + +# Network restrictions: HTTPS out; in push mode, finbot listens on one local port +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +# Resource limits: starting values; read MemoryPeak and TasksCurrent from systemctl show finbot, then tune. Tasks are +# threads: one per message, button press, scheduled job and chart priming, and many jobs can fire in the same minute +MemoryAccounting=yes +MemoryHigh=300M +MemoryMax=400M +TasksMax=256 + [Install] WantedBy=multi-user.target diff --git a/lib/config.py b/lib/config.py index cc39593..eadce32 100644 --- a/lib/config.py +++ b/lib/config.py @@ -37,6 +37,7 @@ RENAMED = { # startup keys under a new name: the old still works (the new wins where both are set), with a note at start 'telegramOutgoingToken': 'telegramSecretToken', # Telegram calls it the secret token } +SERVICE_DATA_DIR = '/var/lib/private/finbot' # systemd's state directory of finbot.service (StateDirectory=) LOG_LEVELS = ('ERROR', 'WARN', 'INFO', 'DEBUG') # log_level's values, the most severe first (lib/log.py) @@ -268,6 +269,13 @@ def settings_path(self): STARTUP_KEYS = ('DATA_DIR', *(field.name for field in dataclasses.fields(Env) if field.name not in DERIVED)) +def service_data_dir(): + """The service's data directory (SERVICE_DATA_DIR) for the util tools (they set FINBOT_CHECK_ONLY) run by hand + outside the service, where systemd's STATE_DIRECTORY is not set: they act on the directory the service reads, not + on var/ in the checkout. '' for the bot itself, and where the service has none.""" + return SERVICE_DATA_DIR if os.environ.get('FINBOT_CHECK_ONLY') and os.path.isdir(SERVICE_DATA_DIR) else '' + + def startup_values(getenv): """The startup settings (finbot.env), computed from getenv(name, default): each key read as its Env field's type says, its default the field's where it is unset; then those worked out from them. Raises ConfigError.""" @@ -281,7 +289,8 @@ def read(name, default): keys = {field.name: readers[field.type](field.name, field.default) for field in dataclasses.fields(Env) if field.name not in DERIVED} r.check() - data_dir = str(REPO_DIR / (getenv('DATA_DIR') or (getenv('STATE_DIRECTORY') or '').split(':')[0] or 'var')) + data_dir = str(REPO_DIR / (getenv('DATA_DIR') or (getenv('STATE_DIRECTORY') or '').split(':')[0] or + service_data_dir() or 'var')) token = keys['telegramBotToken'].rstrip('/') return Env(**keys, data_dir=data_dir, cache_dir=data_dir + '/cache', telegram_api=f'https://api.telegram.org/bot{token}/' if token else '') diff --git a/tests/test_config.py b/tests/test_config.py index 37cd9d2..f186b22 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -245,6 +245,29 @@ def test_check_config_gives_the_startup_warnings_of_a_valid_file(self): self.assertEqual(check_config.warnings(path), []) +class ServiceDataDir(unittest.TestCase): + """The util tools run by hand find the service's data directory, which the bot never looks for.""" + + def data_dir(self, getenv, check_only, service): + """The data dir startup_values gives, FINBOT_CHECK_ONLY set or not, SERVICE_DATA_DIR at `service`.""" + with mock.patch.dict(os.environ), mock.patch.object(config, 'SERVICE_DATA_DIR', service): + os.environ.pop('FINBOT_CHECK_ONLY', None) + if check_only: + os.environ['FINBOT_CHECK_ONLY'] = '1' + return config.startup_values(getenv).data_dir + + def test_the_tools_find_the_service_directory_the_bot_does_not(self): + """With the tools' FINBOT_CHECK_ONLY and a service directory, no DATA_DIR or STATE_DIRECTORY: that directory; + DATA_DIR and STATE_DIRECTORY win; the bot (no FINBOT_CHECK_ONLY) and a host without the directory keep var/.""" + service = common.temp_dir(self) + var = str(config.REPO_DIR / 'var') + self.assertEqual(self.data_dir({}.get, True, service), service) + self.assertEqual(self.data_dir({'STATE_DIRECTORY': '/state'}.get, True, service), '/state') + self.assertEqual(self.data_dir({'DATA_DIR': '/data', 'STATE_DIRECTORY': '/state'}.get, True, service), '/data') + self.assertEqual(self.data_dir({}.get, False, service), var) + self.assertEqual(self.data_dir({}.get, True, service + '/none'), var) + + class RenamedKeys(unittest.TestCase): """A startup key under a new name: the old one still works."""