From ce7bca4fbc70a036a742d3f4e5895ebee156e77f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 15:40:18 +0000 Subject: [PATCH 01/19] finbot.service is the plain unit; the sandbox and limits are an optional drop-in, finbot.service.d/hardening.conf Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/install.md | 23 +++++++++++---- finbot.service | 48 ++----------------------------- finbot.service.d/hardening.conf | 50 +++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 51 deletions(-) create mode 100644 finbot.service.d/hardening.conf diff --git a/doc/install.md b/doc/install.md index d0e01e9..d0309d7 100644 --- a/doc/install.md +++ b/doc/install.md @@ -89,11 +89,24 @@ sudo cp -v /opt/finbot/finbot.service /etc/systemd/system/ sudo systemctl daemon-reload sudo systemctl enable finbot --now ``` -The unit runs finbot as a throwaway user (`DynamicUser=yes`) with its state in `/var/lib/finbot`, inside a sandbox: a -read-only filesystem but for that directory, no access to `/home`, `/boot`, `/media` or `/mnt`, no privileges, no -executable writable memory, and only the ordinary system calls and network families. Hence the checkout must not be -under `/home`. `systemd-analyze security finbot` scores it. `MemoryMax` and `TasksMax` are starting values: after a few -days, read `systemctl show finbot -p MemoryPeak -p TasksCurrent` and adjust them. +The unit runs finbot as a throwaway user (`DynamicUser=yes`: there is no user to make) with its state in +`/var/lib/finbot`. + +### Hardening (recommended) +`finbot.service.d/hardening.conf` adds a sandbox and resource limits to the unit: a read-only filesystem but for +`/var/lib/finbot`, no access to `/home`, `/boot`, `/media` or `/mnt`, no privileges, no executable writable memory, and +only the ordinary system calls and network families, with `MemoryMax=400M` and `TasksMax=256`. Install it beside the +unit: +``` +sudo mkdir -p /etc/systemd/system/finbot.service.d +sudo cp -v /opt/finbot/finbot.service.d/hardening.conf /etc/systemd/system/finbot.service.d/ +sudo systemctl daemon-reload && sudo systemctl restart finbot +``` +It needs the checkout outside `/home` (as in `/opt/finbot`). `systemd-analyze security finbot` scores the result (about 1.3, OK, with the file; about 8, exposed, without). The +memory and task limits are starting values: after a few days, read +`systemctl show finbot -p MemoryPeak -p TasksCurrent` and adjust them. If the sandbox ever gets in the way, delete that +file and restart finbot, which leaves the plain unit. + The bot's log: ``` journalctl -fu finbot diff --git a/finbot.service b/finbot.service index e5068ae..3ebb76b 100644 --- a/finbot.service +++ b/finbot.service @@ -3,8 +3,8 @@ Description=Finbot interactive chat bot [Service] Type=simple -# A throwaway user for each start; its state lives in /var/lib/finbot (StateDirectory=). The checkout is in /opt/finbot, -# not /home: ProtectHome=yes hides /home from the service +# No user to make: systemd runs finbot as a throwaway user, and gives it /var/lib/finbot for its state +# (StateDirectory=). The checkout is in /opt/finbot, which that user can read. Optional: finbot.service.d/hardening.conf DynamicUser=yes StateDirectory=finbot ExecStart=/opt/finbot/bot.py @@ -21,49 +21,5 @@ Environment=PYTHONUNBUFFERED=1 # finbot.env: loaded by systemd as root (600 root:root); a change needs a restart EnvironmentFile=/opt/finbot.env -# Filesystem isolation -ProtectSystem=strict -ProtectHome=yes -PrivateTmp=yes -PrivateDevices=yes -InaccessiblePaths=/boot /media /mnt -UMask=0077 - -# Privilege restrictions -AmbientCapabilities= -CapabilityBoundingSet= -LockPersonality=yes -# No JIT or other code generated at run time: finbot and its libraries need none -MemoryDenyWriteExecute=yes -NoNewPrivileges=yes -RestrictRealtime=yes -RestrictSUIDSGID=yes -SystemCallArchitectures=native -SystemCallErrorNumber=EPERM -SystemCallFilter=@system-service - -# Kernel and system protection -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectKernelLogs=yes -ProtectControlGroups=yes -ProtectClock=yes -ProtectHostname=yes - -# Process isolation -PrivateUsers=yes -ProtectProc=invisible -RestrictNamespaces=yes - -# Network restrictions: HTTPS out; in push mode, finbot listens on one local port -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 - -# Resource limits: starting values; read MemoryPeak and TasksCurrent from systemctl show finbot, then tune. Tasks are -# threads: one per message, button press, scheduled job and chart priming, and many jobs can fire in the same minute -MemoryAccounting=yes -MemoryHigh=300M -MemoryMax=400M -TasksMax=256 - [Install] WantedBy=multi-user.target diff --git a/finbot.service.d/hardening.conf b/finbot.service.d/hardening.conf new file mode 100644 index 0000000..cd8621c --- /dev/null +++ b/finbot.service.d/hardening.conf @@ -0,0 +1,50 @@ +# Optional hardening for finbot.service: a sandbox and resource limits. Install it next to the unit: +# sudo mkdir -p /etc/systemd/system/finbot.service.d +# sudo cp /opt/finbot/finbot.service.d/hardening.conf /etc/systemd/system/finbot.service.d/ +# sudo systemctl daemon-reload && sudo systemctl restart finbot +# Remove the file and restart to go back to the plain unit if the sandbox ever gets in the way. It needs the checkout +# outside /home (ProtectHome=yes) and finbot's state in the unit's StateDirectory, the only place it can write. +[Service] +# Filesystem isolation +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +InaccessiblePaths=/boot /media /mnt +UMask=0077 + +# Privilege restrictions +AmbientCapabilities= +CapabilityBoundingSet= +LockPersonality=yes +# No JIT or other code generated at run time: finbot and its libraries need none +MemoryDenyWriteExecute=yes +NoNewPrivileges=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +SystemCallArchitectures=native +SystemCallErrorNumber=EPERM +SystemCallFilter=@system-service + +# Kernel and system protection +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes + +# Process isolation +PrivateUsers=yes +ProtectProc=invisible +RestrictNamespaces=yes + +# Network restrictions: HTTPS out; in push mode, finbot listens on one local port +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +# Resource limits: starting values; read MemoryPeak and TasksCurrent from systemctl show finbot, then tune. Tasks are +# threads: one per message, button press, scheduled job and chart priming, and many jobs can fire in the same minute +MemoryAccounting=yes +MemoryHigh=300M +MemoryMax=400M +TasksMax=256 From f25c31d332c041b6a4969009a0dc88233aa8be89 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 15:44:44 +0000 Subject: [PATCH 02/19] The startup file is /etc/finbot.env, for systemd and Docker alike Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 2 +- compose.yaml | 10 +++++----- defaults/finbot.env | 8 ++++---- doc/configuration.md | 8 ++++---- doc/docker.md | 25 +++++++++++++------------ doc/install.md | 13 +++++++------ finbot.service | 4 ++-- tests/test_config.py | 8 ++++---- util/check_config.py | 2 +- util/docker_setup.sh | 8 ++++---- 10 files changed, 45 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index f51647b..76d2d88 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Run finbot with Docker or with Python. On a Linux host with Docker: 1. `git clone https://github.com/robdevops/finbot /opt/finbot` 2. `sudo /opt/finbot/util/docker_setup.sh` -3. Set the keys of each network you use in `/opt/finbot.env` (below) +3. Set the keys of each network you use in `/etc/finbot.env` (below) 4. `cd /opt/finbot && sudo docker compose up -d --build` 5. `journalctl CONTAINER_NAME=finbot -f` diff --git a/compose.yaml b/compose.yaml index b29ae0f..3a10a63 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,6 @@ -# finbot with Docker Compose (doc/docker.md). The configuration lives beside the checkout, outside the image and the -# container: ../finbot.env, the startup file, and ../finbot-data, DATA_DIR (settings.conf, state and cache), both -# made by util/docker_setup.sh. Build and start: docker compose up -d --build +# finbot with Docker Compose (doc/docker.md). The configuration lives outside the image and the container: +# /etc/finbot.env, the startup file, and ../finbot-data beside the checkout, DATA_DIR (settings.conf, state and cache), +# both made by util/docker_setup.sh. Build and start: docker compose up -d --build services: finbot: build: @@ -10,13 +10,13 @@ services: image: finbot container_name: finbot restart: unless-stopped - env_file: ../finbot.env + env_file: /etc/finbot.env environment: DATA_DIR: /data ip: 0.0.0.0 # push mode listens inside the container: publish its port below volumes: - ../finbot-data:/data - - ../finbot.env:/run/finbot.env:ro # checked before the bot starts + - /etc/finbot.env:/run/finbot.env:ro # checked before the bot starts # ports: # push mode only, on the host's loopback, behind your reverse proxy # - 127.0.0.1:5000:5000 logging: diff --git a/defaults/finbot.env b/defaults/finbot.env index 2b1bbdb..e2c100e 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -1,15 +1,15 @@ # finbot.env: THE STARTUP FILE. Where finbot runs, and who it talks to. # The bot never reads this file. Your launcher loads it into the bot's environment, and a change needs a RESTART: -# systemd: EnvironmentFile=/opt/finbot.env (in finbot.service) -# docker: --env-file /opt/finbot.env -# init.d: set -a; . /opt/finbot.env; set +a; ./bot.py +# systemd: EnvironmentFile=/etc/finbot.env (in finbot.service) +# docker: --env-file /etc/finbot.env +# init.d: set -a; . /etc/finbot.env; set +a; ./bot.py # Keep it private: chmod 600, owned by root:root. It holds the credentials (check_config.py warns if not). # Flat KEY=value lines: no spaces around "=", no quotes (docker keeps quotes literally), no trailing comments; comments # on their own line. # Everything else (thresholds, windows, display, caching, ...) is in settings.conf, which the bot reloads without a # restart. # A setting key in this file is an error: finbot refuses to start. Lint both files with: -# util/check_config.py /opt/finbot.env +# util/check_config.py /etc/finbot.env # Where state and cache live. Leave unset under systemd with StateDirectory=finbot (it uses $STATE_DIRECTORY, # /var/lib/finbot, the only directory the shipped unit can write). diff --git a/doc/configuration.md b/doc/configuration.md index 03f0a02..b698d9a 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -4,14 +4,14 @@ Configuration is split into two files -- finbot.env for startup environment vari Copy the shipped template: ``` -sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env +sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env && sudo chown root:root /etc/finbot.env ``` ## `finbot.env` **Load `finbot.env` with your launcher:** -* **systemd:** `EnvironmentFile=/opt/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /opt/finbot.env` for pre-flight lint. +* **systemd:** `EnvironmentFile=/etc/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /etc/finbot.env` for pre-flight lint. * **Docker:** `compose.yaml` loads it, and checks it before the bot starts (see [docker.md](docker.md)). Write the values without quotes. -* **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). +* **init.d / shell:** `util/check_config.py /etc/finbot.env && set -a && . /etc/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). * **Scheduled jobs:** they are run by `bot.py` (see [Scheduling](scheduling.md)), so they run on the same settings. For a one-off run, send `/jobs run` in the chat. The startup keys, read by your launcher. A change needs a restart. Blank means unset. Each network's guide says which @@ -86,7 +86,7 @@ The administrators of the chat a person is in are admins there with no setting: A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. -edit `settings.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `sudo util/check_config.py /opt/finbot.env` (under systemd it reads the service's `settings.conf`). Edit `finbot.env`, then restart: a running bot cannot see it change. +edit `settings.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `sudo util/check_config.py /etc/finbot.env` (under systemd it reads the service's `settings.conf`). Edit `finbot.env`, then restart: a running bot cannot see it change. **Upgrading from a version before 16 Oct 2026?** Start commit `ff52f2b` once first, as [Scheduling](scheduling.md#from-an-earlier-version) says: it splits one old `.env` (`util/split_env.py`, which it asks for) and renames `preferences.conf` to `settings.conf`, which later versions no longer do. diff --git a/doc/docker.md b/doc/docker.md index 8d52cc3..470cd3c 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -2,13 +2,13 @@ finbot runs in a container as it runs on a Debian host: the image is `debian:trixie-slim` with Debian's own `python3-*` packages, every optional library included, so it runs every network in either mode. The image holds only the code. -The configuration lives on the host, beside the checkout, so a rebuild or an update never touches it. +The configuration lives on the host, outside the image (the startup file in `/etc`, the data beside the checkout), so a rebuild or an update never touches it. ## Quick start 1. `git clone https://github.com/robdevops/finbot /opt/finbot` 2. `sudo /opt/finbot/util/docker_setup.sh` -3. Set `telegramBotToken` in `/opt/finbot.env` (or another network's keys: see the [Telegram](telegram.md), +3. Set `telegramBotToken` in `/etc/finbot.env` (or another network's keys: see the [Telegram](telegram.md), [Slack](slack.md) and [Discord](discord.md) guides). 4. `cd /opt/finbot && sudo docker compose up -d --build` 5. `journalctl CONTAINER_NAME=finbot -f` @@ -18,12 +18,13 @@ The configuration lives on the host, beside the checkout, so a rebuild or an upd | Path | What | In the container | |---|---|---| | `/opt/finbot` | the checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | -| `/opt/finbot.env` | the startup file: the tokens, and the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | +| `/etc/finbot.env` | the startup file: the tokens, and the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | | `/opt/finbot-data` | `DATA_DIR`: `settings.conf`, each chat's jobs and watchlist, the records, the cache | `/data` | -`compose.yaml` finds the other two beside the checkout (`../finbot.env`, `../finbot-data`): clone it elsewhere and -they move with it. The startup file stays out of the image because it holds the secrets, and the data dir because it -must outlast every container. +`compose.yaml` reads the startup file at `/etc/finbot.env` and finds the data dir beside the checkout +(`../finbot-data`): clone it elsewhere and the data dir moves with it. The startup file stays out of the image because +it holds the secrets, and the data dir because it must outlast every container. Moving from an earlier version, where +the startup file was `/opt/finbot.env`: `sudo mv /opt/finbot.env /etc/finbot.env`. ## What `util/docker_setup.sh` does @@ -32,15 +33,15 @@ Run it once, as root; it is safe to run again. It: the shipped defaults into it: `settings.conf` and the data files. It never replaces a file you have changed. The script needs only the standard library, so any `python3` on the host runs it. 2. gives the data dir to the container's user: `chown -R 10001:10001 /opt/finbot-data`. -3. copies `defaults/finbot.env` to `/opt/finbot.env` if there is none, owned by `root:10001`, mode 640: root and the +3. copies `defaults/finbot.env` to `/etc/finbot.env` if there is none, owned by `root:10001`, mode 640: root and the container can read it, nobody else. By hand, that is: ``` sudo python3 /opt/finbot/util/sync_defaults.py --create --apply --dest /opt/finbot-data sudo chown -R 10001:10001 /opt/finbot-data -sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env -sudo chown root:10001 /opt/finbot.env && sudo chmod 640 /opt/finbot.env +sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env +sudo chown root:10001 /etc/finbot.env && sudo chmod 640 /etc/finbot.env ``` The startup file is `KEY=value` lines, as for systemd; write the values without quotes. @@ -76,7 +77,7 @@ sudo docker compose kill -s USR1 finbot # flush the cache, and fill it again sudo docker compose exec finbot util/flush.py --after 24h # flush the files of the last day (see caching.md) cd /opt/finbot && sudo git pull && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --build ``` -A change to `/opt/finbot.env` needs a restart: `sudo docker compose up -d`. +A change to `/etc/finbot.env` needs a restart: `sudo docker compose up -d`. ## Push mode @@ -113,12 +114,12 @@ sudo systemctl disable --now finbot sudo cp -a /var/lib/finbot/. /opt/finbot-data/ # or /opt/finbot/var/. for the default DATA_DIR sudo chown -R 10001:10001 /opt/finbot-data ``` -then `util/docker_setup.sh` (it keeps your files) and start as above. The startup file can stay at `/opt/finbot.env`. +then `util/docker_setup.sh` (it keeps your files) and start as above. The startup file is `/etc/finbot.env` (move it from `/opt/finbot.env` if that is where it was). ## Troubleshooting * **The container stops at once and keeps restarting:** the log says why. `ERROR config: ... unknown key ...` is a - mistake in `/opt/finbot.env` or `settings.conf`; fix it and `sudo docker compose up -d`. + mistake in `/etc/finbot.env` or `settings.conf`; fix it and `sudo docker compose up -d`. * **`PermissionError` on `/data`:** the data dir is not the container user's: `sudo chown -R 10001:10001 /opt/finbot-data`. * **`/data does not exist: --create makes it`:** the data dir is not mounted: `compose.yaml` mounts diff --git a/doc/install.md b/doc/install.md index d0309d7..48ccb9b 100644 --- a/doc/install.md +++ b/doc/install.md @@ -35,10 +35,10 @@ systemd, as below. To run it in a container instead, see [docker.md](docker.md). ``` 3. The startup file, from its template; then set each network's keys from its guide, and Sharesight's: ``` - sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env - sudoedit /opt/finbot.env + sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env && sudo chown root:root /etc/finbot.env + sudoedit /etc/finbot.env ``` -4. Check it: `/opt/finbot/util/check_config.py /opt/finbot.env` (it also warns if the file is not `600 root:root`) +4. Check it: `/opt/finbot/util/check_config.py /etc/finbot.env` (it also warns if the file is not `600 root:root`) 5. Run it under systemd ([below](#daemonize-systemd)). The first start makes the data directory, `/var/lib/finbot` (systemd keeps it in `/var/lib/private/finbot`, where `/var/lib/finbot` points); each chat then has the default scheduled jobs, which `/jobs` changes ([scheduling.md](scheduling.md)). @@ -48,7 +48,7 @@ systemd, as below. To run it in a container instead, see [docker.md](docker.md). ``` sudo /opt/finbot/util/sync_defaults.py --apply sudoedit /var/lib/finbot/settings.conf - sudo /opt/finbot/util/check_config.py /opt/finbot.env + sudo /opt/finbot/util/check_config.py /etc/finbot.env sudo systemctl reload finbot ``` The files are root-owned and readable, so the service reads them; systemd gives the directory to the service's user @@ -80,8 +80,9 @@ Under systemd (below), with the checkout in `/opt/finbot` and its venv made ther ## Daemonize (systemd) `finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. It expects the -checkout in `/opt/finbot` and the startup file at `/opt/finbot.env` (see [`finbot.env`](configuration.md#finbotenv)), -as above, and needs no editing then. Copy it to `/etc/systemd/system/`, then enable and start it. +checkout in `/opt/finbot` and the startup file at `/etc/finbot.env` (see [`finbot.env`](configuration.md#finbotenv)), +as above, and needs no editing then (an earlier version kept the startup file at `/opt/finbot.env`: move it with +`sudo mv /opt/finbot.env /etc/finbot.env`). Copy the unit to `/etc/systemd/system/`, then enable and start it. `systemctl reload finbot` re-reads `settings.conf` without a restart; a change to `finbot.env` needs `systemctl restart finbot`: ``` diff --git a/finbot.service b/finbot.service index 3ebb76b..895de50 100644 --- a/finbot.service +++ b/finbot.service @@ -11,7 +11,7 @@ ExecStart=/opt/finbot/bot.py # systemctl reload finbot re-reads settings.conf without a restart ExecReload=/bin/kill -HUP $MAINPID # Refuse to start on a typo in finbot.env or settings.conf ("+" runs the check as root) -ExecStartPre=+/opt/finbot/util/check_config.py /opt/finbot.env +ExecStartPre=+/opt/finbot/util/check_config.py /etc/finbot.env Restart=on-failure RestartSec=30s StandardOutput=null @@ -19,7 +19,7 @@ StandardError=journal SyslogIdentifier=finbot Environment=PYTHONUNBUFFERED=1 # finbot.env: loaded by systemd as root (600 root:root); a change needs a restart -EnvironmentFile=/opt/finbot.env +EnvironmentFile=/etc/finbot.env [Install] WantedBy=multi-user.target diff --git a/tests/test_config.py b/tests/test_config.py index f186b22..10fcfc4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -224,11 +224,11 @@ def stat_of(mode, uid=0, gid=0): """What os.stat says of a file with this mode, owner and group.""" return types.SimpleNamespace(st_mode=0o100000 | mode, st_uid=uid, st_gid=gid) with mock.patch.object(check_config.os, 'stat', return_value=stat_of(0o600)): - self.assertEqual(check_config.permission_warnings('/opt/finbot.env'), []) + self.assertEqual(check_config.permission_warnings('/etc/finbot.env'), []) with mock.patch.object(check_config.os, 'stat', return_value=stat_of(0o644)): - warning, = check_config.permission_warnings('/opt/finbot.env') - self.assertIn('/opt/finbot.env is 644 root:root, not 600 root:root', warning) - self.assertIn('chmod 600 /opt/finbot.env && chown root:root /opt/finbot.env', warning) + warning, = check_config.permission_warnings('/etc/finbot.env') + self.assertIn('/etc/finbot.env is 644 root:root, not 600 root:root', warning) + self.assertIn('chmod 600 /etc/finbot.env && chown root:root /etc/finbot.env', warning) with mock.patch.object(check_config.os, 'stat', return_value=stat_of(0o600, uid=54321, gid=54321)): self.assertIn('600 54321:54321, not 600 root:root', check_config.permission_warnings('x')[0]) self.assertEqual(check_config.permission_warnings('/no/such/finbot.env'), []) diff --git a/util/check_config.py b/util/check_config.py index 0f0466a..868ad64 100755 --- a/util/check_config.py +++ b/util/check_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Lint finbot's two config files with the rules the bot applies, without starting it. - util/check_config.py /opt/finbot.env [/var/lib/finbot/settings.conf] + util/check_config.py /etc/finbot.env [/var/lib/finbot/settings.conf] finbot.env (the startup file) is never read by the bot, so this is the only place its typos are caught: an unknown key, a setting key in it, a malformed line or a bad value. diff --git a/util/docker_setup.sh b/util/docker_setup.sh index 312d5fd..d6ba68f 100755 --- a/util/docker_setup.sh +++ b/util/docker_setup.sh @@ -1,14 +1,14 @@ #!/bin/sh # The host side of finbot's Docker install, set up once, as root; safe to run again (doc/docker.md). Beside the -# checkout, where compose.yaml looks for them: ../finbot-data (DATA_DIR) made, the defaults copied in, owned by the -# container's user (10001); and ../finbot.env from the template if there is none, readable by root and the container's -# group only. Cloned to /opt/finbot, they are /opt/finbot-data and /opt/finbot.env. +# checkout, where compose.yaml looks for it: ../finbot-data (DATA_DIR) made, the defaults copied in, owned by the +# container's user (10001), so /opt/finbot-data for a checkout in /opt/finbot; and /etc/finbot.env, the startup file, +# from the template if there is none, readable by root and the container's group only. set -eu repo=$(cd "$(dirname "$0")/.." && pwd) parent=$(dirname "$repo") data="$parent/finbot-data" -startup="$parent/finbot.env" +startup=/etc/finbot.env if [ "$(id -u)" -ne 0 ]; then echo "docker_setup: run it as root: sudo $0" >&2 From b3c7fa878bbb92c7c4ad2efd12da6fdd2efdf747 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 15:54:13 +0000 Subject: [PATCH 03/19] doc/install.md: a quick start in shorter sentences, with a template, your chatbot's API key, Slack or Discord in pull mode, the Git checkout Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/configuration.md | 2 +- doc/docker.md | 15 ++-- doc/install.md | 150 ++++++++++++++++++++------------ finbot.service | 3 +- finbot.service.d/hardening.conf | 5 +- util/docker_setup.sh | 2 +- 6 files changed, 106 insertions(+), 71 deletions(-) diff --git a/doc/configuration.md b/doc/configuration.md index b698d9a..b48653c 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -95,7 +95,7 @@ All state and cache lives in one data directory: `DATA_DIR` if set, else systemd Reference data (`finbot_adr.json`, `finbot_sws_*.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: ``` util/sync_defaults.py # dry run -util/sync_defaults.py --apply # copy missing files (including a starter settings.conf) +util/sync_defaults.py --apply # copy missing files (including the settings.conf template) util/sync_defaults.py --force # also replace changed files, keeping .bak copies util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory util/sync_defaults.py --dest /srv/finbot --create --apply # and make it (mode 750) if it does not exist yet diff --git a/doc/docker.md b/doc/docker.md index 470cd3c..186ee6c 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -2,14 +2,14 @@ finbot runs in a container as it runs on a Debian host: the image is `debian:trixie-slim` with Debian's own `python3-*` packages, every optional library included, so it runs every network in either mode. The image holds only the code. -The configuration lives on the host, outside the image (the startup file in `/etc`, the data beside the checkout), so a rebuild or an update never touches it. +The configuration lives on the host, outside the image (the startup file in `/etc`, the data beside the Git checkout), so a rebuild or an update never touches it. ## Quick start 1. `git clone https://github.com/robdevops/finbot /opt/finbot` 2. `sudo /opt/finbot/util/docker_setup.sh` -3. Set `telegramBotToken` in `/etc/finbot.env` (or another network's keys: see the [Telegram](telegram.md), - [Slack](slack.md) and [Discord](discord.md) guides). +3. Set your chatbot's API key in `/etc/finbot.env` (`telegramBotToken` for Telegram; see the + [Telegram](telegram.md), [Slack](slack.md) and [Discord](discord.md) guides). 4. `cd /opt/finbot && sudo docker compose up -d --build` 5. `journalctl CONTAINER_NAME=finbot -f` @@ -17,14 +17,13 @@ The configuration lives on the host, outside the image (the startup file in `/et | Path | What | In the container | |---|---|---| -| `/opt/finbot` | the checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | +| `/opt/finbot` | the Git checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | | `/etc/finbot.env` | the startup file: the tokens, and the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | | `/opt/finbot-data` | `DATA_DIR`: `settings.conf`, each chat's jobs and watchlist, the records, the cache | `/data` | -`compose.yaml` reads the startup file at `/etc/finbot.env` and finds the data dir beside the checkout +`compose.yaml` reads the startup file at `/etc/finbot.env` and finds the data dir beside the Git checkout (`../finbot-data`): clone it elsewhere and the data dir moves with it. The startup file stays out of the image because -it holds the secrets, and the data dir because it must outlast every container. Moving from an earlier version, where -the startup file was `/opt/finbot.env`: `sudo mv /opt/finbot.env /etc/finbot.env`. +it holds the secrets, and the data dir because it must outlast every container.. ## What `util/docker_setup.sh` does @@ -114,7 +113,7 @@ sudo systemctl disable --now finbot sudo cp -a /var/lib/finbot/. /opt/finbot-data/ # or /opt/finbot/var/. for the default DATA_DIR sudo chown -R 10001:10001 /opt/finbot-data ``` -then `util/docker_setup.sh` (it keeps your files) and start as above. The startup file is `/etc/finbot.env` (move it from `/opt/finbot.env` if that is where it was). +then `util/docker_setup.sh` (it keeps your files) and start as above. ## Troubleshooting diff --git a/doc/install.md b/doc/install.md index 48ccb9b..d2043f6 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,114 +1,148 @@ # Installing on a host -finbot runs wherever Python 3.12 or later does (Linux, macOS, WSL); production runs Debian's own packages under -systemd, as below. To run it in a container instead, see [docker.md](docker.md). +finbot runs wherever Python 3.12 or later does (Linux, macOS, WSL). Production runs Debian's own packages under +systemd, as below. + +To run it in a container instead, see [docker.md](docker.md). ## Requirements -* Most features need a Sharesight paid plan, preferably with automatic trade imports, and an API key - ([configuration.md](configuration.md#sharesight)). -* A Telegram bot, a Slack app or a Discord bot: see the network's guide, [Telegram](telegram.md), [Slack](slack.md) or + +* Most features need a Sharesight paid plan and an API key ([configuration.md](configuration.md#sharesight)). + +* A Telegram bot, a Slack app or a Discord bot: see its guide, [Telegram](telegram.md), [Slack](slack.md) or [Discord](discord.md). -* By default, finbot fetches its updates from each network itself (pull mode: Telegram's long polling, Slack's Socket - Mode, Discord's gateway API). It only makes outgoing connections, so nothing has to listen on the internet. - Alternatively, if you prefer to run an HTTPS gateway (a web server with a domain name and a matching certificate), - Push mode subscribes the bot to push updates: each network posts its updates to the gateway, which passes them to - the bot (each guide's _push mode_). -* Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum), and these modules, pinned to versions with - 3.13 support in `requirements.txt`: requests, and Pillow for the charts. The charts are drawn in DejaVu Sans, which - most Linux systems have (Debian: `fonts-dejavu-core`); without it, Pillow's own font stands in. -* For a network in push mode (its webhook URL set), also waitress; for Slack's Socket Mode and Discord's gateway API, - websocket-client; for Discord's push mode, PyNaCl. These are in `requirements-optional.txt`, each imported only by - the modes that need it. Docker and a Python venv install them all, Docker in its image and a venv - from that file, so every mode works without choosing; on a Debian host, install the packages of the modes you use. + +* Python 3.13.5 (3.12 is the minimum), with `requests` and Pillow for the charts. The charts use DejaVu Sans, which most + Linux systems have (Debian: `fonts-dejavu-core`). + +* By default, finbot fetches its updates from your chatbot itself (pull mode). It only makes outgoing connections, so + nothing has to listen on the internet. + + Push mode is the alternative: your chatbot posts its updates to an HTTPS server you run. Each guide covers it. + +* One more library for the way you receive updates: + + * `waitress`, for push mode + * `websocket-client`, for Slack or Discord in pull mode + * `PyNaCl`, for Discord in push mode + + Docker and the venv below install them all. On Debian, install the ones you need. ## Debian -1. The packages, and those of the modes you use: + +1. Install the packages, and those for the way you receive updates: + ``` sudo apt install git python3 python3-requests python3-pil fonts-dejavu-core - sudo apt install python3-waitress # any network in push mode - sudo apt install python3-websocket # Slack's Socket Mode, Discord's gateway API - sudo apt install python3-nacl # Discord's push mode + sudo apt install python3-waitress # push mode + sudo apt install python3-websocket # Slack or Discord in pull mode + sudo apt install python3-nacl # Discord in push mode ``` -2. The code, in `/opt/finbot` (not under `/home`: the service unit hides it): + +2. Get the code, in `/opt/finbot`: + ``` sudo git clone https://github.com/robdevops/finbot.git /opt/finbot ``` -3. The startup file, from its template; then set each network's keys from its guide, and Sharesight's: + +3. Make the startup file from its template: + ``` sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env && sudo chown root:root /etc/finbot.env + ``` + + Then add your chatbot's API key (see its guide above), and Sharesight's: + + ``` sudoedit /etc/finbot.env ``` -4. Check it: `/opt/finbot/util/check_config.py /etc/finbot.env` (it also warns if the file is not `600 root:root`) -5. Run it under systemd ([below](#daemonize-systemd)). The first start makes the data directory, `/var/lib/finbot` - (systemd keeps it in `/var/lib/private/finbot`, where `/var/lib/finbot` points); each chat then has the default - scheduled jobs, which `/jobs` changes ([scheduling.md](scheduling.md)). -6. Optional, to change settings: copy the starter `settings.conf` into the data directory, edit it, check it and reload - (`util/sync_defaults.py` and `util/check_config.py` find the service's data directory when run by hand; run them - with `sudo`, as the directory is the service's): + +4. Check it: + + ``` + /opt/finbot/util/check_config.py /etc/finbot.env + ``` + +5. Run it under systemd, [below](#daemonize-systemd). + +6. Optional, to change settings. Copy the `settings.conf` template, edit it, check it, and reload: + ``` sudo /opt/finbot/util/sync_defaults.py --apply sudoedit /var/lib/finbot/settings.conf sudo /opt/finbot/util/check_config.py /etc/finbot.env sudo systemctl reload finbot ``` - The files are root-owned and readable, so the service reads them; systemd gives the directory to the service's user - again at the next start. Leave `DATA_DIR` unset: under the unit, only the state directory is writable. [configuration.md](configuration.md) has every key and setting. ## Anywhere else: a Python venv -On Linux, macOS or WSL, with git and Python 3.12 or later (on Debian and Ubuntu, also `python3-venv`): + +For macOS, WSL, or any setup without the Debian packages. You need git and Python 3.12 or later. + ``` +# Debian or Ubuntu only: sudo apt install python3-venv git clone https://github.com/robdevops/finbot.git ~/finbot cd ~/finbot python3 -m venv .venv .venv/bin/pip install -r requirements.txt -r requirements-optional.txt ``` -Or, with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: + +Or with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: + ``` uv venv && uv pip install -r requirements.txt -r requirements-optional.txt ``` -`requirements-optional.txt` installs the library of every mode; it can be cut down to the modes you use. Then make the -startup file as in step 3 above (anywhere: `~/finbot.env`, say), check it, and start the bot with its values in its -environment: + +Make the startup file as in step 3 above (anywhere: `~/finbot.env`, say), check it, and start the bot: + ``` .venv/bin/python util/check_config.py ~/finbot.env (set -a && . ~/finbot.env && exec .venv/bin/python bot.py) ``` -Under systemd (below), with the checkout in `/opt/finbot` and its venv made there, `ExecStart=` runs the venv's Python: + +To run the venv under systemd, put the Git checkout in `/opt/finbot`, make the venv there, and set `ExecStart=` to `/opt/finbot/.venv/bin/python /opt/finbot/bot.py`. ## Daemonize (systemd) -`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. It expects the -checkout in `/opt/finbot` and the startup file at `/etc/finbot.env` (see [`finbot.env`](configuration.md#finbotenv)), -as above, and needs no editing then (an earlier version kept the startup file at `/opt/finbot.env`: move it with -`sudo mv /opt/finbot.env /etc/finbot.env`). Copy the unit to `/etc/systemd/system/`, then enable and start it. -`systemctl reload finbot` re-reads `settings.conf` without a restart; a change to `finbot.env` needs -`systemctl restart finbot`: + +`finbot.service` keeps `bot.py` running in the background and starts it on boot. + +It expects the Git checkout in `/opt/finbot` and the startup file at `/etc/finbot.env`, as above. It needs no editing. + ``` sudo cp -v /opt/finbot/finbot.service /etc/systemd/system/ sudo systemctl daemon-reload sudo systemctl enable finbot --now ``` -The unit runs finbot as a throwaway user (`DynamicUser=yes`: there is no user to make) with its state in -`/var/lib/finbot`. + +finbot's data (its state and settings) is in `/var/lib/finbot`. + +After a change, `sudo systemctl reload finbot` re-reads `settings.conf`. A change to `finbot.env` needs +`sudo systemctl restart finbot`. + +The log: + +``` +journalctl -fu finbot +``` ### Hardening (recommended) -`finbot.service.d/hardening.conf` adds a sandbox and resource limits to the unit: a read-only filesystem but for -`/var/lib/finbot`, no access to `/home`, `/boot`, `/media` or `/mnt`, no privileges, no executable writable memory, and -only the ordinary system calls and network families, with `MemoryMax=400M` and `TasksMax=256`. Install it beside the -unit: + +An optional file adds a sandbox and resource limits to the unit. finbot can then only write to `/var/lib/finbot`, and +gets no privileges. + ``` sudo mkdir -p /etc/systemd/system/finbot.service.d sudo cp -v /opt/finbot/finbot.service.d/hardening.conf /etc/systemd/system/finbot.service.d/ sudo systemctl daemon-reload && sudo systemctl restart finbot ``` -It needs the checkout outside `/home` (as in `/opt/finbot`). `systemd-analyze security finbot` scores the result (about 1.3, OK, with the file; about 8, exposed, without). The -memory and task limits are starting values: after a few days, read -`systemctl show finbot -p MemoryPeak -p TasksCurrent` and adjust them. If the sandbox ever gets in the way, delete that -file and restart finbot, which leaves the plain unit. -The bot's log: -``` -journalctl -fu finbot -``` +It needs the Git checkout outside `/home`, as in `/opt/finbot`. + +`systemd-analyze security finbot` scores the result: about 1.3 (OK) with the file, about 8 (exposed) without. + +The memory and task limits are starting values. After a few days, check +`systemctl show finbot -p MemoryPeak -p TasksCurrent` and adjust them. + +If the sandbox ever gets in the way, delete the file and restart finbot. diff --git a/finbot.service b/finbot.service index 895de50..14f0414 100644 --- a/finbot.service +++ b/finbot.service @@ -4,7 +4,8 @@ Description=Finbot interactive chat bot [Service] Type=simple # No user to make: systemd runs finbot as a throwaway user, and gives it /var/lib/finbot for its state -# (StateDirectory=). The checkout is in /opt/finbot, which that user can read. Optional: finbot.service.d/hardening.conf +# (StateDirectory=). The Git checkout is in /opt/finbot, which that user can read. +# Optional hardening: finbot.service.d/hardening.conf DynamicUser=yes StateDirectory=finbot ExecStart=/opt/finbot/bot.py diff --git a/finbot.service.d/hardening.conf b/finbot.service.d/hardening.conf index cd8621c..9a88f1b 100644 --- a/finbot.service.d/hardening.conf +++ b/finbot.service.d/hardening.conf @@ -2,8 +2,9 @@ # sudo mkdir -p /etc/systemd/system/finbot.service.d # sudo cp /opt/finbot/finbot.service.d/hardening.conf /etc/systemd/system/finbot.service.d/ # sudo systemctl daemon-reload && sudo systemctl restart finbot -# Remove the file and restart to go back to the plain unit if the sandbox ever gets in the way. It needs the checkout -# outside /home (ProtectHome=yes) and finbot's state in the unit's StateDirectory, the only place it can write. +# Remove the file and restart to go back to the plain unit if the sandbox ever gets in the way. +# It needs the Git checkout outside /home (ProtectHome=yes) and finbot's state in the unit's StateDirectory, the only +# place it can write. [Service] # Filesystem isolation ProtectSystem=strict diff --git a/util/docker_setup.sh b/util/docker_setup.sh index d6ba68f..b686ef4 100755 --- a/util/docker_setup.sh +++ b/util/docker_setup.sh @@ -28,7 +28,7 @@ chmod 640 "$startup" cat < Date: Fri, 23 Oct 2026 15:54:52 +0000 Subject: [PATCH 04/19] doc: drop the repeated pointer to each guide's push mode Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 2 +- doc/install.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76d2d88..3c5a9fd 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ To run it as a service, under systemd on a Debian host, see [doc/install.md](doc ### The chat networks By default, finbot fetches its updates from each network itself (pull mode), which is what this quick start guide covers. Alternatively, if you prefer to run an HTTPS gateway (a reverse proxy on a domain with a valid certificate), push mode -subscribes the bot to the updates each network pushes to it (see each guide's _push mode_). +subscribes the bot to the updates each network pushes to it. * **Telegram:** create the bot with [@BotFather](https://telegram.me/BotFather), set `telegramBotToken`, restart finbot ([guide](doc/telegram.md)). * **Slack:** create the app from the manifest that `util/slack_manifest.py` prints, add an app-level token, set diff --git a/doc/install.md b/doc/install.md index d2043f6..bb281c1 100644 --- a/doc/install.md +++ b/doc/install.md @@ -18,7 +18,7 @@ To run it in a container instead, see [docker.md](docker.md). * By default, finbot fetches its updates from your chatbot itself (pull mode). It only makes outgoing connections, so nothing has to listen on the internet. - Push mode is the alternative: your chatbot posts its updates to an HTTPS server you run. Each guide covers it. + Push mode is the alternative: your chatbot posts its updates to an HTTPS server you run. * One more library for the way you receive updates: From d33242a401a2244aa6fec30eba6121cf336e4b7b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:04:48 +0000 Subject: [PATCH 05/19] doc/install.md: Debian-first wording, spaced apt commands, the venv example on the service's paths; the unit uses /opt/finbot/.venv when there is one Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/install.md | 51 ++++++++++++++++++++++++++++---------------------- finbot.service | 2 ++ 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/doc/install.md b/doc/install.md index bb281c1..2d7def9 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,7 +1,7 @@ # Installing on a host -finbot runs wherever Python 3.12 or later does (Linux, macOS, WSL). Production runs Debian's own packages under -systemd, as below. +finbot runs wherever Python 3.12 or later does (Linux, macOS, WSL). It is developed and most tested on Debian with +Debian's packaged python modules. To run it in a container instead, see [docker.md](docker.md). @@ -20,11 +20,11 @@ To run it in a container instead, see [docker.md](docker.md). Push mode is the alternative: your chatbot posts its updates to an HTTPS server you run. -* One more library for the way you receive updates: +* One more library, for the way you receive updates: - * `waitress`, for push mode - * `websocket-client`, for Slack or Discord in pull mode - * `PyNaCl`, for Discord in push mode + * `waitress` for push mode + * `websocket-client` for Slack or Discord in pull mode + * `PyNaCl` for Discord in push mode Docker and the venv below install them all. On Debian, install the ones you need. @@ -34,9 +34,15 @@ To run it in a container instead, see [docker.md](docker.md). ``` sudo apt install git python3 python3-requests python3-pil fonts-dejavu-core - sudo apt install python3-waitress # push mode - sudo apt install python3-websocket # Slack or Discord in pull mode - sudo apt install python3-nacl # Discord in push mode + + # shared dependency for push modes + sudo apt install python3-waitress + + # Slack and Discord in pull mode + sudo apt install python3-websocket + + # Discord in push mode + sudo apt install python3-nacl ``` 2. Get the code, in `/opt/finbot`: @@ -80,35 +86,36 @@ To run it in a container instead, see [docker.md](docker.md). For macOS, WSL, or any setup without the Debian packages. You need git and Python 3.12 or later. +Use the same paths as the Debian steps, so the service below works unchanged: + ``` # Debian or Ubuntu only: sudo apt install python3-venv -git clone https://github.com/robdevops/finbot.git ~/finbot -cd ~/finbot -python3 -m venv .venv -.venv/bin/pip install -r requirements.txt -r requirements-optional.txt +sudo git clone https://github.com/robdevops/finbot.git /opt/finbot +cd /opt/finbot +sudo python3 -m venv .venv +sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt ``` -Or with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: +Make the startup file as in step 3 above, then check it: ``` -uv venv && uv pip install -r requirements.txt -r requirements-optional.txt +sudo /opt/finbot/.venv/bin/python /opt/finbot/util/check_config.py /etc/finbot.env ``` -Make the startup file as in step 3 above (anywhere: `~/finbot.env`, say), check it, and start the bot: +To try the bot by hand: ``` -.venv/bin/python util/check_config.py ~/finbot.env -(set -a && . ~/finbot.env && exec .venv/bin/python bot.py) +sudo sh -c 'set -a && . /etc/finbot.env && exec /opt/finbot/.venv/bin/python /opt/finbot/bot.py' ``` -To run the venv under systemd, put the Git checkout in `/opt/finbot`, make the venv there, and set `ExecStart=` to -`/opt/finbot/.venv/bin/python /opt/finbot/bot.py`. +For a service, use systemd, below. ## Daemonize (systemd) -`finbot.service` keeps `bot.py` running in the background and starts it on boot. +`finbot.service` runs finbot as a systemd service: it starts on boot, and restarts after a failure. -It expects the Git checkout in `/opt/finbot` and the startup file at `/etc/finbot.env`, as above. It needs no editing. +It expects the Git checkout in `/opt/finbot` and the startup file at `/etc/finbot.env`, as above. It needs no editing, +and uses the venv in `/opt/finbot/.venv` if you made one. ``` sudo cp -v /opt/finbot/finbot.service /etc/systemd/system/ diff --git a/finbot.service b/finbot.service index 14f0414..ab9a991 100644 --- a/finbot.service +++ b/finbot.service @@ -19,6 +19,8 @@ StandardOutput=null StandardError=journal SyslogIdentifier=finbot Environment=PYTHONUNBUFFERED=1 +# The venv in /opt/finbot/.venv, if there is one, is the python3 that bot.py finds +Environment=PATH=/opt/finbot/.venv/bin:/usr/local/bin:/usr/bin:/bin # finbot.env: loaded by systemd as root (600 root:root); a change needs a restart EnvironmentFile=/etc/finbot.env From 7770fe47a100b12b6c4afce36927c803a824cba9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:07:31 +0000 Subject: [PATCH 06/19] doc/install.md: the apt commands by network and mode Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/install.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/install.md b/doc/install.md index 2d7def9..85e9992 100644 --- a/doc/install.md +++ b/doc/install.md @@ -35,14 +35,14 @@ To run it in a container instead, see [docker.md](docker.md). ``` sudo apt install git python3 python3-requests python3-pil fonts-dejavu-core - # shared dependency for push modes - sudo apt install python3-waitress - - # Slack and Discord in pull mode + # Slack or Discord in pull mode sudo apt install python3-websocket + # Telegram or Slack in push mode + sudo apt install python3-waitress + # Discord in push mode - sudo apt install python3-nacl + sudo apt install python3-nacl python3-waitress ``` 2. Get the code, in `/opt/finbot`: From b7a56d820eea8491870005ba429dfa88e8dd5258 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:08:26 +0000 Subject: [PATCH 07/19] doc/install.md: step 1 names the core dependencies and the chat methods Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install.md b/doc/install.md index 85e9992..3a7428a 100644 --- a/doc/install.md +++ b/doc/install.md @@ -30,7 +30,7 @@ To run it in a container instead, see [docker.md](docker.md). ## Debian -1. Install the packages, and those for the way you receive updates: +1. Install the core dependencies, plus those for one or more chat methods: ``` sudo apt install git python3 python3-requests python3-pil fonts-dejavu-core From 986e4986960d1eb44ea10d34bb98db91cde28361 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:11:05 +0000 Subject: [PATCH 08/19] doc/install.md: the venv way names no Debian package Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install.md b/doc/install.md index 3a7428a..f70d0c8 100644 --- a/doc/install.md +++ b/doc/install.md @@ -84,12 +84,12 @@ To run it in a container instead, see [docker.md](docker.md). ## Anywhere else: a Python venv -For macOS, WSL, or any setup without the Debian packages. You need git and Python 3.12 or later. +For macOS, WSL, or any setup that does not use the Debian packages. You need git, and Python 3.12 or later with its +`venv` module. Use the same paths as the Debian steps, so the service below works unchanged: ``` -# Debian or Ubuntu only: sudo apt install python3-venv sudo git clone https://github.com/robdevops/finbot.git /opt/finbot cd /opt/finbot sudo python3 -m venv .venv From 30390870fa12dfbb74f0803fce2c8f29d3a64214 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:16:23 +0000 Subject: [PATCH 09/19] The startup file is called the startup env Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- .github/workflows/ci.yml | 6 +++--- Dockerfile | 4 ++-- compose.yaml | 2 +- defaults/finbot.env | 2 +- doc/docker.md | 12 ++++++------ doc/install.md | 6 +++--- lib/config.py | 2 +- tests/test_config.py | 2 +- tests/test_datadir.py | 2 +- util/check_config.py | 4 ++-- util/docker_setup.sh | 2 +- util/sync_defaults.py | 2 +- 12 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc789ba..1fdaa85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,19 +126,19 @@ jobs: wait_for 'cache: flushed' docker rm -f finbot - if: steps.changes.outputs.docs_only != 'true' - name: A mistake in the startup file stops the container before the bot starts + name: A mistake in the startup env stops the container before the bot starts run: | printf 'telegramBotTokn=x\n' > "$RUNNER_TEMP/bad.env" if docker run --rm --read-only --tmpfs /tmp -e DATA_DIR=/tmp \ -v "$RUNNER_TEMP/bad.env:/run/finbot.env:ro" finbot > "$RUNNER_TEMP/out" 2>&1; then cat "$RUNNER_TEMP/out" - echo "::error::the container started with a mistake in its startup file" + echo "::error::the container started with a mistake in its startup env" exit 1 fi cat "$RUNNER_TEMP/out" grep -q 'unknown key telegramBotTokn' "$RUNNER_TEMP/out" if grep -q 'finbot is running' "$RUNNER_TEMP/out"; then - echo "::error::the bot started with a mistake in its startup file" + echo "::error::the bot started with a mistake in its startup env" exit 1 fi diff --git a/Dockerfile b/Dockerfile index 0c4f32b..d892563 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # finbot in a container, on Debian with Debian's own python3 packages (no pip), as on a Debian host. Every optional -# library is in, so the image runs every network in either mode. The startup file and DATA_DIR stay outside the image: +# library is in, so the image runs every network in either mode. The startup env and DATA_DIR stay outside the image: # see compose.yaml and doc/docker.md. FROM debian:trixie-slim @@ -23,5 +23,5 @@ ARG GIT_COMMIT=unknown ENV FINBOT_COMMIT=$GIT_COMMIT PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 DATA_DIR=/data HOME=/tmp USER finbot -# each start: copy the defaults missing from /data, check the startup file, and only then start the bot +# each start: copy the defaults missing from /data, check the startup env, and only then start the bot CMD ["sh", "-c", "util/sync_defaults.py --apply && util/check_config.py /run/finbot.env && exec ./bot.py"] diff --git a/compose.yaml b/compose.yaml index 3a10a63..d7de113 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,5 @@ # finbot with Docker Compose (doc/docker.md). The configuration lives outside the image and the container: -# /etc/finbot.env, the startup file, and ../finbot-data beside the checkout, DATA_DIR (settings.conf, state and cache), +# /etc/finbot.env, the startup env, and ../finbot-data beside the checkout, DATA_DIR (settings.conf, state and cache), # both made by util/docker_setup.sh. Build and start: docker compose up -d --build services: finbot: diff --git a/defaults/finbot.env b/defaults/finbot.env index e2c100e..81e4bac 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -1,4 +1,4 @@ -# finbot.env: THE STARTUP FILE. Where finbot runs, and who it talks to. +# finbot.env: THE STARTUP ENV. Where finbot runs, and who it talks to. # The bot never reads this file. Your launcher loads it into the bot's environment, and a change needs a RESTART: # systemd: EnvironmentFile=/etc/finbot.env (in finbot.service) # docker: --env-file /etc/finbot.env diff --git a/doc/docker.md b/doc/docker.md index 186ee6c..4196291 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -2,7 +2,7 @@ finbot runs in a container as it runs on a Debian host: the image is `debian:trixie-slim` with Debian's own `python3-*` packages, every optional library included, so it runs every network in either mode. The image holds only the code. -The configuration lives on the host, outside the image (the startup file in `/etc`, the data beside the Git checkout), so a rebuild or an update never touches it. +The configuration lives on the host, outside the image (the startup env in `/etc`, the data beside the Git checkout), so a rebuild or an update never touches it. ## Quick start @@ -18,11 +18,11 @@ The configuration lives on the host, outside the image (the startup file in `/et | Path | What | In the container | |---|---|---| | `/opt/finbot` | the Git checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | -| `/etc/finbot.env` | the startup file: the tokens, and the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | +| `/etc/finbot.env` | the startup env: the tokens, and the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | | `/opt/finbot-data` | `DATA_DIR`: `settings.conf`, each chat's jobs and watchlist, the records, the cache | `/data` | -`compose.yaml` reads the startup file at `/etc/finbot.env` and finds the data dir beside the Git checkout -(`../finbot-data`): clone it elsewhere and the data dir moves with it. The startup file stays out of the image because +`compose.yaml` reads the startup env at `/etc/finbot.env` and finds the data dir beside the Git checkout +(`../finbot-data`): clone it elsewhere and the data dir moves with it. The startup env stays out of the image because it holds the secrets, and the data dir because it must outlast every container.. ## What `util/docker_setup.sh` does @@ -43,7 +43,7 @@ sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env sudo chown root:10001 /etc/finbot.env && sudo chmod 640 /etc/finbot.env ``` -The startup file is `KEY=value` lines, as for systemd; write the values without quotes. +The startup env is `KEY=value` lines, as for systemd; write the values without quotes. ## Build and start @@ -55,7 +55,7 @@ cd /opt/finbot && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --b Each start of the container, in order: 1. `util/sync_defaults.py --apply` copies into `/data` any default it lacks, such as a data file added by an update; -2. `util/check_config.py /run/finbot.env` checks the startup file and `settings.conf`. A mistake stops the start +2. `util/check_config.py /run/finbot.env` checks the startup env and `settings.conf`. A mistake stops the start here, and the log says which key and why; 3. the bot starts. diff --git a/doc/install.md b/doc/install.md index f70d0c8..a0b2a94 100644 --- a/doc/install.md +++ b/doc/install.md @@ -51,7 +51,7 @@ To run it in a container instead, see [docker.md](docker.md). sudo git clone https://github.com/robdevops/finbot.git /opt/finbot ``` -3. Make the startup file from its template: +3. Make the startup env from its template: ``` sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env && sudo chown root:root /etc/finbot.env @@ -96,7 +96,7 @@ sudo python3 -m venv .venv sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt ``` -Make the startup file as in step 3 above, then check it: +Make the startup env as in step 3 above, then check it: ``` sudo /opt/finbot/.venv/bin/python /opt/finbot/util/check_config.py /etc/finbot.env @@ -114,7 +114,7 @@ For a service, use systemd, below. `finbot.service` runs finbot as a systemd service: it starts on boot, and restarts after a failure. -It expects the Git checkout in `/opt/finbot` and the startup file at `/etc/finbot.env`, as above. It needs no editing, +It expects the Git checkout in `/opt/finbot` and the startup env at `/etc/finbot.env`, as above. It needs no editing, and uses the venv in `/opt/finbot/.venv` if you made one. ``` diff --git a/lib/config.py b/lib/config.py index eadce32..44fb134 100644 --- a/lib/config.py +++ b/lib/config.py @@ -8,7 +8,7 @@ from pathlib import Path # TO CONFIGURE, DO NOT EDIT THIS FILE. There are two config files: -# finbot.env the startup file: DATA_DIR, ip, port, credentials and the mode of each network. Never read by the +# finbot.env the startup env: DATA_DIR, ip, port, credentials and the mode of each network. Never read by the # bot: the launcher loads it into the environment (systemd EnvironmentFile=, docker --env-file, # init.d `set -a; . finbot.env`). A change needs a restart. # settings.conf how the bot behaves: /settings.conf, read by the bot itself at start and on every diff --git a/tests/test_config.py b/tests/test_config.py index 10fcfc4..015e389 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,4 @@ -"""The two config files: finbot.env (the startup file, which the launcher loads) and settings.conf (reloadable).""" +"""The two config files: finbot.env (the startup env, which the launcher loads) and settings.conf (reloadable).""" import ast, contextlib, dataclasses, io, os, re, shutil, subprocess, sys, types, unittest from pathlib import Path from unittest import mock diff --git a/tests/test_datadir.py b/tests/test_datadir.py index 8930db0..ac84ce4 100644 --- a/tests/test_datadir.py +++ b/tests/test_datadir.py @@ -94,7 +94,7 @@ def test_sync_defaults(self): copied = Path(config.env.data_dir, 'settings.conf') # the template as it is: every key commented out self.assertEqual(copied.read_bytes(), (config.REPO_DIR / 'defaults' / 'settings.conf').read_bytes()) self.assertEqual(config.read_settings(copied), {}) - # the startup file is never copied into the data dir + # the startup env is never copied into the data dir self.assertFalse(Path(config.env.data_dir, 'finbot.env').exists()) mine = Path(config.env.data_dir, 'finbot_adr.json') mine.write_text('{"mine": 1}') diff --git a/util/check_config.py b/util/check_config.py index 868ad64..b62861b 100755 --- a/util/check_config.py +++ b/util/check_config.py @@ -3,7 +3,7 @@ util/check_config.py /etc/finbot.env [/var/lib/finbot/settings.conf] -finbot.env (the startup file) is never read by the bot, so this is the only place its +finbot.env (the startup env) is never read by the bot, so this is the only place its typos are caught: an unknown key, a setting key in it, a malformed line or a bad value. settings.conf defaults to /settings.conf and is checked as the bot checks it at start and on reload. It also warns when finbot.env is not 600 root:root, as it holds @@ -99,7 +99,7 @@ def permission_warnings(path): if __name__ == '__main__': ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument('startup', help='the finbot.env to check (the startup file)') + ap.add_argument('startup', help='the finbot.env to check (the startup env)') ap.add_argument('settings', nargs='?', default=config.settings_path, help='default: /settings.conf') args = ap.parse_args() diff --git a/util/docker_setup.sh b/util/docker_setup.sh index b686ef4..4c48f93 100755 --- a/util/docker_setup.sh +++ b/util/docker_setup.sh @@ -1,7 +1,7 @@ #!/bin/sh # The host side of finbot's Docker install, set up once, as root; safe to run again (doc/docker.md). Beside the # checkout, where compose.yaml looks for it: ../finbot-data (DATA_DIR) made, the defaults copied in, owned by the -# container's user (10001), so /opt/finbot-data for a checkout in /opt/finbot; and /etc/finbot.env, the startup file, +# container's user (10001), so /opt/finbot-data for a checkout in /opt/finbot; and /etc/finbot.env, the startup env, # from the template if there is none, readable by root and the container's group only. set -eu diff --git a/util/sync_defaults.py b/util/sync_defaults.py index c6ea06e..05f1185 100755 --- a/util/sync_defaults.py +++ b/util/sync_defaults.py @@ -32,7 +32,7 @@ def sync(apply=False, force=False, dest=None, create=False): if not src.is_file(): continue if src.name == 'finbot.env': - # the startup file is not data: it holds credentials, lives wherever the launcher loads it from, and the bot + # the startup env is not data: it holds credentials, lives wherever the launcher loads it from, and the bot # never reads it continue name = src.name From 38ab3a884f9b780ff650ae03152b96949993a0e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:18:11 +0000 Subject: [PATCH 10/19] doc/install.md: drop the line on editing the unit and the venv Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/install.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/install.md b/doc/install.md index a0b2a94..3033a21 100644 --- a/doc/install.md +++ b/doc/install.md @@ -114,8 +114,7 @@ For a service, use systemd, below. `finbot.service` runs finbot as a systemd service: it starts on boot, and restarts after a failure. -It expects the Git checkout in `/opt/finbot` and the startup env at `/etc/finbot.env`, as above. It needs no editing, -and uses the venv in `/opt/finbot/.venv` if you made one. +It expects the Git checkout in `/opt/finbot` and the startup env at `/etc/finbot.env`, as above. ``` sudo cp -v /opt/finbot/finbot.service /etc/systemd/system/ From f1f6e5ca8f6ce63c715a99ff10e1f2fd3f55a554 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:19:57 +0000 Subject: [PATCH 11/19] hardening.conf: no resource limits Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/install.md | 5 +---- finbot.service.d/hardening.conf | 9 +-------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/doc/install.md b/doc/install.md index 3033a21..25f6aa2 100644 --- a/doc/install.md +++ b/doc/install.md @@ -135,7 +135,7 @@ journalctl -fu finbot ### Hardening (recommended) -An optional file adds a sandbox and resource limits to the unit. finbot can then only write to `/var/lib/finbot`, and +An optional file adds a sandbox to the unit. finbot can then only write to `/var/lib/finbot`, and gets no privileges. ``` @@ -148,7 +148,4 @@ It needs the Git checkout outside `/home`, as in `/opt/finbot`. `systemd-analyze security finbot` scores the result: about 1.3 (OK) with the file, about 8 (exposed) without. -The memory and task limits are starting values. After a few days, check -`systemctl show finbot -p MemoryPeak -p TasksCurrent` and adjust them. - If the sandbox ever gets in the way, delete the file and restart finbot. diff --git a/finbot.service.d/hardening.conf b/finbot.service.d/hardening.conf index 9a88f1b..5ccd64b 100644 --- a/finbot.service.d/hardening.conf +++ b/finbot.service.d/hardening.conf @@ -1,4 +1,4 @@ -# Optional hardening for finbot.service: a sandbox and resource limits. Install it next to the unit: +# Optional hardening for finbot.service: a sandbox. Install it next to the unit: # sudo mkdir -p /etc/systemd/system/finbot.service.d # sudo cp /opt/finbot/finbot.service.d/hardening.conf /etc/systemd/system/finbot.service.d/ # sudo systemctl daemon-reload && sudo systemctl restart finbot @@ -42,10 +42,3 @@ RestrictNamespaces=yes # Network restrictions: HTTPS out; in push mode, finbot listens on one local port RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 - -# Resource limits: starting values; read MemoryPeak and TasksCurrent from systemctl show finbot, then tune. Tasks are -# threads: one per message, button press, scheduled job and chart priming, and many jobs can fire in the same minute -MemoryAccounting=yes -MemoryHigh=300M -MemoryMax=400M -TasksMax=256 From 6939a84f27d579073e7fbd9116e201f095028516 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:28:07 +0000 Subject: [PATCH 12/19] README: the keys are set in the env file; install.md says startup env file Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 8 ++++---- doc/install.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3c5a9fd..11eff14 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,12 @@ To run it as a service, under systemd on a Debian host, see [doc/install.md](doc By default, finbot fetches its updates from each network itself (pull mode), which is what this quick start guide covers. Alternatively, if you prefer to run an HTTPS gateway (a reverse proxy on a domain with a valid certificate), push mode subscribes the bot to the updates each network pushes to it. -* **Telegram:** create the bot with [@BotFather](https://telegram.me/BotFather), set `telegramBotToken`, restart +* **Telegram:** create the bot with [@BotFather](https://telegram.me/BotFather), set `telegramBotToken` in the env file, restart finbot ([guide](doc/telegram.md)). * **Slack:** create the app from the manifest that `util/slack_manifest.py` prints, add an app-level token, set - `slackBotToken` and `slackAppToken`, restart finbot ([guide](doc/slack.md)). -* **Discord:** create the application and its bot, turn on its Message Content intent, set `discordBotToken`, restart - finbot ([guide](doc/discord.md)). + `slackBotToken` and `slackAppToken` in the env file, restart finbot ([guide](doc/slack.md)). +* **Discord:** create the application and its bot, turn on its Message Content intent, set `discordBotToken` in the env file, + restart finbot ([guide](doc/discord.md)). Then send `/help` in a chat with the bot. Commands work as plain messages too (`.help`), which are quicker to type on a phone ([Commands](doc/commands.md#supported-commands)). From its first command, each chat has the default scheduled diff --git a/doc/install.md b/doc/install.md index 25f6aa2..154a81b 100644 --- a/doc/install.md +++ b/doc/install.md @@ -51,7 +51,7 @@ To run it in a container instead, see [docker.md](docker.md). sudo git clone https://github.com/robdevops/finbot.git /opt/finbot ``` -3. Make the startup env from its template: +3. Make the startup env file from its template: ``` sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env && sudo chown root:root /etc/finbot.env @@ -96,7 +96,7 @@ sudo python3 -m venv .venv sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt ``` -Make the startup env as in step 3 above, then check it: +Make the startup env file as in step 3 above, then check it: ``` sudo /opt/finbot/.venv/bin/python /opt/finbot/util/check_config.py /etc/finbot.env @@ -114,7 +114,7 @@ For a service, use systemd, below. `finbot.service` runs finbot as a systemd service: it starts on boot, and restarts after a failure. -It expects the Git checkout in `/opt/finbot` and the startup env at `/etc/finbot.env`, as above. +It expects the Git checkout in `/opt/finbot` and the startup env file at `/etc/finbot.env`, as above. ``` sudo cp -v /opt/finbot/finbot.service /etc/systemd/system/ From 4be9ba1157df78d83d565b01ec35e587fa1c0433 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 18:02:53 +0000 Subject: [PATCH 13/19] Docker logs: journald where it runs (docker_setup.sh writes .env), Docker's local driver elsewhere; docker compose logs -f reads both Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 2 +- compose.yaml | 2 +- doc/docker.md | 19 ++++++++++++++----- util/docker_setup.sh | 9 ++++++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 11eff14..96b601b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ On a Linux host with Docker: 2. `sudo /opt/finbot/util/docker_setup.sh` 3. Set the keys of each network you use in `/etc/finbot.env` (below) 4. `cd /opt/finbot && sudo docker compose up -d --build` -5. `journalctl CONTAINER_NAME=finbot -f` +5. `sudo docker compose logs -f` ### With Python On Linux, macOS or WSL, with Python 3.12 or later: diff --git a/compose.yaml b/compose.yaml index d7de113..f83a312 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,7 +20,7 @@ services: # ports: # push mode only, on the host's loopback, behind your reverse proxy # - 127.0.0.1:5000:5000 logging: - driver: ${FINBOT_LOG_DRIVER:-journald} # json-file where the host has no journald + driver: ${FINBOT_LOG_DRIVER:-local} # journald where util/docker_setup.sh found one (it writes .env); else local # hardening: the bot needs none of what these take away user: "10001:10001" read_only: true diff --git a/doc/docker.md b/doc/docker.md index 4196291..5efc747 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -11,7 +11,7 @@ The configuration lives on the host, outside the image (the startup env in `/etc 3. Set your chatbot's API key in `/etc/finbot.env` (`telegramBotToken` for Telegram; see the [Telegram](telegram.md), [Slack](slack.md) and [Discord](discord.md) guides). 4. `cd /opt/finbot && sudo docker compose up -d --build` -5. `journalctl CONTAINER_NAME=finbot -f` +5. `sudo docker compose logs -f` ## The layout on the host @@ -61,12 +61,20 @@ Each start of the container, in order: ## Logs -The log goes to the host's journal: +``` +sudo docker compose logs -f +``` + +Where journald runs, `util/docker_setup.sh` sends the log to the host's journal, which keeps it when you rebuild. Read +it there too: ``` journalctl CONTAINER_NAME=finbot -f ``` -On a host without journald, set `FINBOT_LOG_DRIVER=json-file` (in the environment of `docker compose`, or in a `.env` -file beside `compose.yaml`), and read it with `sudo docker compose logs -f`. + +Elsewhere, Docker keeps the log itself (its `local` driver, which rotates it), and `docker compose logs` is the way to +read it. That log is lost when the container is rebuilt. + +To choose, set `FINBOT_LOG_DRIVER` (`journald`, `local` or `json-file`) in a `.env` file beside `compose.yaml`. ## Reload, flush, update @@ -123,4 +131,5 @@ then `util/docker_setup.sh` (it keeps your files) and start as above. /opt/finbot-data`. * **`/data does not exist: --create makes it`:** the data dir is not mounted: `compose.yaml` mounts `../finbot-data`; run `util/docker_setup.sh` to make it. -* **No log in the journal:** the host has no journald; use `FINBOT_LOG_DRIVER=json-file` (see [Logs](#logs)). +* **No log in `journalctl`:** the container is not using journald; read it with `sudo docker compose logs -f` (see + [Logs](#logs)). diff --git a/util/docker_setup.sh b/util/docker_setup.sh index 4c48f93..35661c9 100755 --- a/util/docker_setup.sh +++ b/util/docker_setup.sh @@ -25,10 +25,17 @@ fi chown root:10001 "$startup" chmod 640 "$startup" +# the host's journal takes the log where journald runs, so that it outlasts a rebuild; Docker's own `local` driver +# (compose.yaml's default) elsewhere. Never replaces a .env that is there already +if [ -S /run/systemd/journal/socket ] && [ ! -e "$repo/.env" ]; then + echo "FINBOT_LOG_DRIVER=journald" > "$repo/.env" + echo "docker_setup: the log goes to the journal ($repo/.env)" +fi + cat < Date: Fri, 23 Oct 2026 18:06:34 +0000 Subject: [PATCH 14/19] README: one first step for Docker and Python, the env file from its template Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 96b601b..4a5a134 100644 --- a/README.md +++ b/README.md @@ -18,25 +18,23 @@ If you don't have Sharesight, you can still add your holdings manually via the w ![Screenshot of showing trade notifications on Slack](img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") ## Quick start -Run finbot with Docker or with Python. +Run finbot with Docker or with Python. Both start the same way: + +1. `sudo git clone https://github.com/robdevops/finbot /opt/finbot`, then + `sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env`, then set the keys of + each network you use in it (below) ### With Docker On a Linux host with Docker: -1. `git clone https://github.com/robdevops/finbot /opt/finbot` 2. `sudo /opt/finbot/util/docker_setup.sh` -3. Set the keys of each network you use in `/etc/finbot.env` (below) -4. `cd /opt/finbot && sudo docker compose up -d --build` -5. `sudo docker compose logs -f` +3. `cd /opt/finbot && sudo docker compose up -d --build` +4. `sudo docker compose logs -f` ### With Python -On Linux, macOS or WSL, with Python 3.12 or later: -1. `git clone https://github.com/robdevops/finbot ~/finbot && cd ~/finbot` -2. `python3 -m venv .venv && .venv/bin/pip install -r requirements.txt -r requirements-optional.txt` (on Debian and - Ubuntu, `sudo apt install python3-venv` first) -3. `cp defaults/finbot.env ~/finbot.env && chmod 600 ~/finbot.env`, then set the keys of each network you use in it - (below) -4. `.venv/bin/python util/check_config.py ~/finbot.env` -5. `(set -a && . ~/finbot.env && exec .venv/bin/python bot.py)`, which logs to the terminal until Ctrl-C +On Linux, macOS or WSL, with Python 3.12 or later and its `venv` module: +2. `cd /opt/finbot && sudo python3 -m venv .venv && sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt` +3. `sudo .venv/bin/python util/check_config.py /etc/finbot.env` +4. `sudo sh -c 'set -a && . /etc/finbot.env && exec .venv/bin/python bot.py'`, which logs to the terminal until Ctrl-C To run it as a service, under systemd on a Debian host, see [doc/install.md](doc/install.md). From 27753b041a2b7213556647102d0da85563969949 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 18:07:21 +0000 Subject: [PATCH 15/19] README: each quick start says how /etc/finbot.env is made Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4a5a134..6971fbf 100644 --- a/README.md +++ b/README.md @@ -18,23 +18,24 @@ If you don't have Sharesight, you can still add your holdings manually via the w ![Screenshot of showing trade notifications on Slack](img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") ## Quick start -Run finbot with Docker or with Python. Both start the same way: - -1. `sudo git clone https://github.com/robdevops/finbot /opt/finbot`, then - `sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env`, then set the keys of - each network you use in it (below) +Run finbot with Docker or with Python. ### With Docker On a Linux host with Docker: -2. `sudo /opt/finbot/util/docker_setup.sh` -3. `cd /opt/finbot && sudo docker compose up -d --build` -4. `sudo docker compose logs -f` +1. `sudo git clone https://github.com/robdevops/finbot /opt/finbot` +2. `sudo /opt/finbot/util/docker_setup.sh`, which makes `/etc/finbot.env` from its template +3. Set the keys of each network you use in `/etc/finbot.env` (below) +4. `cd /opt/finbot && sudo docker compose up -d --build` +5. `sudo docker compose logs -f` ### With Python On Linux, macOS or WSL, with Python 3.12 or later and its `venv` module: -2. `cd /opt/finbot && sudo python3 -m venv .venv && sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt` -3. `sudo .venv/bin/python util/check_config.py /etc/finbot.env` -4. `sudo sh -c 'set -a && . /etc/finbot.env && exec .venv/bin/python bot.py'`, which logs to the terminal until Ctrl-C +1. `sudo git clone https://github.com/robdevops/finbot /opt/finbot && cd /opt/finbot` +2. `sudo python3 -m venv .venv && sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt` +3. `sudo cp defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env`, then set the keys of each network + you use in it (below) +4. `sudo .venv/bin/python util/check_config.py /etc/finbot.env` +5. `sudo sh -c 'set -a && . /etc/finbot.env && exec .venv/bin/python bot.py'`, which logs to the terminal until Ctrl-C To run it as a service, under systemd on a Debian host, see [doc/install.md](doc/install.md). From 8771817ab031516a4fd057491341bf432348bac6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:36:14 +0000 Subject: [PATCH 16/19] /jobs add performance takes a period as a chat command does (1w, 7d, 3m, ytd), not only a number of days Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/commands.md | 2 +- doc/scheduling.md | 2 +- lib/reports/__init__.py | 18 ++++++++++++------ lib/reports/performance.py | 4 ++-- tests/test_schedule.py | 5 +++++ tests/test_scheduler.py | 2 +- 6 files changed, 22 insertions(+), 11 deletions(-) diff --git a/doc/commands.md b/doc/commands.md index 7996416..2b4e588 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -204,7 +204,7 @@ past_days=28 Job (see [Scheduling](scheduling.md)), such as `/jobs add performance weekly`: ``` -performance [DAYS] +performance [PERIOD] ``` Command: diff --git a/doc/scheduling.md b/doc/scheduling.md index 24d4514..0a5a470 100644 --- a/doc/scheduling.md +++ b/doc/scheduling.md @@ -93,7 +93,7 @@ A job is a report and what it takes, then when it runs: seen from Melbourne reads `weekdays at 9:00am Brisbane` in winter, when both clocks agree. The reports are `trades`, `movers midsession|interday|premarket|close|DAYS`, `earnings`, -`performance [DAYS]`, `rating`, `milestone`, `shorts`, `podcasts FEED_URL` and `reminder "TEXT"` (`message` works too). A job a chat added +`performance [PERIOD]`, `rating`, `milestone`, `shorts`, `podcasts FEED_URL` and `reminder "TEXT"` (`message` works too). A job a chat added as `price` before the movers report took its name runs as `movers`; `price` in a job added now is the line chart. A job can also run a command of `/help`, with what it takes, and posts what the command posts when typed in the chat: diff --git a/lib/reports/__init__.py b/lib/reports/__init__.py index 59c2d48..e68fe13 100644 --- a/lib/reports/__init__.py +++ b/lib/reports/__init__.py @@ -6,18 +6,24 @@ now. And the chat-only reports: about (.help, .jobs), history (.price, also called .compare and .history), holdings (.holdings, .who, .treemap, .watchlist), pick (the pickers' buttons), profile (.SYMBOL) and rankings (.marketcap, .ratio and the rest).""" +from lib import formats -def optional_days(name, key): - """The job() of a report, called name, whose scheduled run may give a number of days (its run's argument `key`): - its label is its name, with the days where given; anything else raises ValueError with what it takes.""" +def optional_days(name, key, usage='[DAYS]'): + """The job() of a report, called name, whose scheduled run may give a period (its run's argument `key`, in days): + a number of days, or a period as a chat command takes it (7d, 1w, 3m, ytd); its label is its name, with the period + as given; anything else raises ValueError with `usage`, what it takes.""" def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case []: return name, {} - case [days] if days.isdigit(): - return f'{name} {days}', {key: int(days)} + case [period]: + try: + days = formats.days_from_human_days(period) + except ValueError: + raise ValueError(usage) from None + return f'{name} {period}', {key: days} case _: - raise ValueError('[DAYS]') + raise ValueError(usage) return job diff --git a/lib/reports/performance.py b/lib/reports/performance.py index 97e93f4..c2c5cc9 100644 --- a/lib/reports/performance.py +++ b/lib/reports/performance.py @@ -56,5 +56,5 @@ def run(chat, *, past_days=None, portfolio_select=None): delivery.deliver(chat, delivery.Report(payload, graph, buttons.keyboard(*attached))) return attached -ARGUMENTS = '[DAYS]' # what a scheduled run takes -job = reports.optional_days('performance', 'past_days') +ARGUMENTS = '[PERIOD]' # what a scheduled run takes +job = reports.optional_days('performance', 'past_days', ARGUMENTS) diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 118830d..adbe7db 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -130,6 +130,11 @@ def test_cal_and_performance(self): """earnings takes none; performance an optional number of days.""" self.assertEqual(performance.job([]), ('performance', {})) self.assertEqual(performance.job(['7']), ('performance 7', {'past_days': 7})) + self.assertEqual(performance.job(['1w']), ('performance 1w', {'past_days': 7})) + self.assertEqual(performance.job(['7d'])[1], {'past_days': 7}) + for text in ('performance 1w every saturday at 9am', 'performance 7d saturdays at 9am', + 'performance 1w weekly on saturday at 9am'): + self.assertEqual(jobs.parse_add(text, 'UTC')[:2], ('performance', ['1w']), text) for bad, module, name in ((['x'], earnings, 'earnings'), (['x'], performance, 'performance')): with self.assertRaises(ValueError): jobs.job_args(module, name, bad) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 5bdef7d..779ea97 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -658,7 +658,7 @@ def test_add_alone_lists_the_reports_a_job_can_run(self): and examples.""" lines = self.say('.jobs add') self.assertEqual(lines[0], 'What a job can run:') - for line in ('reminder "TEXT"', 'performance [DAYS]', 'podcasts FEED_URL', + for line in ('reminder "TEXT"', 'performance [PERIOD]', 'podcasts FEED_URL', 'movers midsession|interday|premarket|close|DAYS', 'trades', '.jobs add movers weekly', 'beta', 'ratio [SYMBOL] [forward|current|peg] [top|bottom|negative|reverse]', 'price [SYMBOL ... up to 8] [period]', '.jobs add ratio peg bottom weekly'): From ec100fb3c9ae42e06a697bc66656865a33f5bfd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:36:59 +0000 Subject: [PATCH 17/19] Fix the test of performance's period forms Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- tests/test_schedule.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_schedule.py b/tests/test_schedule.py index adbe7db..cc13ac6 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -132,9 +132,9 @@ def test_cal_and_performance(self): self.assertEqual(performance.job(['7']), ('performance 7', {'past_days': 7})) self.assertEqual(performance.job(['1w']), ('performance 1w', {'past_days': 7})) self.assertEqual(performance.job(['7d'])[1], {'past_days': 7}) - for text in ('performance 1w every saturday at 9am', 'performance 7d saturdays at 9am', - 'performance 1w weekly on saturday at 9am'): - self.assertEqual(jobs.parse_add(text, 'UTC')[:2], ('performance', ['1w']), text) + for text, period in (('performance 1w every saturday at 9am', '1w'), ('performance 7d saturdays at 9am', '7d'), + ('performance 1w weekly on saturday at 9am', '1w')): + self.assertEqual(jobs.parse_add(text, 'UTC')[:2], ('performance', [period]), text) for bad, module, name in ((['x'], earnings, 'earnings'), (['x'], performance, 'performance')): with self.assertRaises(ValueError): jobs.job_args(module, name, bad) From 768ac64fa93ed0ead2397356766b2aa80ae3101f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:54:44 +0000 Subject: [PATCH 18/19] /jobs add REPORT ARGUMENT alone opens Add's buttons at how often, the argument carried Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/commands.md | 2 +- doc/scheduling.md | 6 ++++-- lib/jobs.py | 18 ++++++++++++++++++ lib/reports/about.py | 31 +++++++++++++++++++++++++++---- lib/worker.py | 13 +++++++++++-- tests/test_scheduler.py | 18 +++++++++++++++++- 6 files changed, 78 insertions(+), 10 deletions(-) diff --git a/doc/commands.md b/doc/commands.md index 2b4e588..c245ec1 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -38,7 +38,7 @@ Where: /holdings [portfolio] /jobs [upcoming|system] /jobs enable|disable|delete|run [number] -/jobs add report schedule +/jobs add report [argument] [schedule] /jobs add reminder schedule "text" /marketcap [symbol|top|bottom] /movers [percent|top|bottom|session|premarket] [period] diff --git a/doc/scheduling.md b/doc/scheduling.md index 0a5a470..8c893b5 100644 --- a/doc/scheduling.md +++ b/doc/scheduling.md @@ -17,7 +17,7 @@ never run, its name and its schedule in words. Under the list, in italics, is ho * not run yet -/jobs add REPORT SCHEDULE +/jobs add REPORT [ARGUMENT] [SCHEDULE] /jobs add reminder SCHEDULE "TEXT" ``` Under it is one row of buttons, each there only where it has something to act on (**+** always; **Edit** and **Run...** with any job, **Upcoming** with one that is on). Each screen is drawn in place of the last: @@ -53,7 +53,8 @@ them; `/treemap`, `/holdings` and `/price` run as typed bare. Then how often: A span across days (`every 20 mins from mon 9am to sat 3pm`), or a window of hours across midnight, is typed. The last screen is the whole list, headed "Added", the new job in bold with a 👉 before it, with the list's buttons (**+** adds another). A job added by -`/jobs add` shows the same. +`/jobs add` shows the same. `/jobs add performance 3m`, a report (and its one argument) without a schedule, opens the +same buttons at the how-often step, the argument carried to the job. Or type: ``` @@ -62,6 +63,7 @@ Or type: /jobs system the system jobs /jobs add what a job can run, what each takes, and examples /jobs add movers weekly +/jobs add performance 3m a report and its argument alone: buttons ask the rest /jobs add movers close weekdays at 4:30pm New York /jobs add peg top weekly /jobs add price AAPL MSFT 1y monthly diff --git a/lib/jobs.py b/lib/jobs.py index ea2e378..74722df 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -312,6 +312,24 @@ def parse_add(text, zone): raise wrong or unread +def report_alone(text): + """What .jobs add was given (its text after add), where that is a report and perhaps its one argument, such as a + period, and no schedule: (action, its argument words), for Add's buttons to ask the schedule of; None for anything + else (a schedule is there, or the report is none, takes more than one word, or does not take what was typed).""" + known, words = actions(), text.split() + name = next((' '.join(words[:size]).lower() for size in (2, 1) if ' '.join(words[:size]).lower() in known), None) + if name is None or name in SYSTEM_ACTIONS: + return None + args = words[len(name.split()):] + if len(args) > 1 or any(not (arg.isascii() and arg.isalnum()) for arg in args): + return None + try: + job_args(known[name], name, args) + except ValueError: + return None + return name, args + + class Scheduler: """Runs the jobs: each chat's, from JOBS_FILE and the catalog, and the system jobs. clock, sleep and spawn can be replaced, which the tests do. Its lock covers what the loop, a change from chat and the runs share: the diff --git a/lib/reports/about.py b/lib/reports/about.py index ddadb7e..104a21c 100644 --- a/lib/reports/about.py +++ b/lib/reports/about.py @@ -39,6 +39,7 @@ USUAL = {'frequency': 'd', 'weekday': '5', 'day': '1', 'month': '1', 'hour': '9', 'minute': '0', 'interval': WEEKDAY, 'repeat': 'd', 'start hour': '0', 'start minute': '0', 'end': END} MORE = 'p1' # Add's data for the second page of the days of a month +TYPED = 'v' # starts Add's data for a report's argument typed with it (.jobs add performance 3m) INTERVALS = (5, 10, 15, 20, 30, 120, 180, 240, 360, 720) # minutes between a repeat's runs, as every... offers them MINUTES = tuple(range(0, 60, 5)) # the minutes Add offers YEARS = 10 # the years a one-off may be in, this one first @@ -240,7 +241,10 @@ def add_screen(service, chat, steps): steps, page = list(steps), 0 if steps and steps[-1] == MORE: steps, page = steps[:-1], 1 - back = buttons.menu_button('Back', 'add', steps[:-1], crc) + typed, core = None, steps # core: the steps without the report's typed argument (carried after its number) + if len(steps) > 1 and len(steps[1]) > 1 and steps[1][0] == TYPED: # as 'v3m'; no choice's data starts so + typed, core = steps[1][1:], [steps[0], *steps[2:]] + back = buttons.menu_button('Back', 'add', [] if typed and len(core) == 1 else steps[:-1], crc) cancel = buttons.menu_button('Cancel', 'top', [0], crc) def choices(heading, offer, row, lines=(), usual=None, nav=None): """A choice's screen: its heading and lines, and '† indicates default' in italics where one is (usual: its @@ -268,7 +272,13 @@ def turn(more): raise ValueError('no such page') nav = turn(len(offer) > most) + [cancel] return choices('Add a job: which report?', offer[page * most:(page + 1) * most], row, nav=nav) - label, action, args = reports[choice(steps[0], 1, len(reports)) - 1] + label, action, args = reports[choice(core[0], 1, len(reports)) - 1] + if typed: + try: + jobs.job_args(jobs.actions()[action], action, [typed]) + except (KeyError, ValueError): + raise ValueError('no such argument') from None + label, args = f'{label} {typed}', [typed] if action == 'message': return [service.bold('Add a reminder'), "A reminder's text is what you type:", '.jobs add reminder SCHEDULE "TEXT"', '', service.bold('For example:'), @@ -279,10 +289,10 @@ def turn(more): return [service.bold(f'Add {label}'), f'What {label} is about is typed:', f'.jobs add {action} {command.ARGUMENTS} SCHEDULE', '', service.bold('For example:'), f'.jobs add {action} {example(command.command)} weekly'], [[back, cancel]] - if len(steps) == 1: + if len(core) == 1: return choices(f'Add {label}: how often?', [(name, code) for code, name in FREQUENCIES], 3, usual=USUAL['frequency']) - frequency, made = steps[1], steps[2:] + frequency, made = core[1], core[2:] levels = levels_of(frequency, made) if len(made) > len(levels) or (page and levels[len(made):len(made) + 1] != ('day',)): raise ValueError('no such choices') @@ -314,6 +324,19 @@ def turn(more): 'start minute': ('start minute?', 4, []), 'end': ('end time?', 4, [])}[level] return choices(f'Add {label}, {said}: {question}', offer, row, lines, usual) +def add_typed(service, chat, name, args): + """Add's first screen for a report typed with its argument and no schedule (.jobs add performance 3m): how often, + the argument carried along: (lines, buttons); None where Add cannot ask the rest by buttons (a reminder, a command + that needs a stock or a portfolio typed, a report not offered).""" + reports = add_reports() + found = [number for number, (_, action, given) in enumerate(reports, 1) if action == name and (given is None or + given == ())] + if not found or (name in jobs.command_actions() and jobs.command_actions()[name].needs) or name == 'message': + return None + steps = [str(found[0]), *([TYPED + args[0]] if args else [])] + lines, markup = add_screen(service, chat, steps) + return listed_with(service, chat, lines), markup + def add_reports(): """What Add offers a job to run: the reports (REPORTS), then the commands of .help a job may run, in its order, each (label, action, arguments; None where the period follows how often it runs). Each one's place in this list is its diff --git a/lib/worker.py b/lib/worker.py index 43d788b..48aaf43 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -627,7 +627,8 @@ def on_jobs(request, args): """.jobs: this chat's jobs, with the menu's buttons; .jobs upcoming: the next runs; .jobs system: the system's; .jobs enable, disable, delete, run or toggle NUMBER: one switched on or off, deleted, run now, or switched the other way; without a number, enable, disable and delete open Edit, run a button per job to run it, toggle the list; - .jobs add REPORT SCHEDULE, or .jobs add message SCHEDULE "TEXT": one more (alone, the reports a job can run).""" + .jobs add REPORT SCHEDULE, or .jobs add message SCHEDULE "TEXT": one more (alone, the reports a job can run; a + report and its period with no schedule, .jobs add performance 3m, has the buttons ask for the schedule).""" scheduler, chat, service = jobs.current, request.dest.key, request.service if scheduler is None: raise answer.Answer('No schedule is running') @@ -646,7 +647,15 @@ def on_jobs(request, args): raise Usage if not text.strip(): return request.reply(about.prepare_reports(service)) - number = scheduler.add(chat, *jobs.parse_add(text, formats.zone().key)) + try: + parsed = jobs.parse_add(text, formats.zone().key) + except jobs.ScheduleError: + alone = jobs.report_alone(text) # a report and no schedule: Add's buttons ask for it + started = about.add_typed(service, chat, *alone) if alone else None + if started is None: + raise + return request.reply(started[0], buttons=started[1]) + number = scheduler.add(chat, *parsed) lines, markup = about.added_screen(service, chat, number) request.reply(lines, buttons=markup) elif args.keyword in JOB_SCREENS and not args.words: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 779ea97..9393c27 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -672,7 +672,7 @@ def test_a_job_that_cannot_be_is_refused_saying_why(self): ('yahoo_warm daily', "no report 'yahoo_warm'"), ('os.system daily', "no report 'os.system'"), ('movers bogus weekly', 'movers takes midsession|interday|premarket|close|DAYS'), ('trades 7 daily', 'trades takes no arguments'), ('trades fortnightly', "can't read 'fortnightly'"), - ('shorts daily at 9am Atlantis', "can't read 'atlantis'"), ('trades', 'no days or times'), + ('shorts daily at 9am Atlantis', "can't read 'atlantis'"), ('message on 1 jan 2020 "late"', 'once, 1 Jan 2020 has gone by'), ('trades every 20 mins weekdays UTC', 'that job is here already')): with self.subTest(text=text): @@ -990,6 +990,22 @@ def test_add_by_buttons_back_and_cancel(self): zone='Australia/Melbourne')) self.assertIn('that job is here already', self.say('.jobs add movers weekly on friday at 14:30')[0]) + def test_a_report_and_its_argument_typed_alone_open_the_buttons_for_the_rest(self): + """'.jobs add performance 3m' (or a report with no argument) asks how often with Add's buttons, the argument + carried to the job they make; a reminder, or an argument the report refuses, is not taken.""" + lines = self.say('.jobs add performance 3m') + self.assertEqual(lines[0], 'Add performance 3m: how often?') + data = {button.label: button.data for row in self.replies[-1][2] for button in row} + lines, rows, data = self.press(data['quarterly']) + self.assertIn('Add performance 3m, quarterly', lines[0]) + for label in ('Feb', '15', '09', '09:00'): + lines, rows, data = self.press(next(data[key] for key in data if key.startswith(label))) + self.assertEqual(state.load(jobs.JOBS_FILE)[CHAT]['jobs'][-1], entry('performance 3m quarterly from feb on the' + ' 15th at 9am', zone='Australia/Melbourne')) + self.assertIn('how often?', self.say('.jobs add performance')[0]) + self.assertNotIn('how often?', ' '.join(self.say('.jobs add performance nonsense'))) + self.assertNotIn('how often?', ' '.join(self.say('.jobs add reminder'))) + def test_the_month_steps_start_a_cycle(self): """Monthly asks for the day of the month; quarterly, bi-annual and annual first for the start month, the cycle following from it, then the day, hour and minute; annual offers only the days its month has. On Discord the From f4f95bcd573b1a7ba71079e53870a29efd9c3906 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:59:33 +0000 Subject: [PATCH 19/19] Add asks movers and performance for a period first; movers title and bottom order The movers list is headed 'Top N movers', its chart shows the bottom ten worst last, and a movers job takes a period as performance does (1w, 3m). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- doc/commands.md | 2 +- doc/scheduling.md | 4 ++-- lib/reports/about.py | 3 +++ lib/reports/movers.py | 16 ++++++++++------ tests/test_charts.py | 4 ++-- tests/test_discord.py | 2 +- tests/test_edges.py | 2 +- tests/test_scheduler.py | 31 ++++++++++++++++++------------- 8 files changed, 38 insertions(+), 26 deletions(-) diff --git a/doc/commands.md b/doc/commands.md index c245ec1..e27d18f 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -156,7 +156,7 @@ demote_volatile=True Job (see [Scheduling](scheduling.md)), such as `/jobs add movers close weekdays at 4:30pm New York`: ``` -movers premarket|midsession|interday|close|DAYS +movers premarket|midsession|interday|close|PERIOD ``` When scheduled, the mode is its argument: * If `premarket` is passed, it only reports on pre/post market price movements. diff --git a/doc/scheduling.md b/doc/scheduling.md index 8c893b5..9a47c3f 100644 --- a/doc/scheduling.md +++ b/doc/scheduling.md @@ -21,7 +21,7 @@ never run, its name and its schedule in words. Under the list, in italics, is ho /jobs add reminder SCHEDULE "TEXT" ``` Under it is one row of buttons, each there only where it has something to act on (**+** always; **Edit** and **Run...** with any job, **Upcoming** with one that is on). Each screen is drawn in place of the last: -* **+**: what the job runs, how often, then the rest down to the minute ([below](#add-step-by-step)). +* **+**: what the job runs (for `movers` and `performance`, then its period: 1d, 7d, 1w, 1m, 3m or 1y), how often, then the rest down to the minute ([below](#add-step-by-step)). * **Edit**: the list with a button per job, the default ones too; a press selects the job (its button turns red, on Telegram, Slack and Discord alike) or takes it out of the selection again. It starts with the jobs that are off selected, and nothing changes until **OK**, which switches the selected jobs off and the rest on. **OK** shows only where the selection differs from the jobs as they are, and **Delete** only while any job is selected; **Delete** asks first ("Delete the 3 selected jobs?"), then deletes them. **Select all** selects every job, and reads **Select none** once all are selected. **Cancel** goes back to the list, whatever was selected. A default job that is deleted comes back when the same job is added again. * **Run...**: a button per job; a press runs it now, as its schedule would. What the job posts shows it ran; a message says so only when it failed, or was running already. The same button pressed again within a minute does @@ -94,7 +94,7 @@ A job is a report and what it takes, then when it runs: 4:30pm New York (7:30am local, Tue–Sat)`. A zone that keeps another clock is named in every season: a Brisbane job seen from Melbourne reads `weekdays at 9:00am Brisbane` in winter, when both clocks agree. -The reports are `trades`, `movers midsession|interday|premarket|close|DAYS`, `earnings`, +The reports are `trades`, `movers midsession|interday|premarket|close|PERIOD`, `earnings`, `performance [PERIOD]`, `rating`, `milestone`, `shorts`, `podcasts FEED_URL` and `reminder "TEXT"` (`message` works too). A job a chat added as `price` before the movers report took its name runs as `movers`; `price` in a job added now is the line chart. diff --git a/lib/reports/about.py b/lib/reports/about.py index 104a21c..8b6a7f4 100644 --- a/lib/reports/about.py +++ b/lib/reports/about.py @@ -38,6 +38,7 @@ # the default of each choice, marked † on its button; a one-off's are today's year and month, and tomorrow USUAL = {'frequency': 'd', 'weekday': '5', 'day': '1', 'month': '1', 'hour': '9', 'minute': '0', 'interval': WEEKDAY, 'repeat': 'd', 'start hour': '0', 'start minute': '0', 'end': END} +PERIODS = ('1d', '7d', '1w', '1m', '3m', '1y') # what Add offers a movers or performance job's period MORE = 'p1' # Add's data for the second page of the days of a month TYPED = 'v' # starts Add's data for a report's argument typed with it (.jobs add performance 3m) INTERVALS = (5, 10, 15, 20, 30, 120, 180, 240, 360, 720) # minutes between a repeat's runs, as every... offers them @@ -289,6 +290,8 @@ def turn(more): return [service.bold(f'Add {label}'), f'What {label} is about is typed:', f'.jobs add {action} {command.ARGUMENTS} SCHEDULE', '', service.bold('For example:'), f'.jobs add {action} {example(command.command)} weekly'], [[back, cancel]] + if len(core) == 1 and not typed and args is None and action in ('movers', 'performance'): + return choices(f'Add {label}: which period?', [(period, TYPED + period) for period in PERIODS], 3) if len(core) == 1: return choices(f'Add {label}: how often?', [(name, code) for code, name in FREQUENCIES], 3, usual=USUAL['frequency']) diff --git a/lib/reports/movers.py b/lib/reports/movers.py index 8bbc4fc..79dd4bd 100644 --- a/lib/reports/movers.py +++ b/lib/reports/movers.py @@ -142,12 +142,12 @@ def report(service, found, mode='interday', threshold=None, days=None, top=None, chart = render.Chart('bars', heading(''), subtitle, tuple(label(found.rows)), session=session) return [service.bold(heading(f', {span}'))] + lines, chart worst = [i for i in reversed(range(len(found.rows))) if i >= top][:top] # the worst, worst first, none of the best - payload = [service.bold(f'Top {top} performers {span}')] + lines[:top] + payload = [service.bold(f'Top {top} movers {span}')] + lines[:top] if worst: payload += ['', service.bold(f'Bottom {top}')] + [lines[i] for i in worst] - bars = label(found.rows[:top]) # the best, then after a gap the worst, as the caption's two lists + bars = label(found.rows[:top]) # the best, then after a gap the worst, the worst last if worst: - bars += [('', None)] + label(found.rows[i] for i in worst) + bars += [("", None)] + label(found.rows[i] for i in reversed(worst)) return payload, render.Chart('bars', f"Top & bottom {top}", subtitle, tuple(bars)) def nothing(mode, threshold, user, ticker, in_session): @@ -191,14 +191,18 @@ def list_buttons(threshold=None, top=None, days=None): period = buttons.period_for_days(days, None if days else buttons.DEFAULT_PERIOD['l']) return [f"{threshold:g}", str(top or 0)], period -ARGUMENTS = 'midsession|interday|premarket|close|DAYS' # what a scheduled run takes +ARGUMENTS = 'midsession|interday|premarket|close|PERIOD' # what a scheduled run takes def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case ['midsession' | 'interday' | 'premarket' | 'close' as mode]: return f'movers {mode}', {'mode': mode} - case [days] if days.isdigit(): - return f'movers {days}', {'mode': 'period', 'threshold': 0, 'days': int(days), 'top': TOP} + case [period]: + try: + days = formats.days_from_human_days(period) + except ValueError: + raise ValueError(ARGUMENTS) from None + return f'movers {period}', {'mode': 'period', 'threshold': 0, 'days': days, 'top': TOP} case _: raise ValueError(ARGUMENTS) diff --git a/tests/test_charts.py b/tests/test_charts.py index 0248c13..2a5717f 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -359,12 +359,12 @@ def test_a_label_too_wide_is_cut_short_to_fit(self): self.assertIsNone(canvas.fitted('Packaged Software', 8, 5, bold=True)) def test_movers_top_lists_best_first_then_worst(self): - """.movers top: the ten best, best first, a spacer, then the ten worst, worst first, as in the caption.""" + """.movers top: the ten best, best first, a spacer, then the ten worst, the worst last.""" chart = self.run_report(lambda: self.command('.movers top 7d')) values = [v for _, v in chart.rows if v is not None] top, bottom = values[:10], values[10:] self.assertEqual(top, sorted(top, reverse=True)) - self.assertEqual(bottom, sorted(bottom)) # the worst performer comes first, like the caption's second list + self.assertEqual(bottom, sorted(bottom, reverse=True)) # the worst performer is last self.assertGreater(min(top), max(bottom)) self.assertIn(None, [v for _, v in chart.rows]) # spacer between the two lists diff --git a/tests/test_discord.py b/tests/test_discord.py index 667f451..b60b4bd 100644 --- a/tests/test_discord.py +++ b/tests/test_discord.py @@ -463,7 +463,7 @@ def test_the_jobs_subcommands_numbers_and_reports(self): self.assertEqual(self.values('/jobs disable '), ['disable 1', 'disable 3']) self.assertEqual(self.values('/jobs run ', chat='discord:88'), []) reports = worker.suggest('discord:77', '/jobs add ') - self.assertIn(('add movers midsession|interday|premarket|close|DAYS', 'add movers'), reports) + self.assertIn(('add movers midsession|interday|premarket|close|PERIOD', 'add movers'), reports) self.assertIn(('add message "TEXT"', 'add message'), reports) self.assertFalse(any(name in value for _, value in reports for name in jobs.SYSTEM_ACTIONS)) self.assertEqual(self.values('/jobs add movers m'), ['add movers midsession', 'add movers monthly']) diff --git a/tests/test_edges.py b/tests/test_edges.py index 4ef3ad6..7aeb52a 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -263,7 +263,7 @@ def test_keywords_in_any_case(self): ('.watchlist Remove x', 'delete')): with self.subTest(message=message): self.assertEqual(self.args(message)[1].keyword, keyword) - self.assertIn('Top 10 performers', str(self.command('.movers TOP 7d'))) + self.assertIn('Top 10 movers', str(self.command('.movers TOP 7d'))) def test_a_bare_number_on_movers_is_a_percent(self): """As on .session, .premarket and .shorts, and as doc/commands.md has it; a period carries its unit.""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9393c27..f5308ed 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -659,7 +659,7 @@ def test_add_alone_lists_the_reports_a_job_can_run(self): lines = self.say('.jobs add') self.assertEqual(lines[0], 'What a job can run:') for line in ('reminder "TEXT"', 'performance [PERIOD]', 'podcasts FEED_URL', - 'movers midsession|interday|premarket|close|DAYS', 'trades', '.jobs add movers weekly', 'beta', + 'movers midsession|interday|premarket|close|PERIOD', 'trades', '.jobs add movers weekly', 'beta', 'ratio [SYMBOL] [forward|current|peg] [top|bottom|negative|reverse]', 'price [SYMBOL ... up to 8] [period]', '.jobs add ratio peg bottom weekly'): self.assertIn(line, lines) @@ -670,7 +670,7 @@ def test_a_job_that_cannot_be_is_refused_saying_why(self): that are no schedule, a zone that is none, a one-off gone by, a job here already: refused, saying why.""" for text, why in (('nothere daily', "no report 'nothere'"), ('trades.py daily', "no report 'trades.py'"), ('yahoo_warm daily', "no report 'yahoo_warm'"), ('os.system daily', "no report 'os.system'"), - ('movers bogus weekly', 'movers takes midsession|interday|premarket|close|DAYS'), + ('movers bogus weekly', 'movers takes midsession|interday|premarket|close|PERIOD'), ('trades 7 daily', 'trades takes no arguments'), ('trades fortnightly', "can't read 'fortnightly'"), ('shorts daily at 9am Atlantis', "can't read 'atlantis'"), ('message on 1 jan 2020 "late"', 'once, 1 Jan 2020 has gone by'), @@ -969,26 +969,30 @@ def test_add_by_buttons_back_and_cancel(self): ['treemap', 'who', 'marketcap'], ['beta', 'price', 'buysell'], ['ratio', 'stock'], ['Cancel']]) self.assertEqual((data['movers'], data['Cancel']), (f'm|add|5|{self.crc()}', f'm|top|0|{self.crc()}')) lines, rows, data = self.press(data['movers']) - self.assertEqual((lines[:3], rows), (['Add movers: how often?', '', '† indicates default'], + self.assertEqual((lines[:2], rows), (['Add movers: which period?', ''], + [['1d', '7d', '1w'], ['1m', '3m', '1y'], ['Back', 'Cancel']])) + self.assertEqual(data['Back'], f'm|add|{self.crc()}') + lines, rows, data = self.press(data['1w']) + self.assertEqual((lines[:3], rows), (['Add movers 1w: how often?', '', '† indicates default'], [['one-off', 'hourly', 'daily†'], ['weekly', 'monthly', 'quarterly'], ['bi-annual', 'annual', 'every...'], ['Back', 'Cancel']])) lines, rows, data = self.press(data['weekly']) self.assertEqual(rows[:2], [['Mon', 'Tue', 'Wed', 'Thu'], ['Fri', 'Sat†', 'Sun']]) self.assertEqual(data['Cancel'], f'm|top|0|{self.crc()}') lines, rows, data = self.press(data['Fri']) - self.assertEqual(lines[:2], ['Add movers, weekly, Fridays: at what hour?', "By Melbourne's clock"]) + self.assertEqual(lines[:2], ['Add movers 1w, weekly, Fridays: at what hour?', "By Melbourne's clock"]) self.assertEqual((len(rows), [len(row) for row in rows], rows[1][4], rows[-1]), (5, [5] * 5, '09†', ['20', '21', '22', '23', 'Back'])) - self.assertEqual(data['Back'], f'm|add|5|w|{self.crc()}') + self.assertEqual(data['Back'], f'm|add|5|v1w|w|{self.crc()}') lines, rows, data = self.press(data['14']) self.assertEqual((rows[0], rows[-1]), (['14:00†', '14:05', '14:10', '14:15'], ['Back', 'Cancel'])) lines, rows, data = self.press(data['14:30']) - self.assertEqual(newest(lines), '3. * movers · weekly, Fridays at 2:30pm') + self.assertEqual(newest(lines), '3. * movers 1w · weekly, Fridays at 2:30pm') self.assertEqual((lines[0], rows, data['+']), ('Added', [['+', 'Edit', 'Run...', 'Upcoming']], f'm|add|{self.crc()}')) # the whole list, the job added in bold, with the list's buttons - self.assertEqual(state.load(jobs.JOBS_FILE)[CHAT]['jobs'][-1], entry('movers every friday at 2:30pm', + self.assertEqual(state.load(jobs.JOBS_FILE)[CHAT]['jobs'][-1], entry('movers 1w every friday at 2:30pm', zone='Australia/Melbourne')) - self.assertIn('that job is here already', self.say('.jobs add movers weekly on friday at 14:30')[0]) + self.assertIn('that job is here already', self.say('.jobs add movers 1w weekly on friday at 14:30')[0]) def test_a_report_and_its_argument_typed_alone_open_the_buttons_for_the_rest(self): """'.jobs add performance 3m' (or a report with no argument) asks how often with Add's buttons, the argument @@ -1002,7 +1006,7 @@ def test_a_report_and_its_argument_typed_alone_open_the_buttons_for_the_rest(sel lines, rows, data = self.press(next(data[key] for key in data if key.startswith(label))) self.assertEqual(state.load(jobs.JOBS_FILE)[CHAT]['jobs'][-1], entry('performance 3m quarterly from feb on the' ' 15th at 9am', zone='Australia/Melbourne')) - self.assertIn('how often?', self.say('.jobs add performance')[0]) + self.assertIn('which period?', self.say('.jobs add performance')[0]) self.assertNotIn('how often?', ' '.join(self.say('.jobs add performance nonsense'))) self.assertNotIn('how often?', ' '.join(self.say('.jobs add reminder'))) @@ -1026,14 +1030,15 @@ def test_the_month_steps_start_a_cycle(self): with self.subTest(frequency=frequency): lines, rows, data = self.press(f'm|add|{self.crc()}') lines, rows, data = self.press(data['movers']) + lines, rows, data = self.press(data['1w']) lines, rows, data = self.press(data[frequency]) - self.assertEqual((lines[0], rows[0]), (f'Add movers, {frequency}: start month?', ['Jan†', 'Feb', - 'Mar', 'Apr'])) + self.assertEqual((lines[0], rows[0]), (f'Add movers 1w, {frequency}: start month?', + ['Jan†', 'Feb', 'Mar', 'Apr'])) lines, rows, data = self.press(data[start]) - self.assertEqual(lines[0], f'Add movers, {said}: on which day of the month?') + self.assertEqual(lines[0], f'Add movers 1w, {said}: on which day of the month?') for pressed in ('15', '09†', '09:00†'): lines, rows, data = self.press(data[pressed]) - self.assertTrue(newest(lines).endswith(f'* movers · {added}'), lines) + self.assertTrue(newest(lines).endswith(f'* movers 1w · {added}'), lines) lines, rows, data = self.press(f'm|add|5|a|2|{self.crc()}') self.assertEqual((lines[0], rows[-2]), ('Add movers, annual, in Feb: on which day of the month?', ['26', '27', '28', '29']))