diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc789ba..1fdaa85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,19 +126,19 @@ jobs: wait_for 'cache: flushed' docker rm -f finbot - if: steps.changes.outputs.docs_only != 'true' - name: A mistake in the startup file stops the container before the bot starts + name: A mistake in the startup env stops the container before the bot starts run: | printf 'telegramBotTokn=x\n' > "$RUNNER_TEMP/bad.env" if docker run --rm --read-only --tmpfs /tmp -e DATA_DIR=/tmp \ -v "$RUNNER_TEMP/bad.env:/run/finbot.env:ro" finbot > "$RUNNER_TEMP/out" 2>&1; then cat "$RUNNER_TEMP/out" - echo "::error::the container started with a mistake in its startup file" + echo "::error::the container started with a mistake in its startup env" exit 1 fi cat "$RUNNER_TEMP/out" grep -q 'unknown key telegramBotTokn' "$RUNNER_TEMP/out" if grep -q 'finbot is running' "$RUNNER_TEMP/out"; then - echo "::error::the bot started with a mistake in its startup file" + echo "::error::the bot started with a mistake in its startup env" exit 1 fi diff --git a/Dockerfile b/Dockerfile index 0c4f32b..d892563 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # finbot in a container, on Debian with Debian's own python3 packages (no pip), as on a Debian host. Every optional -# library is in, so the image runs every network in either mode. The startup file and DATA_DIR stay outside the image: +# library is in, so the image runs every network in either mode. The startup env and DATA_DIR stay outside the image: # see compose.yaml and doc/docker.md. FROM debian:trixie-slim @@ -23,5 +23,5 @@ ARG GIT_COMMIT=unknown ENV FINBOT_COMMIT=$GIT_COMMIT PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 DATA_DIR=/data HOME=/tmp USER finbot -# each start: copy the defaults missing from /data, check the startup file, and only then start the bot +# each start: copy the defaults missing from /data, check the startup env, and only then start the bot CMD ["sh", "-c", "util/sync_defaults.py --apply && util/check_config.py /run/finbot.env && exec ./bot.py"] diff --git a/README.md b/README.md index f51647b..6971fbf 100644 --- a/README.md +++ b/README.md @@ -22,34 +22,33 @@ Run finbot with Docker or with Python. ### With Docker On a Linux host with Docker: -1. `git clone https://github.com/robdevops/finbot /opt/finbot` -2. `sudo /opt/finbot/util/docker_setup.sh` -3. Set the keys of each network you use in `/opt/finbot.env` (below) +1. `sudo git clone https://github.com/robdevops/finbot /opt/finbot` +2. `sudo /opt/finbot/util/docker_setup.sh`, which makes `/etc/finbot.env` from its template +3. Set the keys of each network you use in `/etc/finbot.env` (below) 4. `cd /opt/finbot && sudo docker compose up -d --build` -5. `journalctl CONTAINER_NAME=finbot -f` +5. `sudo docker compose logs -f` ### With Python -On Linux, macOS or WSL, with Python 3.12 or later: -1. `git clone https://github.com/robdevops/finbot ~/finbot && cd ~/finbot` -2. `python3 -m venv .venv && .venv/bin/pip install -r requirements.txt -r requirements-optional.txt` (on Debian and - Ubuntu, `sudo apt install python3-venv` first) -3. `cp defaults/finbot.env ~/finbot.env && chmod 600 ~/finbot.env`, then set the keys of each network you use in it - (below) -4. `.venv/bin/python util/check_config.py ~/finbot.env` -5. `(set -a && . ~/finbot.env && exec .venv/bin/python bot.py)`, which logs to the terminal until Ctrl-C +On Linux, macOS or WSL, with Python 3.12 or later and its `venv` module: +1. `sudo git clone https://github.com/robdevops/finbot /opt/finbot && cd /opt/finbot` +2. `sudo python3 -m venv .venv && sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt` +3. `sudo cp defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env`, then set the keys of each network + you use in it (below) +4. `sudo .venv/bin/python util/check_config.py /etc/finbot.env` +5. `sudo sh -c 'set -a && . /etc/finbot.env && exec .venv/bin/python bot.py'`, which logs to the terminal until Ctrl-C To run it as a service, under systemd on a Debian host, see [doc/install.md](doc/install.md). ### The chat networks By default, finbot fetches its updates from each network itself (pull mode), which is what this quick start guide covers. Alternatively, if you prefer to run an HTTPS gateway (a reverse proxy on a domain with a valid certificate), push mode -subscribes the bot to the updates each network pushes to it (see each guide's _push mode_). -* **Telegram:** create the bot with [@BotFather](https://telegram.me/BotFather), set `telegramBotToken`, restart +subscribes the bot to the updates each network pushes to it. +* **Telegram:** create the bot with [@BotFather](https://telegram.me/BotFather), set `telegramBotToken` in the env file, restart finbot ([guide](doc/telegram.md)). * **Slack:** create the app from the manifest that `util/slack_manifest.py` prints, add an app-level token, set - `slackBotToken` and `slackAppToken`, restart finbot ([guide](doc/slack.md)). -* **Discord:** create the application and its bot, turn on its Message Content intent, set `discordBotToken`, restart - finbot ([guide](doc/discord.md)). + `slackBotToken` and `slackAppToken` in the env file, restart finbot ([guide](doc/slack.md)). +* **Discord:** create the application and its bot, turn on its Message Content intent, set `discordBotToken` in the env file, + restart finbot ([guide](doc/discord.md)). Then send `/help` in a chat with the bot. Commands work as plain messages too (`.help`), which are quicker to type on a phone ([Commands](doc/commands.md#supported-commands)). From its first command, each chat has the default scheduled diff --git a/compose.yaml b/compose.yaml index b29ae0f..f83a312 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,6 @@ -# finbot with Docker Compose (doc/docker.md). The configuration lives beside the checkout, outside the image and the -# container: ../finbot.env, the startup file, and ../finbot-data, DATA_DIR (settings.conf, state and cache), both -# made by util/docker_setup.sh. Build and start: docker compose up -d --build +# finbot with Docker Compose (doc/docker.md). The configuration lives outside the image and the container: +# /etc/finbot.env, the startup env, and ../finbot-data beside the checkout, DATA_DIR (settings.conf, state and cache), +# both made by util/docker_setup.sh. Build and start: docker compose up -d --build services: finbot: build: @@ -10,17 +10,17 @@ services: image: finbot container_name: finbot restart: unless-stopped - env_file: ../finbot.env + env_file: /etc/finbot.env environment: DATA_DIR: /data ip: 0.0.0.0 # push mode listens inside the container: publish its port below volumes: - ../finbot-data:/data - - ../finbot.env:/run/finbot.env:ro # checked before the bot starts + - /etc/finbot.env:/run/finbot.env:ro # checked before the bot starts # ports: # push mode only, on the host's loopback, behind your reverse proxy # - 127.0.0.1:5000:5000 logging: - driver: ${FINBOT_LOG_DRIVER:-journald} # json-file where the host has no journald + driver: ${FINBOT_LOG_DRIVER:-local} # journald where util/docker_setup.sh found one (it writes .env); else local # hardening: the bot needs none of what these take away user: "10001:10001" read_only: true diff --git a/defaults/finbot.env b/defaults/finbot.env index 2b1bbdb..81e4bac 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -1,15 +1,15 @@ -# finbot.env: THE STARTUP FILE. Where finbot runs, and who it talks to. +# finbot.env: THE STARTUP ENV. Where finbot runs, and who it talks to. # The bot never reads this file. Your launcher loads it into the bot's environment, and a change needs a RESTART: -# systemd: EnvironmentFile=/opt/finbot.env (in finbot.service) -# docker: --env-file /opt/finbot.env -# init.d: set -a; . /opt/finbot.env; set +a; ./bot.py +# systemd: EnvironmentFile=/etc/finbot.env (in finbot.service) +# docker: --env-file /etc/finbot.env +# init.d: set -a; . /etc/finbot.env; set +a; ./bot.py # Keep it private: chmod 600, owned by root:root. It holds the credentials (check_config.py warns if not). # Flat KEY=value lines: no spaces around "=", no quotes (docker keeps quotes literally), no trailing comments; comments # on their own line. # Everything else (thresholds, windows, display, caching, ...) is in settings.conf, which the bot reloads without a # restart. # A setting key in this file is an error: finbot refuses to start. Lint both files with: -# util/check_config.py /opt/finbot.env +# util/check_config.py /etc/finbot.env # Where state and cache live. Leave unset under systemd with StateDirectory=finbot (it uses $STATE_DIRECTORY, # /var/lib/finbot, the only directory the shipped unit can write). diff --git a/doc/commands.md b/doc/commands.md index 7996416..e27d18f 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -38,7 +38,7 @@ Where: /holdings [portfolio] /jobs [upcoming|system] /jobs enable|disable|delete|run [number] -/jobs add report schedule +/jobs add report [argument] [schedule] /jobs add reminder schedule "text" /marketcap [symbol|top|bottom] /movers [percent|top|bottom|session|premarket] [period] @@ -156,7 +156,7 @@ demote_volatile=True Job (see [Scheduling](scheduling.md)), such as `/jobs add movers close weekdays at 4:30pm New York`: ``` -movers premarket|midsession|interday|close|DAYS +movers premarket|midsession|interday|close|PERIOD ``` When scheduled, the mode is its argument: * If `premarket` is passed, it only reports on pre/post market price movements. @@ -204,7 +204,7 @@ past_days=28 Job (see [Scheduling](scheduling.md)), such as `/jobs add performance weekly`: ``` -performance [DAYS] +performance [PERIOD] ``` Command: diff --git a/doc/configuration.md b/doc/configuration.md index 03f0a02..b48653c 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -4,14 +4,14 @@ Configuration is split into two files -- finbot.env for startup environment vari Copy the shipped template: ``` -sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env +sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env && sudo chown root:root /etc/finbot.env ``` ## `finbot.env` **Load `finbot.env` with your launcher:** -* **systemd:** `EnvironmentFile=/opt/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /opt/finbot.env` for pre-flight lint. +* **systemd:** `EnvironmentFile=/etc/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /etc/finbot.env` for pre-flight lint. * **Docker:** `compose.yaml` loads it, and checks it before the bot starts (see [docker.md](docker.md)). Write the values without quotes. -* **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). +* **init.d / shell:** `util/check_config.py /etc/finbot.env && set -a && . /etc/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). * **Scheduled jobs:** they are run by `bot.py` (see [Scheduling](scheduling.md)), so they run on the same settings. For a one-off run, send `/jobs run` in the chat. The startup keys, read by your launcher. A change needs a restart. Blank means unset. Each network's guide says which @@ -86,7 +86,7 @@ The administrators of the chat a person is in are admins there with no setting: A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. -edit `settings.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `sudo util/check_config.py /opt/finbot.env` (under systemd it reads the service's `settings.conf`). Edit `finbot.env`, then restart: a running bot cannot see it change. +edit `settings.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `sudo util/check_config.py /etc/finbot.env` (under systemd it reads the service's `settings.conf`). Edit `finbot.env`, then restart: a running bot cannot see it change. **Upgrading from a version before 16 Oct 2026?** Start commit `ff52f2b` once first, as [Scheduling](scheduling.md#from-an-earlier-version) says: it splits one old `.env` (`util/split_env.py`, which it asks for) and renames `preferences.conf` to `settings.conf`, which later versions no longer do. @@ -95,7 +95,7 @@ All state and cache lives in one data directory: `DATA_DIR` if set, else systemd Reference data (`finbot_adr.json`, `finbot_sws_*.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: ``` util/sync_defaults.py # dry run -util/sync_defaults.py --apply # copy missing files (including a starter settings.conf) +util/sync_defaults.py --apply # copy missing files (including the settings.conf template) util/sync_defaults.py --force # also replace changed files, keeping .bak copies util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory util/sync_defaults.py --dest /srv/finbot --create --apply # and make it (mode 750) if it does not exist yet diff --git a/doc/docker.md b/doc/docker.md index 8d52cc3..5efc747 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -2,28 +2,28 @@ finbot runs in a container as it runs on a Debian host: the image is `debian:trixie-slim` with Debian's own `python3-*` packages, every optional library included, so it runs every network in either mode. The image holds only the code. -The configuration lives on the host, beside the checkout, so a rebuild or an update never touches it. +The configuration lives on the host, outside the image (the startup env in `/etc`, the data beside the Git checkout), so a rebuild or an update never touches it. ## Quick start 1. `git clone https://github.com/robdevops/finbot /opt/finbot` 2. `sudo /opt/finbot/util/docker_setup.sh` -3. Set `telegramBotToken` in `/opt/finbot.env` (or another network's keys: see the [Telegram](telegram.md), - [Slack](slack.md) and [Discord](discord.md) guides). +3. Set your chatbot's API key in `/etc/finbot.env` (`telegramBotToken` for Telegram; see the + [Telegram](telegram.md), [Slack](slack.md) and [Discord](discord.md) guides). 4. `cd /opt/finbot && sudo docker compose up -d --build` -5. `journalctl CONTAINER_NAME=finbot -f` +5. `sudo docker compose logs -f` ## The layout on the host | Path | What | In the container | |---|---|---| -| `/opt/finbot` | the checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | -| `/opt/finbot.env` | the startup file: the tokens, and the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | +| `/opt/finbot` | the Git checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | +| `/etc/finbot.env` | the startup env: the tokens, and the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | | `/opt/finbot-data` | `DATA_DIR`: `settings.conf`, each chat's jobs and watchlist, the records, the cache | `/data` | -`compose.yaml` finds the other two beside the checkout (`../finbot.env`, `../finbot-data`): clone it elsewhere and -they move with it. The startup file stays out of the image because it holds the secrets, and the data dir because it -must outlast every container. +`compose.yaml` reads the startup env at `/etc/finbot.env` and finds the data dir beside the Git checkout +(`../finbot-data`): clone it elsewhere and the data dir moves with it. The startup env stays out of the image because +it holds the secrets, and the data dir because it must outlast every container.. ## What `util/docker_setup.sh` does @@ -32,18 +32,18 @@ Run it once, as root; it is safe to run again. It: the shipped defaults into it: `settings.conf` and the data files. It never replaces a file you have changed. The script needs only the standard library, so any `python3` on the host runs it. 2. gives the data dir to the container's user: `chown -R 10001:10001 /opt/finbot-data`. -3. copies `defaults/finbot.env` to `/opt/finbot.env` if there is none, owned by `root:10001`, mode 640: root and the +3. copies `defaults/finbot.env` to `/etc/finbot.env` if there is none, owned by `root:10001`, mode 640: root and the container can read it, nobody else. By hand, that is: ``` sudo python3 /opt/finbot/util/sync_defaults.py --create --apply --dest /opt/finbot-data sudo chown -R 10001:10001 /opt/finbot-data -sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env -sudo chown root:10001 /opt/finbot.env && sudo chmod 640 /opt/finbot.env +sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env +sudo chown root:10001 /etc/finbot.env && sudo chmod 640 /etc/finbot.env ``` -The startup file is `KEY=value` lines, as for systemd; write the values without quotes. +The startup env is `KEY=value` lines, as for systemd; write the values without quotes. ## Build and start @@ -55,18 +55,26 @@ cd /opt/finbot && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --b Each start of the container, in order: 1. `util/sync_defaults.py --apply` copies into `/data` any default it lacks, such as a data file added by an update; -2. `util/check_config.py /run/finbot.env` checks the startup file and `settings.conf`. A mistake stops the start +2. `util/check_config.py /run/finbot.env` checks the startup env and `settings.conf`. A mistake stops the start here, and the log says which key and why; 3. the bot starts. ## Logs -The log goes to the host's journal: +``` +sudo docker compose logs -f +``` + +Where journald runs, `util/docker_setup.sh` sends the log to the host's journal, which keeps it when you rebuild. Read +it there too: ``` journalctl CONTAINER_NAME=finbot -f ``` -On a host without journald, set `FINBOT_LOG_DRIVER=json-file` (in the environment of `docker compose`, or in a `.env` -file beside `compose.yaml`), and read it with `sudo docker compose logs -f`. + +Elsewhere, Docker keeps the log itself (its `local` driver, which rotates it), and `docker compose logs` is the way to +read it. That log is lost when the container is rebuilt. + +To choose, set `FINBOT_LOG_DRIVER` (`journald`, `local` or `json-file`) in a `.env` file beside `compose.yaml`. ## Reload, flush, update @@ -76,7 +84,7 @@ sudo docker compose kill -s USR1 finbot # flush the cache, and fill it again sudo docker compose exec finbot util/flush.py --after 24h # flush the files of the last day (see caching.md) cd /opt/finbot && sudo git pull && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --build ``` -A change to `/opt/finbot.env` needs a restart: `sudo docker compose up -d`. +A change to `/etc/finbot.env` needs a restart: `sudo docker compose up -d`. ## Push mode @@ -113,14 +121,15 @@ sudo systemctl disable --now finbot sudo cp -a /var/lib/finbot/. /opt/finbot-data/ # or /opt/finbot/var/. for the default DATA_DIR sudo chown -R 10001:10001 /opt/finbot-data ``` -then `util/docker_setup.sh` (it keeps your files) and start as above. The startup file can stay at `/opt/finbot.env`. +then `util/docker_setup.sh` (it keeps your files) and start as above. ## Troubleshooting * **The container stops at once and keeps restarting:** the log says why. `ERROR config: ... unknown key ...` is a - mistake in `/opt/finbot.env` or `settings.conf`; fix it and `sudo docker compose up -d`. + mistake in `/etc/finbot.env` or `settings.conf`; fix it and `sudo docker compose up -d`. * **`PermissionError` on `/data`:** the data dir is not the container user's: `sudo chown -R 10001:10001 /opt/finbot-data`. * **`/data does not exist: --create makes it`:** the data dir is not mounted: `compose.yaml` mounts `../finbot-data`; run `util/docker_setup.sh` to make it. -* **No log in the journal:** the host has no journald; use `FINBOT_LOG_DRIVER=json-file` (see [Logs](#logs)). +* **No log in `journalctl`:** the container is not using journald; read it with `sudo docker compose logs -f` (see + [Logs](#logs)). diff --git a/doc/install.md b/doc/install.md index d0e01e9..154a81b 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,100 +1,151 @@ # Installing on a host -finbot runs wherever Python 3.12 or later does (Linux, macOS, WSL); production runs Debian's own packages under -systemd, as below. To run it in a container instead, see [docker.md](docker.md). +finbot runs wherever Python 3.12 or later does (Linux, macOS, WSL). It is developed and most tested on Debian with +Debian's packaged python modules. + +To run it in a container instead, see [docker.md](docker.md). ## Requirements -* Most features need a Sharesight paid plan, preferably with automatic trade imports, and an API key - ([configuration.md](configuration.md#sharesight)). -* A Telegram bot, a Slack app or a Discord bot: see the network's guide, [Telegram](telegram.md), [Slack](slack.md) or + +* Most features need a Sharesight paid plan and an API key ([configuration.md](configuration.md#sharesight)). + +* A Telegram bot, a Slack app or a Discord bot: see its guide, [Telegram](telegram.md), [Slack](slack.md) or [Discord](discord.md). -* By default, finbot fetches its updates from each network itself (pull mode: Telegram's long polling, Slack's Socket - Mode, Discord's gateway API). It only makes outgoing connections, so nothing has to listen on the internet. - Alternatively, if you prefer to run an HTTPS gateway (a web server with a domain name and a matching certificate), - Push mode subscribes the bot to push updates: each network posts its updates to the gateway, which passes them to - the bot (each guide's _push mode_). -* Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum), and these modules, pinned to versions with - 3.13 support in `requirements.txt`: requests, and Pillow for the charts. The charts are drawn in DejaVu Sans, which - most Linux systems have (Debian: `fonts-dejavu-core`); without it, Pillow's own font stands in. -* For a network in push mode (its webhook URL set), also waitress; for Slack's Socket Mode and Discord's gateway API, - websocket-client; for Discord's push mode, PyNaCl. These are in `requirements-optional.txt`, each imported only by - the modes that need it. Docker and a Python venv install them all, Docker in its image and a venv - from that file, so every mode works without choosing; on a Debian host, install the packages of the modes you use. + +* Python 3.13.5 (3.12 is the minimum), with `requests` and Pillow for the charts. The charts use DejaVu Sans, which most + Linux systems have (Debian: `fonts-dejavu-core`). + +* By default, finbot fetches its updates from your chatbot itself (pull mode). It only makes outgoing connections, so + nothing has to listen on the internet. + + Push mode is the alternative: your chatbot posts its updates to an HTTPS server you run. + +* One more library, for the way you receive updates: + + * `waitress` for push mode + * `websocket-client` for Slack or Discord in pull mode + * `PyNaCl` for Discord in push mode + + Docker and the venv below install them all. On Debian, install the ones you need. ## Debian -1. The packages, and those of the modes you use: + +1. Install the core dependencies, plus those for one or more chat methods: + ``` sudo apt install git python3 python3-requests python3-pil fonts-dejavu-core - sudo apt install python3-waitress # any network in push mode - sudo apt install python3-websocket # Slack's Socket Mode, Discord's gateway API - sudo apt install python3-nacl # Discord's push mode + + # Slack or Discord in pull mode + sudo apt install python3-websocket + + # Telegram or Slack in push mode + sudo apt install python3-waitress + + # Discord in push mode + sudo apt install python3-nacl python3-waitress ``` -2. The code, in `/opt/finbot` (not under `/home`: the service unit hides it): + +2. Get the code, in `/opt/finbot`: + ``` sudo git clone https://github.com/robdevops/finbot.git /opt/finbot ``` -3. The startup file, from its template; then set each network's keys from its guide, and Sharesight's: + +3. Make the startup env file from its template: + + ``` + sudo cp /opt/finbot/defaults/finbot.env /etc/finbot.env && sudo chmod 600 /etc/finbot.env && sudo chown root:root /etc/finbot.env + ``` + + Then add your chatbot's API key (see its guide above), and Sharesight's: + + ``` + sudoedit /etc/finbot.env + ``` + +4. Check it: + ``` - sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env && sudo chown root:root /opt/finbot.env - sudoedit /opt/finbot.env + /opt/finbot/util/check_config.py /etc/finbot.env ``` -4. Check it: `/opt/finbot/util/check_config.py /opt/finbot.env` (it also warns if the file is not `600 root:root`) -5. Run it under systemd ([below](#daemonize-systemd)). The first start makes the data directory, `/var/lib/finbot` - (systemd keeps it in `/var/lib/private/finbot`, where `/var/lib/finbot` points); each chat then has the default - scheduled jobs, which `/jobs` changes ([scheduling.md](scheduling.md)). -6. Optional, to change settings: copy the starter `settings.conf` into the data directory, edit it, check it and reload - (`util/sync_defaults.py` and `util/check_config.py` find the service's data directory when run by hand; run them - with `sudo`, as the directory is the service's): + +5. Run it under systemd, [below](#daemonize-systemd). + +6. Optional, to change settings. Copy the `settings.conf` template, edit it, check it, and reload: + ``` sudo /opt/finbot/util/sync_defaults.py --apply sudoedit /var/lib/finbot/settings.conf - sudo /opt/finbot/util/check_config.py /opt/finbot.env + sudo /opt/finbot/util/check_config.py /etc/finbot.env sudo systemctl reload finbot ``` - The files are root-owned and readable, so the service reads them; systemd gives the directory to the service's user - again at the next start. Leave `DATA_DIR` unset: under the unit, only the state directory is writable. [configuration.md](configuration.md) has every key and setting. ## Anywhere else: a Python venv -On Linux, macOS or WSL, with git and Python 3.12 or later (on Debian and Ubuntu, also `python3-venv`): + +For macOS, WSL, or any setup that does not use the Debian packages. You need git, and Python 3.12 or later with its +`venv` module. + +Use the same paths as the Debian steps, so the service below works unchanged: + ``` -git clone https://github.com/robdevops/finbot.git ~/finbot -cd ~/finbot -python3 -m venv .venv -.venv/bin/pip install -r requirements.txt -r requirements-optional.txt +sudo git clone https://github.com/robdevops/finbot.git /opt/finbot +cd /opt/finbot +sudo python3 -m venv .venv +sudo .venv/bin/pip install -r requirements.txt -r requirements-optional.txt ``` -Or, with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: + +Make the startup env file as in step 3 above, then check it: + ``` -uv venv && uv pip install -r requirements.txt -r requirements-optional.txt +sudo /opt/finbot/.venv/bin/python /opt/finbot/util/check_config.py /etc/finbot.env ``` -`requirements-optional.txt` installs the library of every mode; it can be cut down to the modes you use. Then make the -startup file as in step 3 above (anywhere: `~/finbot.env`, say), check it, and start the bot with its values in its -environment: + +To try the bot by hand: + ``` -.venv/bin/python util/check_config.py ~/finbot.env -(set -a && . ~/finbot.env && exec .venv/bin/python bot.py) +sudo sh -c 'set -a && . /etc/finbot.env && exec /opt/finbot/.venv/bin/python /opt/finbot/bot.py' ``` -Under systemd (below), with the checkout in `/opt/finbot` and its venv made there, `ExecStart=` runs the venv's Python: -`/opt/finbot/.venv/bin/python /opt/finbot/bot.py`. + +For a service, use systemd, below. ## Daemonize (systemd) -`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. It expects the -checkout in `/opt/finbot` and the startup file at `/opt/finbot.env` (see [`finbot.env`](configuration.md#finbotenv)), -as above, and needs no editing then. Copy it to `/etc/systemd/system/`, then enable and start it. -`systemctl reload finbot` re-reads `settings.conf` without a restart; a change to `finbot.env` needs -`systemctl restart finbot`: + +`finbot.service` runs finbot as a systemd service: it starts on boot, and restarts after a failure. + +It expects the Git checkout in `/opt/finbot` and the startup env file at `/etc/finbot.env`, as above. + ``` sudo cp -v /opt/finbot/finbot.service /etc/systemd/system/ sudo systemctl daemon-reload sudo systemctl enable finbot --now ``` -The unit runs finbot as a throwaway user (`DynamicUser=yes`) with its state in `/var/lib/finbot`, inside a sandbox: a -read-only filesystem but for that directory, no access to `/home`, `/boot`, `/media` or `/mnt`, no privileges, no -executable writable memory, and only the ordinary system calls and network families. Hence the checkout must not be -under `/home`. `systemd-analyze security finbot` scores it. `MemoryMax` and `TasksMax` are starting values: after a few -days, read `systemctl show finbot -p MemoryPeak -p TasksCurrent` and adjust them. -The bot's log: + +finbot's data (its state and settings) is in `/var/lib/finbot`. + +After a change, `sudo systemctl reload finbot` re-reads `settings.conf`. A change to `finbot.env` needs +`sudo systemctl restart finbot`. + +The log: + ``` journalctl -fu finbot ``` + +### Hardening (recommended) + +An optional file adds a sandbox to the unit. finbot can then only write to `/var/lib/finbot`, and +gets no privileges. + +``` +sudo mkdir -p /etc/systemd/system/finbot.service.d +sudo cp -v /opt/finbot/finbot.service.d/hardening.conf /etc/systemd/system/finbot.service.d/ +sudo systemctl daemon-reload && sudo systemctl restart finbot +``` + +It needs the Git checkout outside `/home`, as in `/opt/finbot`. + +`systemd-analyze security finbot` scores the result: about 1.3 (OK) with the file, about 8 (exposed) without. + +If the sandbox ever gets in the way, delete the file and restart finbot. diff --git a/doc/scheduling.md b/doc/scheduling.md index 24d4514..9a47c3f 100644 --- a/doc/scheduling.md +++ b/doc/scheduling.md @@ -17,11 +17,11 @@ never run, its name and its schedule in words. Under the list, in italics, is ho * not run yet -/jobs add REPORT SCHEDULE +/jobs add REPORT [ARGUMENT] [SCHEDULE] /jobs add reminder SCHEDULE "TEXT" ``` Under it is one row of buttons, each there only where it has something to act on (**+** always; **Edit** and **Run...** with any job, **Upcoming** with one that is on). Each screen is drawn in place of the last: -* **+**: what the job runs, how often, then the rest down to the minute ([below](#add-step-by-step)). +* **+**: what the job runs (for `movers` and `performance`, then its period: 1d, 7d, 1w, 1m, 3m or 1y), how often, then the rest down to the minute ([below](#add-step-by-step)). * **Edit**: the list with a button per job, the default ones too; a press selects the job (its button turns red, on Telegram, Slack and Discord alike) or takes it out of the selection again. It starts with the jobs that are off selected, and nothing changes until **OK**, which switches the selected jobs off and the rest on. **OK** shows only where the selection differs from the jobs as they are, and **Delete** only while any job is selected; **Delete** asks first ("Delete the 3 selected jobs?"), then deletes them. **Select all** selects every job, and reads **Select none** once all are selected. **Cancel** goes back to the list, whatever was selected. A default job that is deleted comes back when the same job is added again. * **Run...**: a button per job; a press runs it now, as its schedule would. What the job posts shows it ran; a message says so only when it failed, or was running already. The same button pressed again within a minute does @@ -53,7 +53,8 @@ them; `/treemap`, `/holdings` and `/price` run as typed bare. Then how often: A span across days (`every 20 mins from mon 9am to sat 3pm`), or a window of hours across midnight, is typed. The last screen is the whole list, headed "Added", the new job in bold with a 👉 before it, with the list's buttons (**+** adds another). A job added by -`/jobs add` shows the same. +`/jobs add` shows the same. `/jobs add performance 3m`, a report (and its one argument) without a schedule, opens the +same buttons at the how-often step, the argument carried to the job. Or type: ``` @@ -62,6 +63,7 @@ Or type: /jobs system the system jobs /jobs add what a job can run, what each takes, and examples /jobs add movers weekly +/jobs add performance 3m a report and its argument alone: buttons ask the rest /jobs add movers close weekdays at 4:30pm New York /jobs add peg top weekly /jobs add price AAPL MSFT 1y monthly @@ -92,8 +94,8 @@ A job is a report and what it takes, then when it runs: 4:30pm New York (7:30am local, Tue–Sat)`. A zone that keeps another clock is named in every season: a Brisbane job seen from Melbourne reads `weekdays at 9:00am Brisbane` in winter, when both clocks agree. -The reports are `trades`, `movers midsession|interday|premarket|close|DAYS`, `earnings`, -`performance [DAYS]`, `rating`, `milestone`, `shorts`, `podcasts FEED_URL` and `reminder "TEXT"` (`message` works too). A job a chat added +The reports are `trades`, `movers midsession|interday|premarket|close|PERIOD`, `earnings`, +`performance [PERIOD]`, `rating`, `milestone`, `shorts`, `podcasts FEED_URL` and `reminder "TEXT"` (`message` works too). A job a chat added as `price` before the movers report took its name runs as `movers`; `price` in a job added now is the line chart. A job can also run a command of `/help`, with what it takes, and posts what the command posts when typed in the chat: diff --git a/finbot.service b/finbot.service index e5068ae..ab9a991 100644 --- a/finbot.service +++ b/finbot.service @@ -3,67 +3,26 @@ Description=Finbot interactive chat bot [Service] Type=simple -# A throwaway user for each start; its state lives in /var/lib/finbot (StateDirectory=). The checkout is in /opt/finbot, -# not /home: ProtectHome=yes hides /home from the service +# No user to make: systemd runs finbot as a throwaway user, and gives it /var/lib/finbot for its state +# (StateDirectory=). The Git checkout is in /opt/finbot, which that user can read. +# Optional hardening: finbot.service.d/hardening.conf DynamicUser=yes StateDirectory=finbot ExecStart=/opt/finbot/bot.py # systemctl reload finbot re-reads settings.conf without a restart ExecReload=/bin/kill -HUP $MAINPID # Refuse to start on a typo in finbot.env or settings.conf ("+" runs the check as root) -ExecStartPre=+/opt/finbot/util/check_config.py /opt/finbot.env +ExecStartPre=+/opt/finbot/util/check_config.py /etc/finbot.env Restart=on-failure RestartSec=30s StandardOutput=null StandardError=journal SyslogIdentifier=finbot Environment=PYTHONUNBUFFERED=1 +# The venv in /opt/finbot/.venv, if there is one, is the python3 that bot.py finds +Environment=PATH=/opt/finbot/.venv/bin:/usr/local/bin:/usr/bin:/bin # finbot.env: loaded by systemd as root (600 root:root); a change needs a restart -EnvironmentFile=/opt/finbot.env - -# Filesystem isolation -ProtectSystem=strict -ProtectHome=yes -PrivateTmp=yes -PrivateDevices=yes -InaccessiblePaths=/boot /media /mnt -UMask=0077 - -# Privilege restrictions -AmbientCapabilities= -CapabilityBoundingSet= -LockPersonality=yes -# No JIT or other code generated at run time: finbot and its libraries need none -MemoryDenyWriteExecute=yes -NoNewPrivileges=yes -RestrictRealtime=yes -RestrictSUIDSGID=yes -SystemCallArchitectures=native -SystemCallErrorNumber=EPERM -SystemCallFilter=@system-service - -# Kernel and system protection -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectKernelLogs=yes -ProtectControlGroups=yes -ProtectClock=yes -ProtectHostname=yes - -# Process isolation -PrivateUsers=yes -ProtectProc=invisible -RestrictNamespaces=yes - -# Network restrictions: HTTPS out; in push mode, finbot listens on one local port -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 - -# Resource limits: starting values; read MemoryPeak and TasksCurrent from systemctl show finbot, then tune. Tasks are -# threads: one per message, button press, scheduled job and chart priming, and many jobs can fire in the same minute -MemoryAccounting=yes -MemoryHigh=300M -MemoryMax=400M -TasksMax=256 +EnvironmentFile=/etc/finbot.env [Install] WantedBy=multi-user.target diff --git a/finbot.service.d/hardening.conf b/finbot.service.d/hardening.conf new file mode 100644 index 0000000..5ccd64b --- /dev/null +++ b/finbot.service.d/hardening.conf @@ -0,0 +1,44 @@ +# Optional hardening for finbot.service: a sandbox. Install it next to the unit: +# sudo mkdir -p /etc/systemd/system/finbot.service.d +# sudo cp /opt/finbot/finbot.service.d/hardening.conf /etc/systemd/system/finbot.service.d/ +# sudo systemctl daemon-reload && sudo systemctl restart finbot +# Remove the file and restart to go back to the plain unit if the sandbox ever gets in the way. +# It needs the Git checkout outside /home (ProtectHome=yes) and finbot's state in the unit's StateDirectory, the only +# place it can write. +[Service] +# Filesystem isolation +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +InaccessiblePaths=/boot /media /mnt +UMask=0077 + +# Privilege restrictions +AmbientCapabilities= +CapabilityBoundingSet= +LockPersonality=yes +# No JIT or other code generated at run time: finbot and its libraries need none +MemoryDenyWriteExecute=yes +NoNewPrivileges=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +SystemCallArchitectures=native +SystemCallErrorNumber=EPERM +SystemCallFilter=@system-service + +# Kernel and system protection +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes + +# Process isolation +PrivateUsers=yes +ProtectProc=invisible +RestrictNamespaces=yes + +# Network restrictions: HTTPS out; in push mode, finbot listens on one local port +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 diff --git a/lib/config.py b/lib/config.py index eadce32..44fb134 100644 --- a/lib/config.py +++ b/lib/config.py @@ -8,7 +8,7 @@ from pathlib import Path # TO CONFIGURE, DO NOT EDIT THIS FILE. There are two config files: -# finbot.env the startup file: DATA_DIR, ip, port, credentials and the mode of each network. Never read by the +# finbot.env the startup env: DATA_DIR, ip, port, credentials and the mode of each network. Never read by the # bot: the launcher loads it into the environment (systemd EnvironmentFile=, docker --env-file, # init.d `set -a; . finbot.env`). A change needs a restart. # settings.conf how the bot behaves: /settings.conf, read by the bot itself at start and on every diff --git a/lib/jobs.py b/lib/jobs.py index ea2e378..74722df 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -312,6 +312,24 @@ def parse_add(text, zone): raise wrong or unread +def report_alone(text): + """What .jobs add was given (its text after add), where that is a report and perhaps its one argument, such as a + period, and no schedule: (action, its argument words), for Add's buttons to ask the schedule of; None for anything + else (a schedule is there, or the report is none, takes more than one word, or does not take what was typed).""" + known, words = actions(), text.split() + name = next((' '.join(words[:size]).lower() for size in (2, 1) if ' '.join(words[:size]).lower() in known), None) + if name is None or name in SYSTEM_ACTIONS: + return None + args = words[len(name.split()):] + if len(args) > 1 or any(not (arg.isascii() and arg.isalnum()) for arg in args): + return None + try: + job_args(known[name], name, args) + except ValueError: + return None + return name, args + + class Scheduler: """Runs the jobs: each chat's, from JOBS_FILE and the catalog, and the system jobs. clock, sleep and spawn can be replaced, which the tests do. Its lock covers what the loop, a change from chat and the runs share: the diff --git a/lib/reports/__init__.py b/lib/reports/__init__.py index 59c2d48..e68fe13 100644 --- a/lib/reports/__init__.py +++ b/lib/reports/__init__.py @@ -6,18 +6,24 @@ now. And the chat-only reports: about (.help, .jobs), history (.price, also called .compare and .history), holdings (.holdings, .who, .treemap, .watchlist), pick (the pickers' buttons), profile (.SYMBOL) and rankings (.marketcap, .ratio and the rest).""" +from lib import formats -def optional_days(name, key): - """The job() of a report, called name, whose scheduled run may give a number of days (its run's argument `key`): - its label is its name, with the days where given; anything else raises ValueError with what it takes.""" +def optional_days(name, key, usage='[DAYS]'): + """The job() of a report, called name, whose scheduled run may give a period (its run's argument `key`, in days): + a number of days, or a period as a chat command takes it (7d, 1w, 3m, ytd); its label is its name, with the period + as given; anything else raises ValueError with `usage`, what it takes.""" def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case []: return name, {} - case [days] if days.isdigit(): - return f'{name} {days}', {key: int(days)} + case [period]: + try: + days = formats.days_from_human_days(period) + except ValueError: + raise ValueError(usage) from None + return f'{name} {period}', {key: days} case _: - raise ValueError('[DAYS]') + raise ValueError(usage) return job diff --git a/lib/reports/about.py b/lib/reports/about.py index ddadb7e..8b6a7f4 100644 --- a/lib/reports/about.py +++ b/lib/reports/about.py @@ -38,7 +38,9 @@ # the default of each choice, marked † on its button; a one-off's are today's year and month, and tomorrow USUAL = {'frequency': 'd', 'weekday': '5', 'day': '1', 'month': '1', 'hour': '9', 'minute': '0', 'interval': WEEKDAY, 'repeat': 'd', 'start hour': '0', 'start minute': '0', 'end': END} +PERIODS = ('1d', '7d', '1w', '1m', '3m', '1y') # what Add offers a movers or performance job's period MORE = 'p1' # Add's data for the second page of the days of a month +TYPED = 'v' # starts Add's data for a report's argument typed with it (.jobs add performance 3m) INTERVALS = (5, 10, 15, 20, 30, 120, 180, 240, 360, 720) # minutes between a repeat's runs, as every... offers them MINUTES = tuple(range(0, 60, 5)) # the minutes Add offers YEARS = 10 # the years a one-off may be in, this one first @@ -240,7 +242,10 @@ def add_screen(service, chat, steps): steps, page = list(steps), 0 if steps and steps[-1] == MORE: steps, page = steps[:-1], 1 - back = buttons.menu_button('Back', 'add', steps[:-1], crc) + typed, core = None, steps # core: the steps without the report's typed argument (carried after its number) + if len(steps) > 1 and len(steps[1]) > 1 and steps[1][0] == TYPED: # as 'v3m'; no choice's data starts so + typed, core = steps[1][1:], [steps[0], *steps[2:]] + back = buttons.menu_button('Back', 'add', [] if typed and len(core) == 1 else steps[:-1], crc) cancel = buttons.menu_button('Cancel', 'top', [0], crc) def choices(heading, offer, row, lines=(), usual=None, nav=None): """A choice's screen: its heading and lines, and '† indicates default' in italics where one is (usual: its @@ -268,7 +273,13 @@ def turn(more): raise ValueError('no such page') nav = turn(len(offer) > most) + [cancel] return choices('Add a job: which report?', offer[page * most:(page + 1) * most], row, nav=nav) - label, action, args = reports[choice(steps[0], 1, len(reports)) - 1] + label, action, args = reports[choice(core[0], 1, len(reports)) - 1] + if typed: + try: + jobs.job_args(jobs.actions()[action], action, [typed]) + except (KeyError, ValueError): + raise ValueError('no such argument') from None + label, args = f'{label} {typed}', [typed] if action == 'message': return [service.bold('Add a reminder'), "A reminder's text is what you type:", '.jobs add reminder SCHEDULE "TEXT"', '', service.bold('For example:'), @@ -279,10 +290,12 @@ def turn(more): return [service.bold(f'Add {label}'), f'What {label} is about is typed:', f'.jobs add {action} {command.ARGUMENTS} SCHEDULE', '', service.bold('For example:'), f'.jobs add {action} {example(command.command)} weekly'], [[back, cancel]] - if len(steps) == 1: + if len(core) == 1 and not typed and args is None and action in ('movers', 'performance'): + return choices(f'Add {label}: which period?', [(period, TYPED + period) for period in PERIODS], 3) + if len(core) == 1: return choices(f'Add {label}: how often?', [(name, code) for code, name in FREQUENCIES], 3, usual=USUAL['frequency']) - frequency, made = steps[1], steps[2:] + frequency, made = core[1], core[2:] levels = levels_of(frequency, made) if len(made) > len(levels) or (page and levels[len(made):len(made) + 1] != ('day',)): raise ValueError('no such choices') @@ -314,6 +327,19 @@ def turn(more): 'start minute': ('start minute?', 4, []), 'end': ('end time?', 4, [])}[level] return choices(f'Add {label}, {said}: {question}', offer, row, lines, usual) +def add_typed(service, chat, name, args): + """Add's first screen for a report typed with its argument and no schedule (.jobs add performance 3m): how often, + the argument carried along: (lines, buttons); None where Add cannot ask the rest by buttons (a reminder, a command + that needs a stock or a portfolio typed, a report not offered).""" + reports = add_reports() + found = [number for number, (_, action, given) in enumerate(reports, 1) if action == name and (given is None or + given == ())] + if not found or (name in jobs.command_actions() and jobs.command_actions()[name].needs) or name == 'message': + return None + steps = [str(found[0]), *([TYPED + args[0]] if args else [])] + lines, markup = add_screen(service, chat, steps) + return listed_with(service, chat, lines), markup + def add_reports(): """What Add offers a job to run: the reports (REPORTS), then the commands of .help a job may run, in its order, each (label, action, arguments; None where the period follows how often it runs). Each one's place in this list is its diff --git a/lib/reports/movers.py b/lib/reports/movers.py index 8bbc4fc..79dd4bd 100644 --- a/lib/reports/movers.py +++ b/lib/reports/movers.py @@ -142,12 +142,12 @@ def report(service, found, mode='interday', threshold=None, days=None, top=None, chart = render.Chart('bars', heading(''), subtitle, tuple(label(found.rows)), session=session) return [service.bold(heading(f', {span}'))] + lines, chart worst = [i for i in reversed(range(len(found.rows))) if i >= top][:top] # the worst, worst first, none of the best - payload = [service.bold(f'Top {top} performers {span}')] + lines[:top] + payload = [service.bold(f'Top {top} movers {span}')] + lines[:top] if worst: payload += ['', service.bold(f'Bottom {top}')] + [lines[i] for i in worst] - bars = label(found.rows[:top]) # the best, then after a gap the worst, as the caption's two lists + bars = label(found.rows[:top]) # the best, then after a gap the worst, the worst last if worst: - bars += [('', None)] + label(found.rows[i] for i in worst) + bars += [("", None)] + label(found.rows[i] for i in reversed(worst)) return payload, render.Chart('bars', f"Top & bottom {top}", subtitle, tuple(bars)) def nothing(mode, threshold, user, ticker, in_session): @@ -191,14 +191,18 @@ def list_buttons(threshold=None, top=None, days=None): period = buttons.period_for_days(days, None if days else buttons.DEFAULT_PERIOD['l']) return [f"{threshold:g}", str(top or 0)], period -ARGUMENTS = 'midsession|interday|premarket|close|DAYS' # what a scheduled run takes +ARGUMENTS = 'midsession|interday|premarket|close|PERIOD' # what a scheduled run takes def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case ['midsession' | 'interday' | 'premarket' | 'close' as mode]: return f'movers {mode}', {'mode': mode} - case [days] if days.isdigit(): - return f'movers {days}', {'mode': 'period', 'threshold': 0, 'days': int(days), 'top': TOP} + case [period]: + try: + days = formats.days_from_human_days(period) + except ValueError: + raise ValueError(ARGUMENTS) from None + return f'movers {period}', {'mode': 'period', 'threshold': 0, 'days': days, 'top': TOP} case _: raise ValueError(ARGUMENTS) diff --git a/lib/reports/performance.py b/lib/reports/performance.py index 97e93f4..c2c5cc9 100644 --- a/lib/reports/performance.py +++ b/lib/reports/performance.py @@ -56,5 +56,5 @@ def run(chat, *, past_days=None, portfolio_select=None): delivery.deliver(chat, delivery.Report(payload, graph, buttons.keyboard(*attached))) return attached -ARGUMENTS = '[DAYS]' # what a scheduled run takes -job = reports.optional_days('performance', 'past_days') +ARGUMENTS = '[PERIOD]' # what a scheduled run takes +job = reports.optional_days('performance', 'past_days', ARGUMENTS) diff --git a/lib/worker.py b/lib/worker.py index 43d788b..48aaf43 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -627,7 +627,8 @@ def on_jobs(request, args): """.jobs: this chat's jobs, with the menu's buttons; .jobs upcoming: the next runs; .jobs system: the system's; .jobs enable, disable, delete, run or toggle NUMBER: one switched on or off, deleted, run now, or switched the other way; without a number, enable, disable and delete open Edit, run a button per job to run it, toggle the list; - .jobs add REPORT SCHEDULE, or .jobs add message SCHEDULE "TEXT": one more (alone, the reports a job can run).""" + .jobs add REPORT SCHEDULE, or .jobs add message SCHEDULE "TEXT": one more (alone, the reports a job can run; a + report and its period with no schedule, .jobs add performance 3m, has the buttons ask for the schedule).""" scheduler, chat, service = jobs.current, request.dest.key, request.service if scheduler is None: raise answer.Answer('No schedule is running') @@ -646,7 +647,15 @@ def on_jobs(request, args): raise Usage if not text.strip(): return request.reply(about.prepare_reports(service)) - number = scheduler.add(chat, *jobs.parse_add(text, formats.zone().key)) + try: + parsed = jobs.parse_add(text, formats.zone().key) + except jobs.ScheduleError: + alone = jobs.report_alone(text) # a report and no schedule: Add's buttons ask for it + started = about.add_typed(service, chat, *alone) if alone else None + if started is None: + raise + return request.reply(started[0], buttons=started[1]) + number = scheduler.add(chat, *parsed) lines, markup = about.added_screen(service, chat, number) request.reply(lines, buttons=markup) elif args.keyword in JOB_SCREENS and not args.words: diff --git a/tests/test_charts.py b/tests/test_charts.py index 0248c13..2a5717f 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -359,12 +359,12 @@ def test_a_label_too_wide_is_cut_short_to_fit(self): self.assertIsNone(canvas.fitted('Packaged Software', 8, 5, bold=True)) def test_movers_top_lists_best_first_then_worst(self): - """.movers top: the ten best, best first, a spacer, then the ten worst, worst first, as in the caption.""" + """.movers top: the ten best, best first, a spacer, then the ten worst, the worst last.""" chart = self.run_report(lambda: self.command('.movers top 7d')) values = [v for _, v in chart.rows if v is not None] top, bottom = values[:10], values[10:] self.assertEqual(top, sorted(top, reverse=True)) - self.assertEqual(bottom, sorted(bottom)) # the worst performer comes first, like the caption's second list + self.assertEqual(bottom, sorted(bottom, reverse=True)) # the worst performer is last self.assertGreater(min(top), max(bottom)) self.assertIn(None, [v for _, v in chart.rows]) # spacer between the two lists diff --git a/tests/test_config.py b/tests/test_config.py index f186b22..015e389 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,4 @@ -"""The two config files: finbot.env (the startup file, which the launcher loads) and settings.conf (reloadable).""" +"""The two config files: finbot.env (the startup env, which the launcher loads) and settings.conf (reloadable).""" import ast, contextlib, dataclasses, io, os, re, shutil, subprocess, sys, types, unittest from pathlib import Path from unittest import mock @@ -224,11 +224,11 @@ def stat_of(mode, uid=0, gid=0): """What os.stat says of a file with this mode, owner and group.""" return types.SimpleNamespace(st_mode=0o100000 | mode, st_uid=uid, st_gid=gid) with mock.patch.object(check_config.os, 'stat', return_value=stat_of(0o600)): - self.assertEqual(check_config.permission_warnings('/opt/finbot.env'), []) + self.assertEqual(check_config.permission_warnings('/etc/finbot.env'), []) with mock.patch.object(check_config.os, 'stat', return_value=stat_of(0o644)): - warning, = check_config.permission_warnings('/opt/finbot.env') - self.assertIn('/opt/finbot.env is 644 root:root, not 600 root:root', warning) - self.assertIn('chmod 600 /opt/finbot.env && chown root:root /opt/finbot.env', warning) + warning, = check_config.permission_warnings('/etc/finbot.env') + self.assertIn('/etc/finbot.env is 644 root:root, not 600 root:root', warning) + self.assertIn('chmod 600 /etc/finbot.env && chown root:root /etc/finbot.env', warning) with mock.patch.object(check_config.os, 'stat', return_value=stat_of(0o600, uid=54321, gid=54321)): self.assertIn('600 54321:54321, not 600 root:root', check_config.permission_warnings('x')[0]) self.assertEqual(check_config.permission_warnings('/no/such/finbot.env'), []) diff --git a/tests/test_datadir.py b/tests/test_datadir.py index 8930db0..ac84ce4 100644 --- a/tests/test_datadir.py +++ b/tests/test_datadir.py @@ -94,7 +94,7 @@ def test_sync_defaults(self): copied = Path(config.env.data_dir, 'settings.conf') # the template as it is: every key commented out self.assertEqual(copied.read_bytes(), (config.REPO_DIR / 'defaults' / 'settings.conf').read_bytes()) self.assertEqual(config.read_settings(copied), {}) - # the startup file is never copied into the data dir + # the startup env is never copied into the data dir self.assertFalse(Path(config.env.data_dir, 'finbot.env').exists()) mine = Path(config.env.data_dir, 'finbot_adr.json') mine.write_text('{"mine": 1}') diff --git a/tests/test_discord.py b/tests/test_discord.py index 667f451..b60b4bd 100644 --- a/tests/test_discord.py +++ b/tests/test_discord.py @@ -463,7 +463,7 @@ def test_the_jobs_subcommands_numbers_and_reports(self): self.assertEqual(self.values('/jobs disable '), ['disable 1', 'disable 3']) self.assertEqual(self.values('/jobs run ', chat='discord:88'), []) reports = worker.suggest('discord:77', '/jobs add ') - self.assertIn(('add movers midsession|interday|premarket|close|DAYS', 'add movers'), reports) + self.assertIn(('add movers midsession|interday|premarket|close|PERIOD', 'add movers'), reports) self.assertIn(('add message "TEXT"', 'add message'), reports) self.assertFalse(any(name in value for _, value in reports for name in jobs.SYSTEM_ACTIONS)) self.assertEqual(self.values('/jobs add movers m'), ['add movers midsession', 'add movers monthly']) diff --git a/tests/test_edges.py b/tests/test_edges.py index 4ef3ad6..7aeb52a 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -263,7 +263,7 @@ def test_keywords_in_any_case(self): ('.watchlist Remove x', 'delete')): with self.subTest(message=message): self.assertEqual(self.args(message)[1].keyword, keyword) - self.assertIn('Top 10 performers', str(self.command('.movers TOP 7d'))) + self.assertIn('Top 10 movers', str(self.command('.movers TOP 7d'))) def test_a_bare_number_on_movers_is_a_percent(self): """As on .session, .premarket and .shorts, and as doc/commands.md has it; a period carries its unit.""" diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 118830d..cc13ac6 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -130,6 +130,11 @@ def test_cal_and_performance(self): """earnings takes none; performance an optional number of days.""" self.assertEqual(performance.job([]), ('performance', {})) self.assertEqual(performance.job(['7']), ('performance 7', {'past_days': 7})) + self.assertEqual(performance.job(['1w']), ('performance 1w', {'past_days': 7})) + self.assertEqual(performance.job(['7d'])[1], {'past_days': 7}) + for text, period in (('performance 1w every saturday at 9am', '1w'), ('performance 7d saturdays at 9am', '7d'), + ('performance 1w weekly on saturday at 9am', '1w')): + self.assertEqual(jobs.parse_add(text, 'UTC')[:2], ('performance', [period]), text) for bad, module, name in ((['x'], earnings, 'earnings'), (['x'], performance, 'performance')): with self.assertRaises(ValueError): jobs.job_args(module, name, bad) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 5bdef7d..f5308ed 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -658,8 +658,8 @@ def test_add_alone_lists_the_reports_a_job_can_run(self): and examples.""" lines = self.say('.jobs add') self.assertEqual(lines[0], 'What a job can run:') - for line in ('reminder "TEXT"', 'performance [DAYS]', 'podcasts FEED_URL', - 'movers midsession|interday|premarket|close|DAYS', 'trades', '.jobs add movers weekly', 'beta', + for line in ('reminder "TEXT"', 'performance [PERIOD]', 'podcasts FEED_URL', + 'movers midsession|interday|premarket|close|PERIOD', 'trades', '.jobs add movers weekly', 'beta', 'ratio [SYMBOL] [forward|current|peg] [top|bottom|negative|reverse]', 'price [SYMBOL ... up to 8] [period]', '.jobs add ratio peg bottom weekly'): self.assertIn(line, lines) @@ -670,9 +670,9 @@ def test_a_job_that_cannot_be_is_refused_saying_why(self): that are no schedule, a zone that is none, a one-off gone by, a job here already: refused, saying why.""" for text, why in (('nothere daily', "no report 'nothere'"), ('trades.py daily', "no report 'trades.py'"), ('yahoo_warm daily', "no report 'yahoo_warm'"), ('os.system daily', "no report 'os.system'"), - ('movers bogus weekly', 'movers takes midsession|interday|premarket|close|DAYS'), + ('movers bogus weekly', 'movers takes midsession|interday|premarket|close|PERIOD'), ('trades 7 daily', 'trades takes no arguments'), ('trades fortnightly', "can't read 'fortnightly'"), - ('shorts daily at 9am Atlantis', "can't read 'atlantis'"), ('trades', 'no days or times'), + ('shorts daily at 9am Atlantis', "can't read 'atlantis'"), ('message on 1 jan 2020 "late"', 'once, 1 Jan 2020 has gone by'), ('trades every 20 mins weekdays UTC', 'that job is here already')): with self.subTest(text=text): @@ -969,26 +969,46 @@ def test_add_by_buttons_back_and_cancel(self): ['treemap', 'who', 'marketcap'], ['beta', 'price', 'buysell'], ['ratio', 'stock'], ['Cancel']]) self.assertEqual((data['movers'], data['Cancel']), (f'm|add|5|{self.crc()}', f'm|top|0|{self.crc()}')) lines, rows, data = self.press(data['movers']) - self.assertEqual((lines[:3], rows), (['Add movers: how often?', '', '† indicates default'], + self.assertEqual((lines[:2], rows), (['Add movers: which period?', ''], + [['1d', '7d', '1w'], ['1m', '3m', '1y'], ['Back', 'Cancel']])) + self.assertEqual(data['Back'], f'm|add|{self.crc()}') + lines, rows, data = self.press(data['1w']) + self.assertEqual((lines[:3], rows), (['Add movers 1w: how often?', '', '† indicates default'], [['one-off', 'hourly', 'daily†'], ['weekly', 'monthly', 'quarterly'], ['bi-annual', 'annual', 'every...'], ['Back', 'Cancel']])) lines, rows, data = self.press(data['weekly']) self.assertEqual(rows[:2], [['Mon', 'Tue', 'Wed', 'Thu'], ['Fri', 'Sat†', 'Sun']]) self.assertEqual(data['Cancel'], f'm|top|0|{self.crc()}') lines, rows, data = self.press(data['Fri']) - self.assertEqual(lines[:2], ['Add movers, weekly, Fridays: at what hour?', "By Melbourne's clock"]) + self.assertEqual(lines[:2], ['Add movers 1w, weekly, Fridays: at what hour?', "By Melbourne's clock"]) self.assertEqual((len(rows), [len(row) for row in rows], rows[1][4], rows[-1]), (5, [5] * 5, '09†', ['20', '21', '22', '23', 'Back'])) - self.assertEqual(data['Back'], f'm|add|5|w|{self.crc()}') + self.assertEqual(data['Back'], f'm|add|5|v1w|w|{self.crc()}') lines, rows, data = self.press(data['14']) self.assertEqual((rows[0], rows[-1]), (['14:00†', '14:05', '14:10', '14:15'], ['Back', 'Cancel'])) lines, rows, data = self.press(data['14:30']) - self.assertEqual(newest(lines), '3. * movers · weekly, Fridays at 2:30pm') + self.assertEqual(newest(lines), '3. * movers 1w · weekly, Fridays at 2:30pm') self.assertEqual((lines[0], rows, data['+']), ('Added', [['+', 'Edit', 'Run...', 'Upcoming']], f'm|add|{self.crc()}')) # the whole list, the job added in bold, with the list's buttons - self.assertEqual(state.load(jobs.JOBS_FILE)[CHAT]['jobs'][-1], entry('movers every friday at 2:30pm', + self.assertEqual(state.load(jobs.JOBS_FILE)[CHAT]['jobs'][-1], entry('movers 1w every friday at 2:30pm', zone='Australia/Melbourne')) - self.assertIn('that job is here already', self.say('.jobs add movers weekly on friday at 14:30')[0]) + self.assertIn('that job is here already', self.say('.jobs add movers 1w weekly on friday at 14:30')[0]) + + def test_a_report_and_its_argument_typed_alone_open_the_buttons_for_the_rest(self): + """'.jobs add performance 3m' (or a report with no argument) asks how often with Add's buttons, the argument + carried to the job they make; a reminder, or an argument the report refuses, is not taken.""" + lines = self.say('.jobs add performance 3m') + self.assertEqual(lines[0], 'Add performance 3m: how often?') + data = {button.label: button.data for row in self.replies[-1][2] for button in row} + lines, rows, data = self.press(data['quarterly']) + self.assertIn('Add performance 3m, quarterly', lines[0]) + for label in ('Feb', '15', '09', '09:00'): + lines, rows, data = self.press(next(data[key] for key in data if key.startswith(label))) + self.assertEqual(state.load(jobs.JOBS_FILE)[CHAT]['jobs'][-1], entry('performance 3m quarterly from feb on the' + ' 15th at 9am', zone='Australia/Melbourne')) + self.assertIn('which period?', self.say('.jobs add performance')[0]) + self.assertNotIn('how often?', ' '.join(self.say('.jobs add performance nonsense'))) + self.assertNotIn('how often?', ' '.join(self.say('.jobs add reminder'))) def test_the_month_steps_start_a_cycle(self): """Monthly asks for the day of the month; quarterly, bi-annual and annual first for the start month, the cycle @@ -1010,14 +1030,15 @@ def test_the_month_steps_start_a_cycle(self): with self.subTest(frequency=frequency): lines, rows, data = self.press(f'm|add|{self.crc()}') lines, rows, data = self.press(data['movers']) + lines, rows, data = self.press(data['1w']) lines, rows, data = self.press(data[frequency]) - self.assertEqual((lines[0], rows[0]), (f'Add movers, {frequency}: start month?', ['Jan†', 'Feb', - 'Mar', 'Apr'])) + self.assertEqual((lines[0], rows[0]), (f'Add movers 1w, {frequency}: start month?', + ['Jan†', 'Feb', 'Mar', 'Apr'])) lines, rows, data = self.press(data[start]) - self.assertEqual(lines[0], f'Add movers, {said}: on which day of the month?') + self.assertEqual(lines[0], f'Add movers 1w, {said}: on which day of the month?') for pressed in ('15', '09†', '09:00†'): lines, rows, data = self.press(data[pressed]) - self.assertTrue(newest(lines).endswith(f'* movers · {added}'), lines) + self.assertTrue(newest(lines).endswith(f'* movers 1w · {added}'), lines) lines, rows, data = self.press(f'm|add|5|a|2|{self.crc()}') self.assertEqual((lines[0], rows[-2]), ('Add movers, annual, in Feb: on which day of the month?', ['26', '27', '28', '29'])) diff --git a/util/check_config.py b/util/check_config.py index 0f0466a..b62861b 100755 --- a/util/check_config.py +++ b/util/check_config.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Lint finbot's two config files with the rules the bot applies, without starting it. - util/check_config.py /opt/finbot.env [/var/lib/finbot/settings.conf] + util/check_config.py /etc/finbot.env [/var/lib/finbot/settings.conf] -finbot.env (the startup file) is never read by the bot, so this is the only place its +finbot.env (the startup env) is never read by the bot, so this is the only place its typos are caught: an unknown key, a setting key in it, a malformed line or a bad value. settings.conf defaults to /settings.conf and is checked as the bot checks it at start and on reload. It also warns when finbot.env is not 600 root:root, as it holds @@ -99,7 +99,7 @@ def permission_warnings(path): if __name__ == '__main__': ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument('startup', help='the finbot.env to check (the startup file)') + ap.add_argument('startup', help='the finbot.env to check (the startup env)') ap.add_argument('settings', nargs='?', default=config.settings_path, help='default: /settings.conf') args = ap.parse_args() diff --git a/util/docker_setup.sh b/util/docker_setup.sh index 312d5fd..35661c9 100755 --- a/util/docker_setup.sh +++ b/util/docker_setup.sh @@ -1,14 +1,14 @@ #!/bin/sh # The host side of finbot's Docker install, set up once, as root; safe to run again (doc/docker.md). Beside the -# checkout, where compose.yaml looks for them: ../finbot-data (DATA_DIR) made, the defaults copied in, owned by the -# container's user (10001); and ../finbot.env from the template if there is none, readable by root and the container's -# group only. Cloned to /opt/finbot, they are /opt/finbot-data and /opt/finbot.env. +# checkout, where compose.yaml looks for it: ../finbot-data (DATA_DIR) made, the defaults copied in, owned by the +# container's user (10001), so /opt/finbot-data for a checkout in /opt/finbot; and /etc/finbot.env, the startup env, +# from the template if there is none, readable by root and the container's group only. set -eu repo=$(cd "$(dirname "$0")/.." && pwd) parent=$(dirname "$repo") data="$parent/finbot-data" -startup="$parent/finbot.env" +startup=/etc/finbot.env if [ "$(id -u)" -ne 0 ]; then echo "docker_setup: run it as root: sudo $0" >&2 @@ -25,10 +25,17 @@ fi chown root:10001 "$startup" chmod 640 "$startup" +# the host's journal takes the log where journald runs, so that it outlasts a rebuild; Docker's own `local` driver +# (compose.yaml's default) elsewhere. Never replaces a .env that is there already +if [ -S /run/systemd/journal/socket ] && [ ! -e "$repo/.env" ]; then + echo "FINBOT_LOG_DRIVER=journald" > "$repo/.env" + echo "docker_setup: the log goes to the journal ($repo/.env)" +fi + cat <