diff --git a/.gitignore b/.gitignore index 389ab26..c7af953 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,8 @@ __pycache__/ # GPU epingle sur machine partagee — voir embedding.py /client/active-gpu.conf # reglages propres a la machine, ecrits par scripts/spear-configure +/client/local.conf +/client/local.conf.bak /client/machine.env /client/machine.env.bak diff --git a/client/cli/machine_config.py b/client/cli/machine_config.py index 852855e..6d05721 100644 --- a/client/cli/machine_config.py +++ b/client/cli/machine_config.py @@ -1,9 +1,10 @@ -"""Write client/machine.env from what this machine already has. +"""Write client/local.conf from what this machine already has. -machine.env is this machine's local configuration -- where the session +local.conf is this machine's single configuration -- where the session accumulates, where a deployment's private content lives, which host embeds a -standard and with which pinned model -- and it was written by hand, with no -template, from reading the docs. Every value in it can be found instead: +standard and with which pinned model, and the image-tier and registry choices +below the local marker. This writes the derived top block; everything below +the marker is the operator's own. Every derived value can be found instead: state SPEAR_STATE_DIR, else the workstation default private tree a sibling spear-private/ holding rules.d, skills, benches @@ -35,7 +36,9 @@ APP_DIR = Path(__file__).resolve().parent.parent REPO = APP_DIR.parent -TARGET = APP_DIR / "machine.env" +TARGET = APP_DIR / "local.conf" +LEGACY_TARGET = APP_DIR / "machine.env" +EXAMPLE = APP_DIR / "local.conf.example" DEFAULT_STATE = "$HOME/.local/state/spear" LOCAL_MARKER = "# --- local additions: kept as they are by spear-configure ---" @@ -199,9 +202,9 @@ def _comparable(text: str) -> list[str]: def main(argv=None, *, remote_revision=None, stdin=None) -> int: ap = argparse.ArgumentParser( prog="spear-configure", - description="Write client/machine.env from what this machine has.") + description="Write client/local.conf from what this machine has.") ap.add_argument("--check", action="store_true", - help="say whether machine.env still matches; write nothing") + help="say whether local.conf still matches; write nothing") ap.add_argument("--yes", action="store_true", help="replace a differing machine.env without asking") ap.add_argument("--private", metavar="DIR", @@ -236,7 +239,22 @@ def main(argv=None, *, remote_revision=None, stdin=None) -> int: existing = target.read_text() if target.exists() else None state = args.state or os.environ.get("SPEAR_STATE_DIR") or DEFAULT_STATE - text = render(state, private, embed, split_local(existing or "")) + + # What goes below the marker: an existing file's own additions, else -- when + # first writing local.conf -- those migrated from a legacy machine.env, else + # the commented menu seeded from local.conf.example. + if existing is not None: + local = split_local(existing) + elif target.name == "local.conf" and LEGACY_TARGET.exists(): + local = split_local(LEGACY_TARGET.read_text()) + print(f" adopting {LEGACY_TARGET.name} -> {target.name} " + "(its settings carried over; it is no longer read)") + elif target.name == "local.conf" and EXAMPLE.exists(): + local = split_local(EXAMPLE.read_text()) + else: + local = "" + + text = render(state, private, embed, local) print(f" state {state}") print(f" private {private or '(none: in-tree rules, skills, benches)'}") diff --git a/client/local.conf.example b/client/local.conf.example new file mode 100644 index 0000000..3ebf03a --- /dev/null +++ b/client/local.conf.example @@ -0,0 +1,84 @@ +# client/local.conf -- this machine's single SPEAR configuration. +# +# One file for every operator choice: where a deployment's content lives, what +# the image tiers bake, and which entity and registry an image belongs to. It +# is the source of truth read by spear-chat, spear-corpus, spear-image +# (build and push) and the docker build scripts. It is not tracked. +# +# Copy it to local.conf and uncomment what applies: a line left commented is a +# choice not made, and the harness falls back to its default. `spear-configure` +# writes the top block for you from what the machine has, and keeps everything +# below the local marker untouched -- so the two ways of filling this file do +# not fight. + +# ==================== Deployment: where content lives ==================== + +# Where sessions accumulate: history, memories, audit, the standard store. +export SPEAR_STATE_DIR="$HOME/.local/state/spear" + +# The deployment's own content, from a private tree beside the checkout. +export SPEAR_RULES_DIR=/path/to/spear-private/rules.d +export SPEAR_SKILLS_DIR=/path/to/spear-private/skills +export SPEAR_BENCH_DIR=/path/to/spear-private/benches +[ -r /path/to/spear-private/engagement.env ] && . /path/to/spear-private/engagement.env + +# Document embedding for normative standards, on the GPU host. The revision is +# what both ends' Hub caches resolve `main` to; a licensed standard never +# leaves this machine unless an ingest/rebuild passes --allow-offload. +#export SPEAR_STANDARD_EMBED_MODEL=BAAI/bge-m3 +#export SPEAR_STANDARD_EMBED_REVISION=<40-hex revision> +#export SPEAR_STANDARD_EMBED_REMOTE=user@gpu-host + +# Retrieval mode and structural completion are set per document +# (/standard retrieval), not here: exported, they would override every +# document for every session. + +# --- local additions: kept as they are by spear-configure --- + +# ===================== Image tiers: spear-image build ===================== +# These replace the build-tiered.sh flags. A flag given on the command line +# still overrides the value set here, so an occasional one-off needs no edit. + +# The entity an image belongs to; it decides the registry a private image may +# be pushed to (see Registries below). "local" (the default) means none. +#export SPEAR_ENTITY=acme + +# base -- the common corpora baked into a company's commons image, the shared +# build system included (was --commons). +#export SPEAR_COMMONS=buildsystem,commons-docs + +# base -- the knowledge-seed workspaces to bake, or "all" (was --knowledge). +#export SPEAR_KNOWLEDGE=all + +# project -- this project's own spear-private content trees, colon separated +# (was --tree), and the base it builds on (was --from; empty => spear:-base). +#export SPEAR_PROJECT_TREES=/path/to/project/spear-private +#export SPEAR_FROM_BASE= + +# Corpora to bake writable into the image rather than mount at run time (was +# --bake). Empty => the trees stay mounted and live. +#export SPEAR_BAKE= + +# A store holding a standard this image is entitled to bake (was --standards), +# and whether it carries that licensed document (was --licensed). A licensed +# standard -- or the build system in a base -- makes the image +# non-redistributable. +#export SPEAR_STANDARDS_STORE= +#export SPEAR_LICENSED=0 + +# Further content trees a base merges after this machine's own, colon separated +# (a project checkout carrying its own rules.d/, skills/, benches/). +#export SPEAR_CONTENT_TREES=/path/to/project/spear-private + +# The standard bodies whose identifiers the harness recognises beyond the +# built-in ones, comma or space separated (e.g. a body only this image serves). +#export SPEAR_STANDARD_BODIES= + +# ========================== Registries: push ============================= +# Where a private image of each entity may be pushed. An image goes only to its +# own entity's registry; anywhere else needs push --allow-push. Some entities +# have a built-in default (see scripts/docker/push.sh); any other is named +# here, one variable per entity: +# +# export SPEAR__REGISTRY= +#export SPEAR_ACME_REGISTRY=registry.example.com diff --git a/client/machine.env.example b/client/machine.env.example deleted file mode 100644 index fb9aed7..0000000 --- a/client/machine.env.example +++ /dev/null @@ -1,29 +0,0 @@ -# The shape of client/machine.env, this machine's local configuration. -# Do not copy it by hand: `spear-configure` (after `. ./env.sh` at the -# repository root) writes machine.env from what the machine has, and -# `spear-configure --check` says whether an existing one still matches. - -# Where sessions accumulate: history, memories, audit, the standard store. -export SPEAR_STATE_DIR="$HOME/.local/state/spear" - -# The deployment's own content, from a private tree beside the checkout. -export SPEAR_RULES_DIR=/path/to/spear-private/rules.d -export SPEAR_SKILLS_DIR=/path/to/spear-private/skills -export SPEAR_BENCH_DIR=/path/to/spear-private/benches -[ -r /path/to/spear-private/engagement.env ] && . /path/to/spear-private/engagement.env - -# Document embedding for normative standards, on the GPU host, pinned to the -# revision both ends' Hub caches resolve `main` to. -export SPEAR_STANDARD_EMBED_MODEL=BAAI/bge-m3 -export SPEAR_STANDARD_EMBED_REVISION=<40-hex revision> -export SPEAR_STANDARD_EMBED_REMOTE=user@gpu-host - -# Retrieval mode and structural completion are set per document -# (/standard retrieval), not here: exported, they would override every -# document for every session. - -# Further content trees a reds image merges after this machine's own, colon -# separated: a project checkout that carries its rules.d/, skills/, benches/. -# export SPEAR_CONTENT_TREES=/path/to/project/spear-private - -# --- local additions: kept as they are by spear-configure --- diff --git a/client/spear-chat.sh b/client/spear-chat.sh index e7c8646..dae1bf9 100755 --- a/client/spear-chat.sh +++ b/client/spear-chat.sh @@ -21,10 +21,15 @@ PY="$SCRIPT_DIR/bin/python3" export VIRTUAL_ENV="$SCRIPT_DIR" export PATH="$SCRIPT_DIR/bin:$PATH" -# Settings specific to THIS machine (untracked). Used to point at an inference -# endpoint that is already up: on a shared host the --local :8080 may well be -# taken by something else entirely. -[ -r "$SCRIPT_DIR/machine.env" ] && . "$SCRIPT_DIR/machine.env" +# Settings specific to THIS machine (untracked): local.conf, or the legacy +# machine.env if no local.conf has been written yet. Used to point at an +# inference endpoint that is already up: on a shared host the --local :8080 may +# well be taken by something else entirely. +if [ -r "$SCRIPT_DIR/local.conf" ]; then + . "$SCRIPT_DIR/local.conf" +elif [ -r "$SCRIPT_DIR/machine.env" ]; then + . "$SCRIPT_DIR/machine.env" +fi # ── move under the user manager when needed ── # The sandbox's resource control needs delegated cgroup controllers, and diff --git a/client/spear-corpus.sh b/client/spear-corpus.sh index 1455190..83560f8 100755 --- a/client/spear-corpus.sh +++ b/client/spear-corpus.sh @@ -32,7 +32,11 @@ PY="$SCRIPT_DIR/bin/python3" # where a session's state lives -- and `spear-corpus` silently created a second # state tree beside the code. SPEAR_CORPUS_ROOT, which decides what a relative # corpus path resolves against, was missed the same way. -[ -r "$SCRIPT_DIR/machine.env" ] && . "$SCRIPT_DIR/machine.env" +if [ -r "$SCRIPT_DIR/local.conf" ]; then + . "$SCRIPT_DIR/local.conf" +elif [ -r "$SCRIPT_DIR/machine.env" ]; then + . "$SCRIPT_DIR/machine.env" +fi exec "$PY" -c 'import sys sys.path.insert(0, sys.argv[1]) diff --git a/client/tests/test_launcher_environment.py b/client/tests/test_launcher_environment.py index da01783..3168921 100644 --- a/client/tests/test_launcher_environment.py +++ b/client/tests/test_launcher_environment.py @@ -2,8 +2,9 @@ `spear-corpus` and the in-chat `/corpus` are one implementation on purpose -- a second one had already drifted. But the two entry points did not run in the -same environment: `spear-chat.sh` sources the machine's untracked `machine.env` -and `spear-corpus.sh` did not, so the CLI ran with `SPEAR_STATE_DIR` defaulting +same environment: `spear-chat.sh` sources the machine's untracked `local.conf` +(or legacy `machine.env`) and `spear-corpus.sh` did not, so the CLI ran with +`SPEAR_STATE_DIR` defaulting to the app directory while the chat used the configured one. Importing the module creates that directory, so the CLI quietly grew a second state tree beside the code; and `SPEAR_CORPUS_ROOT`, which decides what a *relative* @@ -40,7 +41,7 @@ class Deployment: """A throwaway app directory holding one launcher and a stub interpreter.""" - def __init__(self, tmp, launcher, machine_env=None): + def __init__(self, tmp, launcher, machine_env=None, local_conf=None): self.dir = Path(tmp) (self.dir / "bin").mkdir(parents=True, exist_ok=True) shutil.copy(APP / launcher, self.dir / launcher) @@ -50,6 +51,8 @@ def __init__(self, tmp, launcher, machine_env=None): if machine_env is not None: (self.dir / "machine.env").write_text(machine_env) + if local_conf is not None: + (self.dir / "local.conf").write_text(local_conf) self.launcher = self.dir / launcher @@ -113,11 +116,27 @@ class TheLaunchersAgree(unittest.TestCase): def test_every_launcher_reads_the_machine_environment(self): for name in LAUNCHERS: with self.subTest(launcher=name): + text = (APP / name).read_text() self.assertIn( - '[ -r "$SCRIPT_DIR/machine.env" ] && . "$SCRIPT_DIR/machine.env"', - (APP / name).read_text(), - f"{name} does not read machine.env; the entry points would " + '. "$SCRIPT_DIR/local.conf"', text, + f"{name} does not read local.conf; the entry points would " f"then disagree about where state lives") + self.assertIn( + '. "$SCRIPT_DIR/machine.env"', text, + f"{name} dropped the legacy machine.env fallback") + + def test_local_conf_is_read_and_wins_over_machine_env(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + app = Deployment( + tmp, "spear-corpus.sh", + machine_env='export SPEAR_STATE_DIR="/var/tmp/legacy"\n', + local_conf='export SPEAR_STATE_DIR="/var/tmp/current"\n') + done, reported = app.run("list") + + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(reported["SPEAR_STATE_DIR"], "/var/tmp/current") def test_the_chat_and_the_corpus_cli_report_the_same_environment(self): import tempfile @@ -155,17 +174,23 @@ def test_the_chat_and_the_corpus_cli_report_the_same_environment(self): class ThePublicCheckoutIsSelfContained(unittest.TestCase): """Sourcing a machine file must not make the repository need one.""" + MACHINE_FILES = ("client/local.conf", "client/machine.env") + def test_no_launcher_is_tracked_with_a_machine_file(self): tracked = subprocess.run(["git", "ls-files"], cwd=ROOT, capture_output=True, text=True, check=True) - self.assertNotIn("client/machine.env", tracked.stdout.split()) + for path in self.MACHINE_FILES: + with self.subTest(path=path): + self.assertNotIn(path, tracked.stdout.split()) def test_the_machine_file_is_ignored_rather_than_merely_absent(self): - done = subprocess.run(["git", "check-ignore", "-q", - "client/machine.env"], cwd=ROOT) - self.assertEqual(done.returncode, 0, - "client/machine.env is not gitignored, so a local " - "deployment's paths could be committed by accident") + for path in self.MACHINE_FILES: + with self.subTest(path=path): + done = subprocess.run(["git", "check-ignore", "-q", path], + cwd=ROOT) + self.assertEqual(done.returncode, 0, + f"{path} is not gitignored, so a local " + "deployment's paths could be committed by accident") # Assembled from fragments, like the public-boundary scanner's own needles: # a test that spells a forbidden path out loud puts it in the tree it is diff --git a/scripts/docker/build-tiered.sh b/scripts/docker/build-tiered.sh index 5af9499..43b72ee 100755 --- a/scripts/docker/build-tiered.sh +++ b/scripts/docker/build-tiered.sh @@ -2,11 +2,16 @@ # Build the tiered images: harness -> base -> project. # # build-tiered.sh harness -# build-tiered.sh base --commons a,b,c [--entity NAME] [--licensed] +# build-tiered.sh base [--commons a,b,c] [--entity NAME] [--licensed] # [--knowledge p,q|all] -# build-tiered.sh project --from spear:-base --tree DIR +# build-tiered.sh project [--from spear:-base] [--tree DIR] # [--bake a,b] [--standards DIR] [--licensed] [--tag TAG] # +# Every option has an equivalent in client/local.conf (SPEAR_COMMONS, +# SPEAR_ENTITY, SPEAR_BAKE, SPEAR_PROJECT_TREES, ...), read before the flags; a +# flag overrides the configured value. So on a machine whose local.conf names +# its commons and entity, `build-tiered.sh base` alone is enough. +# # A project adds its own standards too: --licensed takes the licensed documents # of --standards DIR (a store holding, say, only one private standard), appended over the # base's. So an edgemtech project image = the reds base + a private standard. @@ -38,10 +43,34 @@ case "$TIER" in *) die "tier must be harness, base or project, not '$TIER'" ;; esac -COMMONS="" BAKE="" KNOWLEDGE="" ENTITY="local" FROM_BASE="" TAG="" -STANDARDS_STORE="" -LICENSED=0 +# The machine's configuration, exactly as build.sh and the launchers read it: +# the content locations a base bakes, and the tier choices that used to be +# flags (SPEAR_COMMONS, SPEAR_BAKE, SPEAR_ENTITY, ...). local.conf, or the +# legacy machine.env. Without it a base ships no rules and no skills. + +if [ -r "$APP/local.conf" ]; then + echo " settings $APP/local.conf" + # shellcheck disable=SC1091 + . "$APP/local.conf" +elif [ -r "$APP/machine.env" ]; then + echo " settings $APP/machine.env" + # shellcheck disable=SC1091 + . "$APP/machine.env" +fi + +# The choices come from the configuration above; a flag below overrides it, and +# --tree adds to any SPEAR_PROJECT_TREES already set. + +COMMONS="${SPEAR_COMMONS:-}" +BAKE="${SPEAR_BAKE:-}" +KNOWLEDGE="${SPEAR_KNOWLEDGE:-}" +ENTITY="${SPEAR_ENTITY:-local}" +FROM_BASE="${SPEAR_FROM_BASE:-}" +TAG="${SPEAR_TAG:-}" +STANDARDS_STORE="${SPEAR_STANDARDS_STORE:-}" +LICENSED="${SPEAR_LICENSED:-0}" TREES=() +[ -n "${SPEAR_PROJECT_TREES:-}" ] && IFS=: read -r -a TREES <<<"$SPEAR_PROJECT_TREES" while [ $# -gt 0 ]; do case "$1" in @@ -60,15 +89,6 @@ done [ -f "$APP/cli/rag_chat.py" ] || die "no harness under $APP — set SPEAR_APP" -# The machine's own content locations and content trees, exactly as build.sh and -# the launchers read them. Without it a base ships no rules and no skills. - -if [ -r "$APP/machine.env" ]; then - echo " settings $APP/machine.env" - # shellcheck disable=SC1091 - . "$APP/machine.env" -fi - EMPTY="$(mktemp -d)" STAGE="$(mktemp -d)" trap 'rm -rf "$EMPTY" "$STAGE"' EXIT diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 762fb18..c37a6fc 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -115,9 +115,13 @@ TAG="${TAG:-spear:$("$REPO/scripts/spearversion.sh")-$PROFILE}" # checkout. Without it the OPTIONAL table below resolves to the in-tree # directories -- which hold a README and nothing else -- and the build reports # them as found. The image then ships a harness with no rules and no skills, -# announcing neither, which is exactly what machine.env exists to prevent on -# the workstation. -if [ -r "$APP/machine.env" ]; then +# announcing neither, which is exactly what the configuration exists to prevent +# on the workstation. local.conf, or the legacy machine.env. +if [ -r "$APP/local.conf" ]; then + echo " settings $APP/local.conf" + # shellcheck disable=SC1091 + . "$APP/local.conf" +elif [ -r "$APP/machine.env" ]; then echo " settings $APP/machine.env" # shellcheck disable=SC1091 . "$APP/machine.env" diff --git a/scripts/spear-configure b/scripts/spear-configure index cd437c8..cf4533e 100755 --- a/scripts/spear-configure +++ b/scripts/spear-configure @@ -1,6 +1,6 @@ #!/bin/bash -# Write client/machine.env, this machine's local configuration, from what the -# machine already has. See client/cli/machine_config.py, or spear-configure --help. +# Write client/local.conf, this machine's configuration, from what the machine +# already has. See client/cli/machine_config.py, or spear-configure --help. REPO="$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)" # The release this is: "[spear vX.Y.Z]