Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ __pycache__/
# GPU epingle sur machine partagee — voir embedding.py
/client/active-gpu.conf
# reglages propres a la machine, ecrits par scripts/spear-configure
/client/local.conf
/client/local.conf.bak
/client/machine.env
/client/machine.env.bak

Expand Down
34 changes: 26 additions & 8 deletions client/cli/machine_config.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
"""Write client/machine.env from what this machine already has.
"""Write client/local.conf from what this machine already has.

machine.env is this machine's local configuration -- where the session
local.conf is this machine's single configuration -- where the session
accumulates, where a deployment's private content lives, which host embeds a
standard and with which pinned model -- and it was written by hand, with no
template, from reading the docs. Every value in it can be found instead:
standard and with which pinned model, and the image-tier and registry choices
below the local marker. This writes the derived top block; everything below
the marker is the operator's own. Every derived value can be found instead:

state SPEAR_STATE_DIR, else the workstation default
private tree a sibling spear-private/ holding rules.d, skills, benches
Expand Down Expand Up @@ -35,7 +36,9 @@

APP_DIR = Path(__file__).resolve().parent.parent
REPO = APP_DIR.parent
TARGET = APP_DIR / "machine.env"
TARGET = APP_DIR / "local.conf"
LEGACY_TARGET = APP_DIR / "machine.env"
EXAMPLE = APP_DIR / "local.conf.example"

DEFAULT_STATE = "$HOME/.local/state/spear"
LOCAL_MARKER = "# --- local additions: kept as they are by spear-configure ---"
Expand Down Expand Up @@ -199,9 +202,9 @@ def _comparable(text: str) -> list[str]:
def main(argv=None, *, remote_revision=None, stdin=None) -> int:
ap = argparse.ArgumentParser(
prog="spear-configure",
description="Write client/machine.env from what this machine has.")
description="Write client/local.conf from what this machine has.")
ap.add_argument("--check", action="store_true",
help="say whether machine.env still matches; write nothing")
help="say whether local.conf still matches; write nothing")
ap.add_argument("--yes", action="store_true",
help="replace a differing machine.env without asking")
ap.add_argument("--private", metavar="DIR",
Expand Down Expand Up @@ -236,7 +239,22 @@ def main(argv=None, *, remote_revision=None, stdin=None) -> int:

existing = target.read_text() if target.exists() else None
state = args.state or os.environ.get("SPEAR_STATE_DIR") or DEFAULT_STATE
text = render(state, private, embed, split_local(existing or ""))

# What goes below the marker: an existing file's own additions, else -- when
# first writing local.conf -- those migrated from a legacy machine.env, else
# the commented menu seeded from local.conf.example.
if existing is not None:
local = split_local(existing)
elif target.name == "local.conf" and LEGACY_TARGET.exists():
local = split_local(LEGACY_TARGET.read_text())
print(f" adopting {LEGACY_TARGET.name} -> {target.name} "
"(its settings carried over; it is no longer read)")
elif target.name == "local.conf" and EXAMPLE.exists():
local = split_local(EXAMPLE.read_text())
else:
local = ""

text = render(state, private, embed, local)

print(f" state {state}")
print(f" private {private or '(none: in-tree rules, skills, benches)'}")
Expand Down
84 changes: 84 additions & 0 deletions client/local.conf.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# client/local.conf -- this machine's single SPEAR configuration.
#
# One file for every operator choice: where a deployment's content lives, what
# the image tiers bake, and which entity and registry an image belongs to. It
# is the source of truth read by spear-chat, spear-corpus, spear-image
# (build and push) and the docker build scripts. It is not tracked.
#
# Copy it to local.conf and uncomment what applies: a line left commented is a
# choice not made, and the harness falls back to its default. `spear-configure`
# writes the top block for you from what the machine has, and keeps everything
# below the local marker untouched -- so the two ways of filling this file do
# not fight.

# ==================== Deployment: where content lives ====================

# Where sessions accumulate: history, memories, audit, the standard store.
export SPEAR_STATE_DIR="$HOME/.local/state/spear"

# The deployment's own content, from a private tree beside the checkout.
export SPEAR_RULES_DIR=/path/to/spear-private/rules.d
export SPEAR_SKILLS_DIR=/path/to/spear-private/skills
export SPEAR_BENCH_DIR=/path/to/spear-private/benches
[ -r /path/to/spear-private/engagement.env ] && . /path/to/spear-private/engagement.env

# Document embedding for normative standards, on the GPU host. The revision is
# what both ends' Hub caches resolve `main` to; a licensed standard never
# leaves this machine unless an ingest/rebuild passes --allow-offload.
#export SPEAR_STANDARD_EMBED_MODEL=BAAI/bge-m3
#export SPEAR_STANDARD_EMBED_REVISION=<40-hex revision>
#export SPEAR_STANDARD_EMBED_REMOTE=user@gpu-host

# Retrieval mode and structural completion are set per document
# (/standard retrieval), not here: exported, they would override every
# document for every session.

# --- local additions: kept as they are by spear-configure ---

# ===================== Image tiers: spear-image build =====================
# These replace the build-tiered.sh flags. A flag given on the command line
# still overrides the value set here, so an occasional one-off needs no edit.

# The entity an image belongs to; it decides the registry a private image may
# be pushed to (see Registries below). "local" (the default) means none.
#export SPEAR_ENTITY=acme

# base -- the common corpora baked into a company's commons image, the shared
# build system included (was --commons).
#export SPEAR_COMMONS=buildsystem,commons-docs

# base -- the knowledge-seed workspaces to bake, or "all" (was --knowledge).
#export SPEAR_KNOWLEDGE=all

# project -- this project's own spear-private content trees, colon separated
# (was --tree), and the base it builds on (was --from; empty => spear:<ver>-base).
#export SPEAR_PROJECT_TREES=/path/to/project/spear-private
#export SPEAR_FROM_BASE=

# Corpora to bake writable into the image rather than mount at run time (was
# --bake). Empty => the trees stay mounted and live.
#export SPEAR_BAKE=

# A store holding a standard this image is entitled to bake (was --standards),
# and whether it carries that licensed document (was --licensed). A licensed
# standard -- or the build system in a base -- makes the image
# non-redistributable.
#export SPEAR_STANDARDS_STORE=
#export SPEAR_LICENSED=0

# Further content trees a base merges after this machine's own, colon separated
# (a project checkout carrying its own rules.d/, skills/, benches/).
#export SPEAR_CONTENT_TREES=/path/to/project/spear-private

# The standard bodies whose identifiers the harness recognises beyond the
# built-in ones, comma or space separated (e.g. a body only this image serves).
#export SPEAR_STANDARD_BODIES=

# ========================== Registries: push =============================
# Where a private image of each entity may be pushed. An image goes only to its
# own entity's registry; anywhere else needs push --allow-push. Some entities
# have a built-in default (see scripts/docker/push.sh); any other is named
# here, one variable per entity:
#
# export SPEAR_<ENTITY>_REGISTRY=<registry-host>
#export SPEAR_ACME_REGISTRY=registry.example.com
29 changes: 0 additions & 29 deletions client/machine.env.example

This file was deleted.

13 changes: 9 additions & 4 deletions client/spear-chat.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,15 @@ PY="$SCRIPT_DIR/bin/python3"
export VIRTUAL_ENV="$SCRIPT_DIR"
export PATH="$SCRIPT_DIR/bin:$PATH"

# Settings specific to THIS machine (untracked). Used to point at an inference
# endpoint that is already up: on a shared host the --local :8080 may well be
# taken by something else entirely.
[ -r "$SCRIPT_DIR/machine.env" ] && . "$SCRIPT_DIR/machine.env"
# Settings specific to THIS machine (untracked): local.conf, or the legacy
# machine.env if no local.conf has been written yet. Used to point at an
# inference endpoint that is already up: on a shared host the --local :8080 may
# well be taken by something else entirely.
if [ -r "$SCRIPT_DIR/local.conf" ]; then
. "$SCRIPT_DIR/local.conf"
elif [ -r "$SCRIPT_DIR/machine.env" ]; then
. "$SCRIPT_DIR/machine.env"
fi

# ── move under the user manager when needed ──
# The sandbox's resource control needs delegated cgroup controllers, and
Expand Down
6 changes: 5 additions & 1 deletion client/spear-corpus.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,11 @@ PY="$SCRIPT_DIR/bin/python3"
# where a session's state lives -- and `spear-corpus` silently created a second
# state tree beside the code. SPEAR_CORPUS_ROOT, which decides what a relative
# corpus path resolves against, was missed the same way.
[ -r "$SCRIPT_DIR/machine.env" ] && . "$SCRIPT_DIR/machine.env"
if [ -r "$SCRIPT_DIR/local.conf" ]; then
. "$SCRIPT_DIR/local.conf"
elif [ -r "$SCRIPT_DIR/machine.env" ]; then
. "$SCRIPT_DIR/machine.env"
fi

exec "$PY" -c 'import sys
sys.path.insert(0, sys.argv[1])
Expand Down
49 changes: 37 additions & 12 deletions client/tests/test_launcher_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

`spear-corpus` and the in-chat `/corpus` are one implementation on purpose --
a second one had already drifted. But the two entry points did not run in the
same environment: `spear-chat.sh` sources the machine's untracked `machine.env`
and `spear-corpus.sh` did not, so the CLI ran with `SPEAR_STATE_DIR` defaulting
same environment: `spear-chat.sh` sources the machine's untracked `local.conf`
(or legacy `machine.env`) and `spear-corpus.sh` did not, so the CLI ran with
`SPEAR_STATE_DIR` defaulting
to the app directory while the chat used the configured one. Importing the
module creates that directory, so the CLI quietly grew a second state tree
beside the code; and `SPEAR_CORPUS_ROOT`, which decides what a *relative*
Expand Down Expand Up @@ -40,7 +41,7 @@
class Deployment:
"""A throwaway app directory holding one launcher and a stub interpreter."""

def __init__(self, tmp, launcher, machine_env=None):
def __init__(self, tmp, launcher, machine_env=None, local_conf=None):
self.dir = Path(tmp)
(self.dir / "bin").mkdir(parents=True, exist_ok=True)
shutil.copy(APP / launcher, self.dir / launcher)
Expand All @@ -50,6 +51,8 @@ def __init__(self, tmp, launcher, machine_env=None):

if machine_env is not None:
(self.dir / "machine.env").write_text(machine_env)
if local_conf is not None:
(self.dir / "local.conf").write_text(local_conf)

self.launcher = self.dir / launcher

Expand Down Expand Up @@ -113,11 +116,27 @@ class TheLaunchersAgree(unittest.TestCase):
def test_every_launcher_reads_the_machine_environment(self):
for name in LAUNCHERS:
with self.subTest(launcher=name):
text = (APP / name).read_text()
self.assertIn(
'[ -r "$SCRIPT_DIR/machine.env" ] && . "$SCRIPT_DIR/machine.env"',
(APP / name).read_text(),
f"{name} does not read machine.env; the entry points would "
'. "$SCRIPT_DIR/local.conf"', text,
f"{name} does not read local.conf; the entry points would "
f"then disagree about where state lives")
self.assertIn(
'. "$SCRIPT_DIR/machine.env"', text,
f"{name} dropped the legacy machine.env fallback")

def test_local_conf_is_read_and_wins_over_machine_env(self):
import tempfile

with tempfile.TemporaryDirectory() as tmp:
app = Deployment(
tmp, "spear-corpus.sh",
machine_env='export SPEAR_STATE_DIR="/var/tmp/legacy"\n',
local_conf='export SPEAR_STATE_DIR="/var/tmp/current"\n')
done, reported = app.run("list")

self.assertEqual(done.returncode, 0, done.stderr)
self.assertEqual(reported["SPEAR_STATE_DIR"], "/var/tmp/current")

def test_the_chat_and_the_corpus_cli_report_the_same_environment(self):
import tempfile
Expand Down Expand Up @@ -155,17 +174,23 @@ def test_the_chat_and_the_corpus_cli_report_the_same_environment(self):
class ThePublicCheckoutIsSelfContained(unittest.TestCase):
"""Sourcing a machine file must not make the repository need one."""

MACHINE_FILES = ("client/local.conf", "client/machine.env")

def test_no_launcher_is_tracked_with_a_machine_file(self):
tracked = subprocess.run(["git", "ls-files"], cwd=ROOT,
capture_output=True, text=True, check=True)
self.assertNotIn("client/machine.env", tracked.stdout.split())
for path in self.MACHINE_FILES:
with self.subTest(path=path):
self.assertNotIn(path, tracked.stdout.split())

def test_the_machine_file_is_ignored_rather_than_merely_absent(self):
done = subprocess.run(["git", "check-ignore", "-q",
"client/machine.env"], cwd=ROOT)
self.assertEqual(done.returncode, 0,
"client/machine.env is not gitignored, so a local "
"deployment's paths could be committed by accident")
for path in self.MACHINE_FILES:
with self.subTest(path=path):
done = subprocess.run(["git", "check-ignore", "-q", path],
cwd=ROOT)
self.assertEqual(done.returncode, 0,
f"{path} is not gitignored, so a local "
"deployment's paths could be committed by accident")

# Assembled from fragments, like the public-boundary scanner's own needles:
# a test that spells a forbidden path out loud puts it in the tree it is
Expand Down
48 changes: 34 additions & 14 deletions scripts/docker/build-tiered.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,16 @@
# Build the tiered images: harness -> base -> project.
#
# build-tiered.sh harness
# build-tiered.sh base --commons a,b,c [--entity NAME] [--licensed]
# build-tiered.sh base [--commons a,b,c] [--entity NAME] [--licensed]
# [--knowledge p,q|all]
# build-tiered.sh project --from spear:<ver>-base --tree DIR
# build-tiered.sh project [--from spear:<ver>-base] [--tree DIR]
# [--bake a,b] [--standards DIR] [--licensed] [--tag TAG]
#
# Every option has an equivalent in client/local.conf (SPEAR_COMMONS,
# SPEAR_ENTITY, SPEAR_BAKE, SPEAR_PROJECT_TREES, ...), read before the flags; a
# flag overrides the configured value. So on a machine whose local.conf names
# its commons and entity, `build-tiered.sh base` alone is enough.
#
# A project adds its own standards too: --licensed takes the licensed documents
# of --standards DIR (a store holding, say, only one private standard), appended over the
# base's. So an edgemtech project image = the reds base + a private standard.
Expand Down Expand Up @@ -38,10 +43,34 @@ case "$TIER" in
*) die "tier must be harness, base or project, not '$TIER'" ;;
esac

COMMONS="" BAKE="" KNOWLEDGE="" ENTITY="local" FROM_BASE="" TAG=""
STANDARDS_STORE=""
LICENSED=0
# The machine's configuration, exactly as build.sh and the launchers read it:
# the content locations a base bakes, and the tier choices that used to be
# flags (SPEAR_COMMONS, SPEAR_BAKE, SPEAR_ENTITY, ...). local.conf, or the
# legacy machine.env. Without it a base ships no rules and no skills.

if [ -r "$APP/local.conf" ]; then
echo " settings $APP/local.conf"
# shellcheck disable=SC1091
. "$APP/local.conf"
elif [ -r "$APP/machine.env" ]; then
echo " settings $APP/machine.env"
# shellcheck disable=SC1091
. "$APP/machine.env"
fi

# The choices come from the configuration above; a flag below overrides it, and
# --tree adds to any SPEAR_PROJECT_TREES already set.

COMMONS="${SPEAR_COMMONS:-}"
BAKE="${SPEAR_BAKE:-}"
KNOWLEDGE="${SPEAR_KNOWLEDGE:-}"
ENTITY="${SPEAR_ENTITY:-local}"
FROM_BASE="${SPEAR_FROM_BASE:-}"
TAG="${SPEAR_TAG:-}"
STANDARDS_STORE="${SPEAR_STANDARDS_STORE:-}"
LICENSED="${SPEAR_LICENSED:-0}"
TREES=()
[ -n "${SPEAR_PROJECT_TREES:-}" ] && IFS=: read -r -a TREES <<<"$SPEAR_PROJECT_TREES"

while [ $# -gt 0 ]; do
case "$1" in
Expand All @@ -60,15 +89,6 @@ done

[ -f "$APP/cli/rag_chat.py" ] || die "no harness under $APP — set SPEAR_APP"

# The machine's own content locations and content trees, exactly as build.sh and
# the launchers read them. Without it a base ships no rules and no skills.

if [ -r "$APP/machine.env" ]; then
echo " settings $APP/machine.env"
# shellcheck disable=SC1091
. "$APP/machine.env"
fi

EMPTY="$(mktemp -d)"
STAGE="$(mktemp -d)"
trap 'rm -rf "$EMPTY" "$STAGE"' EXIT
Expand Down
10 changes: 7 additions & 3 deletions scripts/docker/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,13 @@ TAG="${TAG:-spear:$("$REPO/scripts/spearversion.sh")-$PROFILE}"
# checkout. Without it the OPTIONAL table below resolves to the in-tree
# directories -- which hold a README and nothing else -- and the build reports
# them as found. The image then ships a harness with no rules and no skills,
# announcing neither, which is exactly what machine.env exists to prevent on
# the workstation.
if [ -r "$APP/machine.env" ]; then
# announcing neither, which is exactly what the configuration exists to prevent
# on the workstation. local.conf, or the legacy machine.env.
if [ -r "$APP/local.conf" ]; then
echo " settings $APP/local.conf"
# shellcheck disable=SC1091
. "$APP/local.conf"
elif [ -r "$APP/machine.env" ]; then
echo " settings $APP/machine.env"
# shellcheck disable=SC1091
. "$APP/machine.env"
Expand Down
Loading
Loading