From 85ceba36071447706f9b4d62528d7e743ef929f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 23:43:22 +0000 Subject: [PATCH 01/30] Charts: draw superscripts and other glyphs the built-in font lacks as plain stand-ins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pillow's default font has no ² ³ − → € etc., so they drew as boxes. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/charts/raster.py | 20 +++++++++++++++++--- tests/test_charts.py | 15 +++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/features/charts/raster.py b/lib/features/charts/raster.py index b9e55cf..65e0019 100644 --- a/lib/features/charts/raster.py +++ b/lib/features/charts/raster.py @@ -20,6 +20,18 @@ def font(px: int) -> ImageFont.FreeTypeFont: return ImageFont.load_default(size=max(1, px)) +# Pillow's built-in font draws a box for these; plain stand-ins keep units like km² and symbols readable. +_FALLBACK = {ord(k): v for k, v in { + "⁰": "0", "¹": "1", "²": "2", "³": "3", "⁴": "4", "⁵": "5", "⁶": "6", "⁷": "7", "⁸": "8", "⁹": "9", + "−": "-", "–": "-", "—": "-", "→": "->", "×": "x", "≈": "~", "•": "-", "µ": "u", "½": "1/2", + "≥": ">=", "≤": "<=", "‰": "per mille", "€": "EUR ", "£": "GBP ", "¥": "JPY ", +}.items()} + + +def _drawable(text: str) -> str: + return text.translate(_FALLBACK) + + def _stroke(px: float, bold: bool) -> int: """Pillow's font has no bold: an outline in the text's own colour stands in for it.""" return max(1, round(px / 40)) if bold else 0 @@ -28,21 +40,23 @@ def _stroke(px: float, bold: bool) -> int: def measure(text: str, px: float, bold: bool = False) -> tuple[float, float]: """Width and height of a line of text at `px` pixels, in the picture's pixels.""" size = round(px * SCALE) + text = _drawable(text) left, top, right, bottom = font(size).getbbox(text or " ", stroke_width=_stroke(size, bold)) return (right - left) / SCALE, (bottom - top) / SCALE def _text(img: Image.Image, draw: ImageDraw.ImageDraw, op: Text) -> None: size = round(op.size * SCALE) + text = _drawable(op.text) kw = {"font": font(size), "fill": op.color, "stroke_width": _stroke(size, op.bold), "stroke_fill": op.color} x, y = op.x * SCALE, op.y * SCALE if not op.angle: - draw.text((x, y), op.text, anchor=op.anchor, **kw) + draw.text((x, y), text, anchor=op.anchor, **kw) return # Rotated: drawn on its own transparent tile, turned, then placed by the anchor's corner of the turned tile. - left, top, right, bottom = font(size).getbbox(op.text, stroke_width=kw["stroke_width"]) + left, top, right, bottom = font(size).getbbox(text, stroke_width=kw["stroke_width"]) tile = Image.new("RGBA", (right - left + 4, bottom - top + 4), (0, 0, 0, 0)) - ImageDraw.Draw(tile).text((2 - left, 2 - top), op.text, **kw) + ImageDraw.Draw(tile).text((2 - left, 2 - top), text, **kw) tile = tile.rotate(op.angle, expand=True, resample=Image.BICUBIC) across, down = op.anchor[0], op.anchor[1] dx = {"l": 0, "m": tile.width / 2, "r": tile.width}[across] diff --git a/tests/test_charts.py b/tests/test_charts.py index 658022b..72999bf 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -920,3 +920,18 @@ def test_number_ticks_are_round_steps_inside_the_range(): def test_date_ticks_pick_a_unit_for_the_span_and_name_the_bigger_one_at_its_boundary(start, end, labels): from lib.features.charts.ticks import date_ticks assert [label for _, label in date_ticks(start, end)] == labels + + +def test_superscripts_are_drawn_as_plain_digits_not_a_missing_glyph_box(): + from lib.features.charts import raster + from lib.features.charts.scene import Scene, Text + + assert raster.measure("km²", 12) == raster.measure("km2", 12) + + def pixels(text: str) -> bytes: + return raster.png(Scene(bg="#ffffff", ops=[Text(role="title", x=10, y=10, text=text, size=14, color="#000000")])) + + assert pixels("52m km²") == pixels("52m km2") + chart = Chart("Area (km²)", [Series("Russia", ["Russia", "China"], [17.1, 9.6], kind="pie")]) + assert "Area (km²)" in render.scene(chart).texts("title") + assert render.png(chart) From 29560877a29edb5d2e073040807a445f215b16a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 23:44:11 +0000 Subject: [PATCH 02/30] Pie legend: whole-number values instead of percentages (the donut shows the share) Falls back to shares when a value would round to zero. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/charts/pie.py | 4 ++-- lib/features/charts/render.py | 11 ++++++++++- tests/test_charts.py | 8 ++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/features/charts/pie.py b/lib/features/charts/pie.py index 7c67c67..1fa9a06 100644 --- a/lib/features/charts/pie.py +++ b/lib/features/charts/pie.py @@ -1,4 +1,4 @@ -"""The layout of a pie chart: a donut, its total in the middle, and a legend of each slice's share beside it.""" +"""The layout of a pie chart: a donut, its total in the middle, and a legend of each slice's value beside it.""" from __future__ import annotations @@ -43,7 +43,7 @@ def layout(chart, th, slices: list[tuple[str, float]], share) -> Scene: def _legend(scene: Scene, th, slices, colors, whole: float, share) -> None: - """A square and "name share" per slice, the names wrapped, the whole block centred beside the donut.""" + """A square and "name value" per slice, the names wrapped, the whole block centred beside the donut.""" px = 11 * PT line_h = measure("Hg", px)[1] * 1.25 gap = 0.9 * px diff --git a/lib/features/charts/render.py b/lib/features/charts/render.py index 4308f7e..29b9487 100644 --- a/lib/features/charts/render.py +++ b/lib/features/charts/render.py @@ -78,6 +78,14 @@ def share(part: float, whole: float) -> str: return "<0.1%" if 0 < pct < 0.05 else f"{pct:.1f}%" +def legend_value(slices: list[tuple[str, float]]): + """What a pie's legend shows beside each name: the slice's own value as a whole number (the donut already shows the + share), or the share when a value would round to nothing.""" + if all(round(y) >= 1 for _, y in slices): + return lambda part, whole: f"{round(part):,}" + return share + + def _single_axis(chart: Chart) -> Chart: """A chart whose series are all on the right axis is drawn on the left one, with the right axis's settings.""" if not chart.series or any(s.axis == "left" for s in chart.series): @@ -91,7 +99,8 @@ def scene(chart: Chart) -> Scene: chart = _single_axis(chart) th = THEMES["dark" if chart.dark else "light"] if any(s.kind == "pie" for s in chart.series): - return pie.layout(chart, th, pie_slices(chart.series[0]), share) + slices = pie_slices(chart.series[0]) + return pie.layout(chart, th, slices, legend_value(slices)) return cartesian.layout(chart, th) diff --git a/tests/test_charts.py b/tests/test_charts.py index 72999bf..bd84234 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -622,14 +622,18 @@ def test_pie_slices_are_biggest_first_with_gaps_dropped_and_the_smallest_added_u assert [render.share(p, 100.0) for p in (62.43, 0.01, 100.0)] == ["62.4%", "<0.1%", "100.0%"] -def test_a_pie_is_a_donut_with_a_legend_of_shares_and_the_total_in_the_middle(): +def test_a_pie_is_a_donut_with_a_legend_of_whole_numbers_and_the_total_in_the_middle(): chart = Chart("Revenue", [Series("p", ["US", "Taiwan"], [75.0, 25.0], kind="pie", total="$100B")], subtitle="Share of total") scene = render.scene(chart) wedges = scene.find("wedge") assert len(wedges) == 2 and wedges[0].end - wedges[0].start == pytest.approx(270) # 75% of the way round assert scene.texts("total") == ["$100B"] and scene.find("hole") - assert scene.legend == ["US 75.0%", "Taiwan 25.0%"] + assert scene.legend == ["US 75", "Taiwan 25"] + big = Chart("Area", [Series("p", ["Russia", "China"], [16376870.4, 9388211.0], kind="pie")]) + assert render.scene(big).legend == ["Russia 16,376,870", "China 9,388,211"] + tiny = Chart("Mix", [Series("p", ["A", "B"], [0.75, 0.25], kind="pie")]) # would round to 1 and 0: shares instead + assert render.scene(tiny).legend == ["A 75.0%", "B 25.0%"] for dark, bg in ((False, (255, 255, 255)), (True, (0x14, 0x1a, 0x22))): picture = Image.open(io.BytesIO(REAL_PNG(replace(chart, dark=dark)))) assert picture.size == (1224, 714) and picture.getpixel((2, 2))[:3] == bg From 7cb0ac4a2a4751930fb416053401e832d98509df Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 23:46:54 +0000 Subject: [PATCH 03/30] Pie: unit in legend and total from the title's bracket; subtitle matches value legend Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/charts/__init__.py | 21 +++++++++------------ lib/features/charts/render.py | 28 +++++++++++++++++++++++----- tests/test_charts.py | 17 +++++++++++++---- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/lib/features/charts/__init__.py b/lib/features/charts/__init__.py index 26276ea..3ff206a 100644 --- a/lib/features/charts/__init__.py +++ b/lib/features/charts/__init__.py @@ -254,21 +254,18 @@ def unify_x(series: list[Series]) -> list[str]: return labels -UNIT_RE = re.compile(r"\(\s*(A\$|US\$|[$€£¥])?\s*(trillion|billion|million|thousand|[KMBT]n?|bn|mn)?\s*\)", re.IGNORECASE) - - def pie_total(s: Series, title: str) -> str: - """The sum of a pie's slices for its middle, in the unit the title gives in brackets ("($B)", "(€ million)"); - nothing if the title names none, as a bare number could be anything.""" - found = UNIT_RE.search(title) - if not found or not (found.group(1) or found.group(2)): + """The sum of a pie's slices for its middle, in the unit the title gives in brackets ("($B)", "(km²)"); nothing if + the title names none, as a bare number could be anything.""" + prefix, suffix = render.pie_unit(title) + if not prefix and not suffix: return "" whole = sum(y for _, y in render.pie_slices(s)) - short = {"trillion": "T", "billion": "B", "million": "M", "thousand": "K", "bn": "B", "mn": "M"} - unit = found.group(2) or "" - unit = short.get(unit.lower(), unit) + if suffix.startswith(" ") and whole >= 1e6: # a unit of its own: compact, so it fits in the hole + big, mark = (whole / 1e9, "bn") if whole >= 1e9 else (whole / 1e6, "m") + return f"{prefix}{big:,.2f}{mark}{suffix}" digits = f"{whole:,.0f}" if whole >= 1000 else f"{whole:,.1f}" - return f"{found.group(1) or ''}{digits}{unit}" + return f"{prefix}{digits}{suffix}" def total(s: Series, spec: dict) -> str: @@ -381,7 +378,7 @@ async def build(ctx: Ctx, args: dict) -> tuple[Chart, list[str]]: on_side = [spec for s, spec in zip(series, specs, strict=True) if s.axis == side] setattr(chart, f"{side}_percent", bool(on_side) and all(spec.get("percent_change") for spec in on_side)) chart.source = source_for(args, specs) - chart.subtitle = "Share of total" if pie else subtitle_for( + chart.subtitle = "Ring shows share of total" if pie else subtitle_for( specs, series, chart.left_percent and not any(s.axis == "right" for s in series)) return chart, [pie_summary(s) if pie else summary(s, specs[i].get("percent_change"), notes[i]) for i, s in enumerate(series)] diff --git a/lib/features/charts/render.py b/lib/features/charts/render.py index 29b9487..6e114b1 100644 --- a/lib/features/charts/render.py +++ b/lib/features/charts/render.py @@ -4,6 +4,7 @@ from __future__ import annotations import math +import re import threading from dataclasses import dataclass, field, replace @@ -78,11 +79,28 @@ def share(part: float, whole: float) -> str: return "<0.1%" if 0 < pct < 0.05 else f"{pct:.1f}%" -def legend_value(slices: list[tuple[str, float]]): - """What a pie's legend shows beside each name: the slice's own value as a whole number (the donut already shows the - share), or the share when a value would round to nothing.""" +UNIT_RE = re.compile(r"\(\s*(A\$|US\$|[$€£¥])?\s*(trillion|billion|million|thousand|[KMBT]n?|bn|mn)?\s*\)", re.IGNORECASE) +LAST_BRACKET_RE = re.compile(r"\(([^()%\d][^()%]{0,11})\)\s*$") + + +def pie_unit(title: str) -> tuple[str, str]: + """The unit a pie's title gives in brackets, as (before the number, after it): "($B)" -> ("$", "B"), "(km²)" -> + ("", " km²"); empty when it names none.""" + found = UNIT_RE.search(title) + if found and (found.group(1) or found.group(2)): + short = {"trillion": "T", "billion": "B", "million": "M", "thousand": "K", "bn": "B", "mn": "M"} + unit = found.group(2) or "" + return found.group(1) or "", short.get(unit.lower(), unit) + last = LAST_BRACKET_RE.search(title) + return ("", f" {last.group(1).strip()}") if last else ("", "") + + +def legend_value(slices: list[tuple[str, float]], title: str = ""): + """What a pie's legend shows beside each name: the slice's own value as a whole number with the title's unit (the + donut already shows the share), or the share when a value would round to nothing.""" if all(round(y) >= 1 for _, y in slices): - return lambda part, whole: f"{round(part):,}" + prefix, suffix = pie_unit(title) + return lambda part, whole: f"{prefix}{round(part):,}{suffix}" return share @@ -100,7 +118,7 @@ def scene(chart: Chart) -> Scene: th = THEMES["dark" if chart.dark else "light"] if any(s.kind == "pie" for s in chart.series): slices = pie_slices(chart.series[0]) - return pie.layout(chart, th, slices, legend_value(slices)) + return pie.layout(chart, th, slices, legend_value(slices, chart.title)) return cartesian.layout(chart, th) diff --git a/tests/test_charts.py b/tests/test_charts.py index bd84234..ed9114f 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -624,14 +624,14 @@ def test_pie_slices_are_biggest_first_with_gaps_dropped_and_the_smallest_added_u def test_a_pie_is_a_donut_with_a_legend_of_whole_numbers_and_the_total_in_the_middle(): chart = Chart("Revenue", [Series("p", ["US", "Taiwan"], [75.0, 25.0], kind="pie", total="$100B")], - subtitle="Share of total") + subtitle="Ring shows share of total") scene = render.scene(chart) wedges = scene.find("wedge") assert len(wedges) == 2 and wedges[0].end - wedges[0].start == pytest.approx(270) # 75% of the way round assert scene.texts("total") == ["$100B"] and scene.find("hole") assert scene.legend == ["US 75", "Taiwan 25"] - big = Chart("Area", [Series("p", ["Russia", "China"], [16376870.4, 9388211.0], kind="pie")]) - assert render.scene(big).legend == ["Russia 16,376,870", "China 9,388,211"] + big = Chart("Area (km²)", [Series("p", ["Russia", "China"], [16376870.4, 9388211.0], kind="pie")]) + assert render.scene(big).legend == ["Russia 16,376,870 km²", "China 9,388,211 km²"] tiny = Chart("Mix", [Series("p", ["A", "B"], [0.75, 0.25], kind="pie")]) # would round to 1 and 0: shares instead assert render.scene(tiny).legend == ["A 75.0%", "B 25.0%"] for dark, bg in ((False, (255, 255, 255)), (True, (0x14, 0x1a, 0x22))): @@ -644,7 +644,7 @@ async def test_a_pie_call_takes_slices_names_and_shares_for_the_model(env, store out = await make_chart({"title": "t", "series": [pie("Revenue", ["Compute", "Graphics"], [88299, 7922])]}) assert out.startswith("Chart 1 is ready") and "Revenue (pie, 2 slices, total 96,221): Compute 91.8%, Graphics 8.2%" in out chart, _ = await charts.build(make(env, store), {"title": "t", "series": [pie("R", [2024, 2025], [1, 3])]}) - assert chart.series[0].x == ["2024", "2025"] and chart.subtitle == "Share of total" and not chart.labels + assert chart.series[0].x == ["2024", "2025"] and chart.subtitle == "Ring shows share of total" and not chart.labels @pytest.mark.parametrize("series, error", [ @@ -939,3 +939,12 @@ def pixels(text: str) -> bytes: chart = Chart("Area (km²)", [Series("Russia", ["Russia", "China"], [17.1, 9.6], kind="pie")]) assert "Area (km²)" in render.scene(chart).texts("title") assert render.png(chart) + + +def test_a_pie_total_and_legend_carry_the_unit_from_the_title(): + from lib.features import charts as pkg + s = Series("p", ["Russia", "China"], [17098246.0, 9596961.0], kind="pie") + assert render.pie_unit("Land area (km²)") == ("", " km²") and render.pie_unit("Revenue ($B)") == ("$", "B") + assert render.pie_unit("Land area") == ("", "") and render.pie_unit("Share (%)") == ("", "") + assert pkg.pie_total(s, "Land area (km²)") == "26.70m km²" and pkg.pie_total(s, "Land area") == "" + assert pkg.pie_total(Series("p", ["a", "b"], [60.0, 31.8], kind="pie"), "Revenue ($B)") == "$91.8B" From b6280b8213e045a1fff8cb44e902f91342bf8f08 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 23:47:34 +0000 Subject: [PATCH 04/30] Pie: larger donut, donut and legend centred together Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/features/charts/pie.py | 33 ++++++++++++++++++++------------- tests/test_charts.py | 10 ++++++++++ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/features/charts/pie.py b/lib/features/charts/pie.py index 1fa9a06..0de5841 100644 --- a/lib/features/charts/pie.py +++ b/lib/features/charts/pie.py @@ -7,8 +7,10 @@ from .raster import measure, point_on from .scene import PT, Dot, H, Line, Rect, Scene, Text, W, Wedge -CENTRE = (0.27 * W, 0.58 * H) -RADIUS = 0.30 * H +CENTRE_Y = 0.575 * H # between the subtitle and the source line +MAX_RADIUS = 0.35 * H # what fits above and below +SIDE_MARGIN = 0.04 * W +DONUT_GAP = 0.07 * W # between the donut and its legend HOLE = 0.58 # the hole's radius as a share of the donut's GAP = 2.5 * PT # the background-coloured line between two slices WRAP = 26 # characters of a slice's name per legend line @@ -18,21 +20,29 @@ def layout(chart, th, slices: list[tuple[str, float]], share) -> Scene: scene = Scene(bg=th.bg) whole = sum(y for _, y in slices) or 1.0 colors = [th.text if name == "Other" else th.palette[i % len(th.palette)] for i, (name, _) in enumerate(slices)] - cx, cy = CENTRE + px = 11 * PT + line_h = measure("Hg", px)[1] * 1.25 + entries = [(textwrap.fill(name, WRAP) + f" {share(y, whole)}").split("\n") for name, y in slices] + side = 0.75 * px + legend_w = side + px * 0.8 + max(measure(line, px)[0] for lines in entries for line in lines) + # As large as the height and the legend allow, the donut and legend together centred across the picture. + radius = max(0.1 * H, min(MAX_RADIUS, (W - 2 * SIDE_MARGIN - DONUT_GAP - legend_w) / 2)) + left = (W - (2 * radius + DONUT_GAP + legend_w)) / 2 + cx, cy = left + radius, CENTRE_Y angle, bounds = 0.0, [] for (_, y), color in zip(slices, colors, strict=True): sweep = 360 * y / whole - scene.add(Wedge(role="wedge", cx=cx, cy=cy, r=RADIUS, start=angle, end=angle + sweep, color=color)) + scene.add(Wedge(role="wedge", cx=cx, cy=cy, r=radius, start=angle, end=angle + sweep, color=color)) bounds.append(angle) angle += sweep if len(slices) > 1: for a in bounds: - scene.add(Line(role="gap", points=[(cx, cy), point_on(cx, cy, RADIUS + 2, a)], color=th.bg, width=GAP)) - scene.add(Dot(role="hole", x=cx, y=cy, r=RADIUS * HOLE, color=th.bg)) + scene.add(Line(role="gap", points=[(cx, cy), point_on(cx, cy, radius + 2, a)], color=th.bg, width=GAP)) + scene.add(Dot(role="hole", x=cx, y=cy, r=radius * HOLE, color=th.bg)) total = chart.series[0].total if total: scene.add(Text(role="total", x=cx, y=cy, text=total, size=17 * PT, color=th.ink, bold=True, anchor="mm")) - _legend(scene, th, slices, colors, whole, share) + _legend(scene, th, entries, colors, cx + radius + DONUT_GAP, cy, (px, line_h, side)) scene.add(Text(role="title", x=0.03 * W, y=0.05 * H, text=chart.title, size=14 * PT, color=th.ink, bold=True)) if chart.subtitle: scene.add(Text(role="subtitle", x=0.03 * W, y=0.125 * H, text=chart.subtitle, size=10 * PT, color=th.text)) @@ -42,15 +52,12 @@ def layout(chart, th, slices: list[tuple[str, float]], share) -> Scene: return scene -def _legend(scene: Scene, th, slices, colors, whole: float, share) -> None: +def _legend(scene: Scene, th, entries, colors, x: float, middle_y: float, sizes) -> None: """A square and "name value" per slice, the names wrapped, the whole block centred beside the donut.""" - px = 11 * PT - line_h = measure("Hg", px)[1] * 1.25 + px, line_h, side = sizes gap = 0.9 * px - entries = [(textwrap.fill(name, WRAP) + f" {share(y, whole)}").split("\n") for name, y in slices] height = sum(len(lines) * line_h for lines in entries) + gap * (len(entries) - 1) - x, y = 0.56 * W, 0.58 * H - height / 2 - side = 0.75 * px + y = middle_y - height / 2 for lines, color in zip(entries, colors, strict=True): block = len(lines) * line_h middle = y + block / 2 diff --git a/tests/test_charts.py b/tests/test_charts.py index ed9114f..11c89c6 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -948,3 +948,13 @@ def test_a_pie_total_and_legend_carry_the_unit_from_the_title(): assert render.pie_unit("Land area") == ("", "") and render.pie_unit("Share (%)") == ("", "") assert pkg.pie_total(s, "Land area (km²)") == "26.70m km²" and pkg.pie_total(s, "Land area") == "" assert pkg.pie_total(Series("p", ["a", "b"], [60.0, 31.8], kind="pie"), "Revenue ($B)") == "$91.8B" + + +def test_a_pie_is_big_and_centred_with_its_legend_inside_the_picture(): + names = ["United States of America", "China", "Brazil"] + scene = render.scene(Chart("Area (km²)", [Series("p", names, [9.8e6, 9.6e6, 8.5e6], kind="pie")])) + wedge, swatch = scene.find("wedge")[0], scene.find("swatch")[0] + legend = scene.find("legend") + left, right = wedge.cx - wedge.r, max(t.x + render.raster.measure(t.text, t.size)[0] for t in legend) + assert wedge.r > 0.3 * H and left > 0 and right < W + assert left == pytest.approx(W - right, abs=2) and swatch.box[0] > wedge.cx + wedge.r From c3b375b1952a1dcf6840736efdfa485eb5a03f9d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 01:16:33 +0000 Subject: [PATCH 05/30] OpenRouter: end a chat answer with its web-search sources when it links none The url_citation annotations were only counted; now they are kept on the Step and Answer and ask() appends a Sources line (max 3) to chat answers that carry no link of their own. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/how-it-works.md | 2 ++ lib/ask.py | 3 +++ lib/config.py | 1 + lib/features/charts/render.py | 7 ++++++- lib/llm/base.py | 2 ++ lib/llm/chat.py | 3 ++- lib/llm/openrouter.py | 18 ++++++++++++++++++ lib/llm/runner.py | 9 ++++++++- lib/textfmt.py | 17 ++++++++++++++++- tests/test_handlers.py | 9 +++++++++ tests/test_llm_backends.py | 26 ++++++++++++++++++++++++++ tests/test_runner.py | 14 ++++++++++++++ tests/test_textfmt.py | 11 +++++++++++ 13 files changed, 118 insertions(+), 4 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 5d50ab3..ed50358 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -43,6 +43,8 @@ - **OpenRouter:** OpenRouter's `web_search` tool; when it hands a search call back, the bot runs the query through `SEARCH_MODEL` (default `xiaomi/mimo-v2.6-flash:online`) and returns the write-up; that call's cost is added to the answer's cost. Some models print a search call as text instead of making it; the bot recovers those queries too. + The pages the search cited come back as `url_citation` annotations: a chat answer that links nothing itself ends with a + `Sources:` line of up to 3 of them (the last round that cited any). - At most 5 searches per round; extras get a "skipped" note. - `SEARCH=off` turns search off (the prompt then says the model has no web search). - **Forced search:** movers lists, DM preset buttons and the holding-news digest *require* a diff --git a/lib/ask.py b/lib/ask.py index e8c9450..cf47389 100644 --- a/lib/ask.py +++ b/lib/ask.py @@ -12,6 +12,7 @@ from .llm.base import Answer, Request from .llm.gate import NEEDS_TOOLS, Route, wants_tools from .prompts import no_tools_system, system_prompt +from .textfmt import with_sources log = logging.getLogger("bot") @@ -83,6 +84,8 @@ async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = on_text=on_text) answer = await policy.ask(ctx.backend, ctx.registry, req) await _record(ctx, req.model, kind, chat_id, answer) + if kind == "chat": # other kinds (holding news, filings) parse or format their own text + answer.text = with_sources(answer.text, answer.sources) return answer diff --git a/lib/config.py b/lib/config.py index aa34ac7..11eee0c 100644 --- a/lib/config.py +++ b/lib/config.py @@ -57,6 +57,7 @@ SEARCH_RESULTS = 20 SEARCH_TOKENS = 400 # cap on each search write-up (OpenRouter search runner) MAX_SEARCHES = 5 # search calls run per round +SOURCES_MAX = 3 # links in the "Sources:" line added to an answer that cites none MAX_TG_MESSAGE = 4000 # Telegram's limit is 4096 HOLDING_NEWS_RETRY_EVERY = 30 * 60 # seconds between retries of a holding-news check the provider was too busy for HOLDING_NEWS_RETRY_FOR = 12 * 3600 # seconds after which it gives up diff --git a/lib/features/charts/render.py b/lib/features/charts/render.py index 6e114b1..5eec3e4 100644 --- a/lib/features/charts/render.py +++ b/lib/features/charts/render.py @@ -9,7 +9,12 @@ from dataclasses import dataclass, field, replace from . import cartesian, pie, raster -from .cartesian import LEGEND_COLUMNS, RULE_WIDTH, legend_columns, spread # noqa: F401 (part of this module's API) +from .cartesian import ( # noqa: F401 (part of this module's API) + LEGEND_COLUMNS, + RULE_WIDTH, + legend_columns, + spread, +) from .scene import Scene from .ticks import tick_label # noqa: F401 diff --git a/lib/llm/base.py b/lib/llm/base.py index 91a3754..1316d9d 100644 --- a/lib/llm/base.py +++ b/lib/llm/base.py @@ -58,6 +58,7 @@ class Step: finish: str | None = None searches: int = 0 # server-side searches the provider ran for this step raw: Any = None # provider payload add_results() needs to replay this step + sources: list[tuple[str, str]] = field(default_factory=list) # (title, url) the search cited @dataclass @@ -86,6 +87,7 @@ class Answer: usage: Usage rounds: int tool_calls: int + sources: list[tuple[str, str]] = field(default_factory=list) # (title, url) the search cited class Backend: diff --git a/lib/llm/chat.py b/lib/llm/chat.py index 3dd6b4d..6bed859 100644 --- a/lib/llm/chat.py +++ b/lib/llm/chat.py @@ -96,7 +96,8 @@ async def step(self, conv: dict, req: Request, *, tool_choice: str | None) -> St tokens_out=getattr(u, "completion_tokens", 0) or 0, cost=self._cost(u, req.model), ) - return Step(text.strip(), calls, usage, finish, extra.get("searches", 0), raw=assistant) + return Step(text.strip(), calls, usage, finish, extra.get("searches", 0), raw=assistant, + sources=extra.get("sources", [])) def add_results(self, conv: dict, step: Step, results: list[str]) -> None: conv["messages"].append(step.raw) diff --git a/lib/llm/openrouter.py b/lib/llm/openrouter.py index 4a0750a..4b06b41 100644 --- a/lib/llm/openrouter.py +++ b/lib/llm/openrouter.py @@ -22,6 +22,19 @@ ) +def citations(annotations: list) -> list[tuple[str, str]]: + """(title, url) of each web page a message cites, in order, http(s) only, one per URL.""" + out: dict[str, str] = {} + for a in annotations or []: + if not isinstance(a, dict) or a.get("type") != "url_citation": + continue + c = a.get("url_citation") or {} + url = str(c.get("url") or "") + if url.startswith(("http://", "https://")): + out.setdefault(url, str(c.get("title") or "").strip()) + return [(title, url) for url, title in out.items()] + + def merge_reasoning(details: list[dict], new: list) -> None: """Append streamed reasoning_details, joining consecutive text fragments of one block.""" for d in new: @@ -76,6 +89,8 @@ def _read_delta(self, delta, extra: dict) -> None: fields = getattr(delta, "model_extra", None) or {} merge_reasoning(extra.setdefault("details", []), fields.get("reasoning_details") or []) extra["searches"] = extra.get("searches", 0) + len(fields.get("annotations") or []) + have = extra.setdefault("sources", []) + have += [c for c in citations(fields.get("annotations") or []) if c[1] not in {u for _, u in have}] def _assistant_fields(self, extra: dict) -> dict: return {"reasoning_details": extra["details"]} if extra.get("details") else {} @@ -98,6 +113,9 @@ async def run_search(self, query: str, usage: Usage, args: dict) -> str | None: log.warning("Search model returned no text for %r (finish_reason=%s)", query, resp.choices[0].finish_reason) return "The search returned no text (the search model may have run out of tokens)." + links = citations(getattr(resp.choices[0].message, "annotations", None) or []) + if links: # the model only sees this text: hand it the URLs so it can cite them + text += "\nLinks: " + " ".join(url for _, url in links[:config.SOURCES_MAX]) return text async def credits(self) -> str | None: diff --git a/lib/llm/runner.py b/lib/llm/runner.py index f26c856..5c8f86b 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -177,12 +177,18 @@ async def _recover_fake_calls(step: Step, backend: Backend, usage: Usage) -> str return "\n\n".join(parts) + "\n\nAnswer the question now using these results." +def _cited(have: list[tuple[str, str]], new: list[tuple[str, str]]) -> list[tuple[str, str]]: + """The sources of the latest step that cited any (the ones behind the final answer), else what was gathered.""" + return new[:] if new else have + + async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: bool = True) -> Answer: t0 = time.monotonic() conv = backend.start(req) total, rounds, tool_calls = Usage(), 0, 0 seen: dict[tuple[str, str], int] = {} searched = 0 + cited: list[tuple[str, str]] = [] while True: final = rounds >= config.MAX_TOOL_ROUNDS choice = "none" if final else ("required" if (req.must_search or req.must_call) and rounds == 0 else None) @@ -190,6 +196,7 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo rounds += 1 total.add(step.usage) searched += step.searches + cited = _cited(cited, step.sources) u = step.usage log.debug("Round %d: in %d (%d cached) out %d, %s", rounds, u.tokens_in, u.cached, u.tokens_out, step.finish) @@ -228,4 +235,4 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo if step.finish == "length": raise RuntimeError("hit the token cap before writing an answer") raise RuntimeError(f"empty reply (finish_reason={step.finish})") - return Answer(text, total, rounds, tool_calls) + return Answer(text, total, rounds, tool_calls, cited) diff --git a/lib/textfmt.py b/lib/textfmt.py index fa1cffa..ca79df2 100644 --- a/lib/textfmt.py +++ b/lib/textfmt.py @@ -5,7 +5,7 @@ import html import re -from .config import MAX_TG_MESSAGE +from .config import MAX_TG_MESSAGE, SOURCES_MAX # [label](url), where label may itself be bracketed, as in [[1]](url) citations MD_LINK = re.compile(r"\[(\[[^\]]*\]|[^\[\]]+)\]\((https?://[^\s)]+)\)") @@ -51,6 +51,21 @@ def link(m: re.Match) -> str: return MD_CODE.sub(r"\1", text) +def with_sources(text: str, sources: list[tuple[str, str]], limit: int = SOURCES_MAX) -> str: + """An answer with a "Sources:" line of links when it has none of its own.""" + if not sources or " str: + host = re.sub(r"^www\.", "", url.split("/")[2]) if url.count("/") >= 2 else url + name = title if title else host + return html.escape(name if len(name) <= 40 else name[:39].rstrip() + "…") + + links = " · ".join(f'{label(title, url)}' + for title, url in sources[:limit]) + return f"{text}\n\nSources: {links}" + + def plain_text(text: str) -> str: """Telegram HTML as plain text: links become "label (url)", other tags are dropped.""" return html.unescape(ANY_TAG_RE.sub("", A_TAG_RE.sub(r"\2 (\1)", text))) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index a7ad1c1..fab960a 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -25,6 +25,15 @@ async def run(h, msg, edited=False): await asyncio.sleep(0) +async def test_a_chat_answer_that_used_web_search_ends_with_its_sources(env, store): + cited = step("Micron guided higher.") + cited.sources = [("Reuters", "https://www.reuters.com/mu")] + ctx, backend, h = make_ctx(env, store, [cited]) + m = user_msg(ctx.bot, "@stockbot what did Micron say?") + await run(h, m) + assert m.replies[0]["text"].endswith('Sources: Reuters') + + async def test_group_mention_is_answered_with_linked_tickers_and_logged(env, store): ctx, backend, h = make_ctx(env, store, [step("NVDA looks strong, SQX.AX too")]) m = user_msg(ctx.bot, "@stockbot how is NVDA?") diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 7582d42..1477c7d 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -244,3 +244,29 @@ async def test_xai_failed_stream_raises(): r = req() with pytest.raises(RuntimeError, match="failed"): await b.step(b.start(r), r, tool_choice=None) + + +def cite(url, title=""): + return {"type": "url_citation", "url_citation": {"url": url, "title": title}} + + +async def test_openrouter_collects_web_search_citations(env): + chunks = [chat_chunk("a", extra={"annotations": [cite("https://a.example/x", "A"), cite("ftp://no", "N")]}), + chat_chunk("b", extra={"annotations": [cite("https://a.example/x", "A"), cite("https://b.example/y")]}), + chat_chunk(finish="stop")] + b = OpenRouterBackend(config.load(env), FakeChatClient(Stream(chunks))) + r = req(search=True) + step = await b.step(b.start(r), r, tool_choice=None) + assert step.sources == [("A", "https://a.example/x"), ("", "https://b.example/y")] + + +async def test_openrouter_run_search_hands_the_model_the_links(env): + from types import SimpleNamespace as N + msg = N(content="- NVDA 100 [Reuters]", annotations=[cite("https://r.example/n", "R")]) + reply = N(choices=[N(message=msg, finish_reason="stop")], usage=N(cost=0.001)) + + async def create(**kw): + return reply + + b = OpenRouterBackend(config.load(env), N(chat=N(completions=N(create=create)))) + assert await b.run_search("nvda", Usage(), {}) == "- NVDA 100 [Reuters]\nLinks: https://r.example/n" diff --git a/tests/test_runner.py b/tests/test_runner.py index 4379b5c..2b6d6ec 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -310,3 +310,17 @@ async def test_arguments_that_are_not_json_are_a_warning_and_a_message_for_the_m await run(b, registry(mcp), with_tools(mcp)) assert mcp.calls == [] and "weren't valid JSON" in b.results_log[0][0] assert [r.levelname for r in caplog.records] == ["WARNING"] and not caplog.records[0].exc_info + + +async def test_the_sources_of_the_last_step_that_cited_any_land_on_the_answer(): + m = FakeMcp() + first = step("", [("c1", "yahoo__q", "{}")]) + first.sources = [("Old", "https://old.example/")] + last = step("Answer.") + last.sources = [("New", "https://new.example/")] + answer = await run(ScriptedBackend([first, last]), registry(m), with_tools(m)) + assert answer.sources == [("New", "https://new.example/")] + first = step("", [("c1", "yahoo__q", "{}")]) + first.sources = [("Old", "https://old.example/")] + answer = await run(ScriptedBackend([first, step("Answer.")]), registry(m), with_tools(m)) + assert answer.sources == [("Old", "https://old.example/")] diff --git a/tests/test_textfmt.py b/tests/test_textfmt.py index e7d8ba4..c4f8d36 100644 --- a/tests/test_textfmt.py +++ b/tests/test_textfmt.py @@ -52,3 +52,14 @@ def test_split_html_edge_cases(): assert all(len(c) <= 4096 for c in cs) cs = t.split_html("" * 20 + "x " * 3000 + "" * 20) assert len(cs) > 1 and all(len(c) <= 4096 and _balanced(c) for c in cs) + + +def test_with_sources_adds_links_only_when_the_answer_has_none(): + src = [("Reuters: NVDA & co", "https://www.reuters.com/a?x=1&y=2"), ("", "https://www.ft.com/b/c"), + ("x" * 60, "https://c.example/"), ("D", "https://d.example/")] + out = t.with_sources("NVDA rose.", src) + assert out == ('NVDA rose.\n\nSources: Reuters: NVDA & co' + ' · ft.com · ' + "x" * 39 + "…") + assert t.with_sources('See a.', src) == 'See a.' + assert t.with_sources("See [a](https://a.b).", src) == "See [a](https://a.b)." + assert t.with_sources("Plain.", []) == "Plain." From 4307dc8c3c50b336156e3cbdcad91af39331d7c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 01:21:41 +0000 Subject: [PATCH 06/30] OpenRouter: cite sources as [n] links after the cited sentence Positions from the url_citation annotations are applied to the raw reply before any formatting; the Sources line stays as the fallback when no marker could be placed. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/how-it-works.md | 6 ++++-- lib/ask.py | 1 + lib/llm/base.py | 1 + lib/llm/chat.py | 7 ++++++- lib/llm/openrouter.py | 32 ++++++++++++++++++++++++++++++++ tests/test_handlers.py | 8 ++++++++ tests/test_llm_backends.py | 33 +++++++++++++++++++++++++++++++++ 7 files changed, 85 insertions(+), 3 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index ed50358..87c9757 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -43,8 +43,10 @@ - **OpenRouter:** OpenRouter's `web_search` tool; when it hands a search call back, the bot runs the query through `SEARCH_MODEL` (default `xiaomi/mimo-v2.6-flash:online`) and returns the write-up; that call's cost is added to the answer's cost. Some models print a search call as text instead of making it; the bot recovers those queries too. - The pages the search cited come back as `url_citation` annotations: a chat answer that links nothing itself ends with a - `Sources:` line of up to 3 of them (the last round that cited any). + The pages the search cited come back as `url_citation` annotations. In a chat answer each cited sentence gets a + `[1]` after it (the number links to the page, the same page keeps its number); the text is marked before any other + formatting, because the positions are in the raw reply. If no marker could be placed and the answer links nothing + itself, it ends with a `Sources:` line of up to 3 links instead. - At most 5 searches per round; extras get a "skipped" note. - `SEARCH=off` turns search off (the prompt then says the model has no web search). - **Forced search:** movers lists, DM preset buttons and the holding-news digest *require* a diff --git a/lib/ask.py b/lib/ask.py index cf47389..ed1003d 100644 --- a/lib/ask.py +++ b/lib/ask.py @@ -71,6 +71,7 @@ async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = on_text=_hide_needs_tools(on_text) if (route.simple or route.partial) else on_text, no_tools_system=no_tools_system(ctx.first_name), max_tokens=max_tokens or st.max_tokens, + cite=kind == "chat", # other kinds parse or format their own text ) answer = await policy.ask(ctx.backend, ctx.registry, req) await _record(ctx, req.model, kind, chat_id, answer) diff --git a/lib/llm/base.py b/lib/llm/base.py index 1316d9d..dc530d0 100644 --- a/lib/llm/base.py +++ b/lib/llm/base.py @@ -79,6 +79,7 @@ class Request: no_tools_system: str = "" # system prompt for the retry without tools local: dict[str, Callable[[dict], Awaitable[str]]] = field(default_factory=dict) # tools run in the bot max_tokens: int | None = None # reply cap incl. reasoning; None = the provider's setting + cite: bool = False # mark cited sentences with [n] links to their sources, where the provider says which @dataclass diff --git a/lib/llm/chat.py b/lib/llm/chat.py index 6bed859..f6bacf2 100644 --- a/lib/llm/chat.py +++ b/lib/llm/chat.py @@ -46,6 +46,10 @@ def _kwargs(self, req: Request, tool_choice: str | None) -> dict: def _read_delta(self, delta, extra: dict) -> None: """Pick provider-specific fields out of a streamed delta.""" + def _cite(self, text: str, extra: dict) -> str: + """The reply with [n] source links after the sentences the provider says it cited.""" + return text + def _assistant_fields(self, extra: dict) -> dict: """Extra fields the next request must carry on this step's assistant message.""" return {} @@ -96,7 +100,8 @@ async def step(self, conv: dict, req: Request, *, tool_choice: str | None) -> St tokens_out=getattr(u, "completion_tokens", 0) or 0, cost=self._cost(u, req.model), ) - return Step(text.strip(), calls, usage, finish, extra.get("searches", 0), raw=assistant, + shown = self._cite(text, extra) if req.cite else text # the model's own turn is replayed unmarked + return Step(shown.strip(), calls, usage, finish, extra.get("searches", 0), raw=assistant, sources=extra.get("sources", [])) def add_results(self, conv: dict, step: Step, results: list[str]) -> None: diff --git a/lib/llm/openrouter.py b/lib/llm/openrouter.py index 4b06b41..0680459 100644 --- a/lib/llm/openrouter.py +++ b/lib/llm/openrouter.py @@ -2,6 +2,7 @@ from __future__ import annotations +import html import logging import httpx @@ -35,6 +36,30 @@ def citations(annotations: list) -> list[tuple[str, str]]: return [(title, url) for url, title in out.items()] +def mark_citations(text: str, cites: list[tuple[int, str]]) -> str: + """Put "[n]" after each cited sentence, n a link to the page (the same page keeps its number). `cites` are + (end position in `text`, url); a position inside a tag or a link is skipped, and one right before the + sentence's closing punctuation moves past it.""" + numbers: dict[str, int] = {} + at: dict[int, list[int]] = {} + for end, url in sorted(cites): + pos = min(end, len(text)) + while pos < len(text) and text[pos] in ".!?": + pos += 1 + before = text[:pos] + if (pos <= 0 or before.rfind("<") > before.rfind(">") or before.count("") + or before.rfind("](") > before.rfind(")")): + continue + n = numbers.setdefault(url, len(numbers) + 1) + if n not in at.setdefault(pos, []): + at[pos].append(n) + urls = {n: url for url, n in numbers.items()} + for pos in sorted(at, reverse=True): + marks = "".join(f'[{n}]' for n in at[pos]) + text = text[:pos] + marks + text[pos:] + return text + + def merge_reasoning(details: list[dict], new: list) -> None: """Append streamed reasoning_details, joining consecutive text fragments of one block.""" for d in new: @@ -89,9 +114,16 @@ def _read_delta(self, delta, extra: dict) -> None: fields = getattr(delta, "model_extra", None) or {} merge_reasoning(extra.setdefault("details", []), fields.get("reasoning_details") or []) extra["searches"] = extra.get("searches", 0) + len(fields.get("annotations") or []) + for a in fields.get("annotations") or []: + c = a.get("url_citation") if isinstance(a, dict) and a.get("type") == "url_citation" else None + if c and isinstance(c.get("end_index"), int) and str(c.get("url") or "").startswith(("http://", "https://")): + extra.setdefault("cites", []).append((c["end_index"], c["url"])) have = extra.setdefault("sources", []) have += [c for c in citations(fields.get("annotations") or []) if c[1] not in {u for _, u in have}] + def _cite(self, text: str, extra: dict) -> str: + return mark_citations(text, extra.get("cites", [])) + def _assistant_fields(self, extra: dict) -> dict: return {"reasoning_details": extra["details"]} if extra.get("details") else {} diff --git a/tests/test_handlers.py b/tests/test_handlers.py index fab960a..79cb7c5 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -489,3 +489,11 @@ async def test_segments_shows_typing_while_it_works(env, store): m = user_msg(ctx.bot, "/segments", chat_id=-100) await h.segments(update_for(m), N(args=[])) assert ctx.bot.actions[:1] == ["typing"] and m.replies[0]["text"] == "Usage: /segments TICKER" + + +def test_numbered_source_links_survive_the_reply_formatting(): + from lib.telegram.handlers import format_answer + text = 'NVDA rose 3%.[1] **Strong.**[2]' + out = format_answer(text, movers_bots=frozenset(), summary=False) + assert '[1]' in out and '[2]' in out + assert "Strong." in out and "Sources:" not in out diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 1477c7d..9ef26aa 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -270,3 +270,36 @@ async def create(**kw): b = OpenRouterBackend(config.load(env), N(chat=N(completions=N(create=create)))) assert await b.run_search("nvda", Usage(), {}) == "- NVDA 100 [Reuters]\nLinks: https://r.example/n" + + +def cite_at(url, end): + return {"type": "url_citation", "url_citation": {"url": url, "title": "t", "start_index": 0, "end_index": end}} + + +def test_citations_become_numbered_links_after_the_sentence(): + from lib.llm.openrouter import mark_citations + a, b = "https://a.example/x?p=1&q=2", "https://b.example/" + text = "NVDA rose 3%. Micron guided up. Both beat." + out = mark_citations(text, [(12, a), (30, b), (41, a)]) + assert out == ('NVDA rose 3%.[1] Micron guided up.' + '[2] Both beat.[1]') + # two sources for one sentence; a spot inside a tag or a link, or past the end, is handled + assert mark_citations("One.", [(3, a), (3, b)]).endswith('.[1]' + '[2]') + assert mark_citations('See x.', [(10, a)]) == 'See x.' + assert mark_citations("Hi.", [(99, a)]).startswith("Hi.[") + assert mark_citations("Hi.", [(0, a)]) == "Hi." + + +async def test_openrouter_marks_cited_sentences_only_when_asked(env): + def chunks(): + return Stream([chat_chunk("Micron guided up.", extra={"annotations": [cite_at("https://m.example/", 16)]}), + chat_chunk(finish="stop")]) + + b = OpenRouterBackend(config.load(env), FakeChatClient(chunks(), chunks())) + r = req(search=True, cite=True) + step = await b.step(b.start(r), r, tool_choice=None) + assert step.text == 'Micron guided up.[1]' + assert step.raw["content"] == "Micron guided up." # replayed to the model unmarked + plain = req(search=True) + assert (await b.step(b.start(plain), plain, tool_choice=None)).text == "Micron guided up." From 3cc0e2b1c89a5a208414ec5f1a2164396e0b4774 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 01:36:52 +0000 Subject: [PATCH 07/30] Config: drop LOG_WIDTH, rename BOT_TZ/HISTORY_LIMIT/TELEGRAM_PRELOAD_CHAT; log user input as two words - LOG_WIDTH removed: tool-name lists are logged in full. - BOT_TZ -> TZ, HISTORY_LIMIT -> HISTORY_MAX (default 10, and a real maximum: the stepped window now holds max/2+1 .. max messages), TELEGRAM_PRELOAD_CHAT -> TELEGRAM_IMAGE_PRELOAD_CHATID. The old names are not read any more. - User input in the log is cut to its first two words and one ellipsis character; the other log-line truncations (error snippets, failed HTML) are gone. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/configuration.md | 11 +++++------ docs/features.md | 8 ++++---- docs/how-it-works.md | 6 +++--- docs/running.md | 4 ++-- lib/app.py | 3 +-- lib/config.py | 21 +++++++++------------ lib/features/charts/__init__.py | 2 +- lib/features/charts/breakdown.py | 4 ++-- lib/features/charts/buttons.py | 2 +- lib/features/charts/pages.py | 2 +- lib/features/filings.py | 4 ++-- lib/features/periods/backfill.py | 4 ++-- lib/mcp/server.py | 28 +++++++++------------------- lib/msgtext.py | 6 ++++++ lib/store.py | 17 +++++++++-------- lib/telegram/commands.py | 3 ++- lib/telegram/handlers.py | 7 +++---- lib/telegram/send.py | 4 ++-- tests/test_charts.py | 8 ++++---- tests/test_config.py | 7 ------- tests/test_handlers.py | 19 +++++++++++++++++++ tests/test_mcp.py | 26 +++++--------------------- tests/test_store.py | 18 ++++++++++++++---- 23 files changed, 106 insertions(+), 108 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 05d82db..aca091a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -14,11 +14,11 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o ### Common settings | Variable | Meaning | |---|---| -| `BOT_TZ` | Time zone for timestamps, e.g. Australia/Melbourne (default UTC). | +| `TZ` | Time zone for timestamps, e.g. Australia/Melbourne (default UTC). | | `MODEL` | Model ID (default grok-4.3 on xAI, z-ai/glm-5.3-flash on OpenRouter). | | `REASONING` | Reasoning effort: low, medium or high; empty = the model's default. | | `SEARCH` | on|off. Web search (and X search on xAI). Default on. | -| `DARKMODE` | Chart theme: `off` (the default) draws light charts, `on` draws dark ones, `auto` draws dark ones from 9 pm to 7 am in `BOT_TZ`. Before this setting existed, charts were always `auto`. | +| `DARKMODE` | Chart theme: `off` (the default) draws light charts, `on` draws dark ones, `auto` draws dark ones from 9 pm to 7 am in `TZ`. Before this setting existed, charts were always `auto`. | ### Advanced and optional | Variable | Meaning | @@ -26,8 +26,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | `ADMIN_CHAT_IDS` | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/tokens`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. | | `ADMIN_ONLY` | on|off. Only admins may DM the bot or get Sharesight data (default on). | | `DATA_DIR` | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. Default: systemd's `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)), else the working directory. In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. | -| `HISTORY_LIMIT` | Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20). | -| `LOG_WIDTH` | Longest line (characters) for the MCP tool lists in the log; a longer list is cut at a whole name and ends in `…` (default 80, 0 = no limit). | +| `HISTORY_MAX` | The most messages of chat history in the prompt (default 10). Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. | | `MAX_TOKENS` | Reply cap in tokens, reasoning included (default 3000). | | `MOVERS_BOTS` | Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off. | | `PORTFOLIO_NAMES` | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | @@ -35,9 +34,9 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | `POST_TO_GROUPS_FROM_DM` | on|off. A group admin can have the bot post in that group from a DM (default off). | | `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | -| `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (BOT_TZ) for the daily holding-news check (default 08:00). | +| `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (TZ) for the daily holding-news check (default 08:00). | | `TELEGRAM_DM_BUTTONS` | on|off. Preset-prompt buttons in private chats (default off). | -| `TELEGRAM_PRELOAD_CHAT` | Chat ID of a private chat the bot can post in (a private group it is in, or a DM with you). The other periods of a chart are uploaded there in the background and reused when a period button is tapped, so the tap swaps the picture without fetching or drawing. Empty (default) = off, and a tap fetches and draws. See [Charts](features.md#charts). | +| `TELEGRAM_IMAGE_PRELOAD_CHATID` | Chat ID of a private chat the bot can post in (a private group it is in, or a DM with you). The other periods of a chart are uploaded there in the background and reused when a period button is tapped, so the tap swaps the picture without fetching or drawing. Empty (default) = off, and a tap fetches and draws. See [Charts](features.md#charts). | ## Choosing a model Figures from search results and Artificial Analysis at the time of writing (October 2026), so diff --git a/docs/features.md b/docs/features.md index 670515d..996767a 100644 --- a/docs/features.md +++ b/docs/features.md @@ -4,7 +4,7 @@ ## Optional features (off by default) Each is off by default. The holding-news DM and the movers reply are switched on by setting the names they need (recipients, bots); the DM buttons have a flag. - **Daily holding-news DM** (switched on by listing recipients in `SHARESIGHT_HOLDING_NEWS_RECIPIENTS`, at `SHARESIGHT_HOLDING_NEWS_TIME` in - `BOT_TZ`, `portfolio:username` pairs such as + `TZ`, `portfolio:username` pairs such as `MyPortfolio:alice,MySMSF:alice`; empty, the default, means off): reads each person's current Sharesight holdings, searches for *major* news from the past 24 hours (a forced search), and DMs only what qualifies (most days: nothing). Stories already reported in @@ -43,7 +43,7 @@ for its definition. light weight (the one at zero too), short strokes above the x labels, and grey axis text. The subtitle says what the chart shows and when, like "% change over the past month, 30 Sep - 28 Oct 2026". Tick labels share their decimals, shorten big numbers (120M, 1.5B) and show % for % changes. A single y axis has no title (an inverted - one says "inverted"); two axes keep theirs. The theme follows `DARKMODE`: light (`off`, the default), dark (`on`), or dark from 9 pm to 7 am in `BOT_TZ` (`auto`). + one says "inverted"); two axes keep theirs. The theme follows `DARKMODE`: light (`off`, the default), dark (`on`), or dark from 9 pm to 7 am in `TZ` (`auto`). - **Line labels:** with lines on one axis, each line ends in a small square and a label with its name and total ("NVDA -5.7%"), nudged apart where they would overlap, and there is no legend. Two axes, bars or points keep a legend, with the totals in it. @@ -62,12 +62,12 @@ for its definition. - **Several charts, one image:** when a reply has 2 to 5 charts (for example a breakdown by segment, platform and region), the first is the image and each chart has a button under it, labelled with the part of its title that differs ("Segment", "Platform", "Region"). A tap swaps the image in place and keeps the answer as the caption; the one - shown is shaded. With `TELEGRAM_PRELOAD_CHAT` the other charts are uploaded in the background, so even the first tap + shown is shaded. With `TELEGRAM_IMAGE_PRELOAD_CHATID` the other charts are uploaded in the background, so even the first tap is instant; without it the first tap draws the chart (about half a second) and later taps reuse it. - **Period buttons:** a single chart whose series are all tickers comes with buttons: 1D, 1M, 1Y, 5Y and 🔄 (reload); the period shown is shaded. A tap fetches the prices again and redraws the chart in place, without asking the model. Another period gets a plain title, repeated as the caption; the chart's own period brings back the model's caption. - Set `TELEGRAM_PRELOAD_CHAT` to a private chat the bot can post in (a private group, or a DM with you) and the + Set `TELEGRAM_IMAGE_PRELOAD_CHATID` to a private chat the bot can post in (a private group, or a DM with you) and the other periods are drawn and uploaded there in the background right after the reply; a tap then swaps in the uploaded picture without fetching or drawing, and so does going back to the chart's own period. A picture older than an hour (15 minutes for 1D) is drawn again with fresh prices, and 🔄 always is. Without the variable every tap draws. diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 87c9757..37325ad 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -8,8 +8,8 @@ - **Private chats:** every message is answered, and the reply streams into a Telegram message draft ("Thinking..." then the text as it arrives). Replies are not quoted in DMs. - **History:** the prompt carries a window of the chat's recent messages, oldest first. The window - is `HISTORY_LIMIT` to 1.5x that many messages (20-29 by default): its start only moves every - `HISTORY_LIMIT/2` messages, so the start of the prompt stays identical between requests and the + is at most `HISTORY_MAX` messages (6 to 10 by default, once the chat is longer than that): its start only moves every + `HISTORY_MAX/2` messages, so the start of the prompt stays identical between requests and the provider's prompt cache keeps working. The bot's own earlier replies appear as "You" (its latest reply in full, so follow-up questions about it work; older ones shortened); a replied-to message is quoted even if it is older than the window. @@ -33,7 +33,7 @@ do" is answered from the actual tool list (about 400 tokens in the prompt, no tool offered): the data sources and their tools, web search, charts, and the other local tools. Tools the asker may not use (Sharesight for a non-admin, posting to groups when it is off) are left out. -- **Time:** the current date and time (in `BOT_TZ`) is included in every prompt. +- **Time:** the current date and time (in `TZ`) is included in every prompt. - **Replies are final:** the prompt tells the model never to say it will "check later"; it makes its tool calls first (several at once) and answers in one go. - **Errors:** a failed request replies with the error text; the bot never goes silent. diff --git a/docs/running.md b/docs/running.md index cf72089..a6a238d 100644 --- a/docs/running.md +++ b/docs/running.md @@ -75,8 +75,8 @@ from the stream alone: warnings and errors are stored as err, the rest as info, | info | Startup and connections, one line per message the bot answers, and one summary per request. | | debug | The detail: each MCP call and its result size, search queries, tokens per round, attached images, backfill progress. | -A message line looks like `[Jane Doe (@jane) 1234567890] first 60 characters of the message ...` (groups add +A message line looks like `[Jane Doe (@jane) 1234567890] first two words of the message…` (groups add ` @ `). The summary has the time, rounds, tool calls, searches, tokens, cached % and cost; the debug line `Round 2: in 14158 (7400 cached) out 738, stop` gives each round's share. Each tool server logs its tools on one -line (cut at `LOG_WIDTH`), and startup logs the token size of the tool definitions. The `Starting ...` line shows the +line, and startup logs the token size of the tool definitions. The `Starting ...` line shows the git commit the checkout is on. diff --git a/lib/app.py b/lib/app.py index 31e300e..3694bb4 100644 --- a/lib/app.py +++ b/lib/app.py @@ -168,8 +168,7 @@ async def on_down(server: MCPServer) -> None: await alert_down(ctx, server) ctx.registry = Registry.load(str(config.MCP_CONFIG), timeout=config.MCP_TIMEOUT, - max_output=config.MAX_TOOL_OUTPUT, on_down=on_down, - log_width=st.log_width) + max_output=config.MAX_TOOL_OUTPUT, on_down=on_down) ctx.chat_commands = chat_commands.load({label: s.cfg for label, s in ctx.registry.servers.items()}, BUILT_IN_COMMANDS) return ctx diff --git a/lib/config.py b/lib/config.py index 11eee0c..5ef6991 100644 --- a/lib/config.py +++ b/lib/config.py @@ -17,17 +17,16 @@ "TELEGRAM_BOT_TOKEN": "Bot token from @BotFather (required).", "XAI_API_KEY": "xAI key; selects the xAI provider. Set exactly one of the three keys.", # Common settings - "BOT_TZ": "Time zone for timestamps, e.g. Australia/Melbourne (default UTC).", + "TZ": "Time zone for timestamps, e.g. Australia/Melbourne (default UTC).", "MODEL": "Model ID (default grok-4.3 on xAI, z-ai/glm-5.3-flash on OpenRouter).", "REASONING": "Reasoning effort: low, medium or high; empty = the model's default.", "SEARCH": "on|off. Web search (and X search on xAI). Default on.", - "DARKMODE": "Chart theme: off (default, light), on (dark) or auto (dark from 9 pm to 7 am in BOT_TZ).", + "DARKMODE": "Chart theme: off (default, light), on (dark) or auto (dark from 9 pm to 7 am in TZ).", # Advanced and optional "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /tokens; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.", "ADMIN_ONLY": "on|off. Only admins may DM the bot or get Sharesight data (default on).", "DATA_DIR": "Directory for the bot's data (the database); Default: systemd's STATE_DIRECTORY, else the working directory.", - "HISTORY_LIMIT": "Messages of chat history in the prompt; the window is HISTORY_LIMIT to 1.5x (default 20).", - "LOG_WIDTH": "Longest line (characters) for the MCP tool lists in the log; longer lists end in \"…\" (default 80, 0 = no limit).", + "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.", "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).", "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.", "PORTFOLIO_NAMES": "Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names).", @@ -35,9 +34,9 @@ "POST_TO_GROUPS_FROM_DM": "on|off. Lets an admin of a group the bot is in make it post there from a DM (default off).", "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", - "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (BOT_TZ) for the daily holding-news check (default 08:00).", + "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (TZ) for the daily holding-news check (default 08:00).", "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).", - "TELEGRAM_PRELOAD_CHAT": "Chat ID of a private chat the bot can post in. Chart periods are uploaded there ahead of time and reused when a period button is tapped. Empty (default) = off.", + "TELEGRAM_IMAGE_PRELOAD_CHATID": "Chat ID of a private chat the bot can post in. Chart periods are uploaded there ahead of time and reused when a period button is tapped. Empty (default) = off.", } DEFAULT_MODELS = {"xai": "grok-4.3", "openrouter": "z-ai/glm-5.3-flash"} @@ -77,8 +76,7 @@ class Settings: search: bool search_model: str max_tokens: int - history_limit: int - log_width: int + history_max: int darkmode: str # "off" | "on" | "auto": the charts' theme db_path: str tz: ZoneInfo @@ -186,11 +184,10 @@ def load(env: Mapping[str, str] | None = None) -> Settings: search=search, search_model=search_model, max_tokens=int(env.get("MAX_TOKENS") or 3000), - history_limit=int(env.get("HISTORY_LIMIT") or 20), - log_width=int(env.get("LOG_WIDTH") or 80), + history_max=int(env.get("HISTORY_MAX") or 10), darkmode=darkmode, db_path=database_path(env), - tz=ZoneInfo(env.get("BOT_TZ", "UTC")), + tz=ZoneInfo(env.get("TZ", "UTC")), admin_chats=_ids(env.get("ADMIN_CHAT_IDS", "")), admin_only=_flag(env, "ADMIN_ONLY", default=True), movers_bots=frozenset( @@ -198,7 +195,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings: ), dm_buttons=_flag(env, "TELEGRAM_DM_BUTTONS"), post_to_groups=_flag(env, "POST_TO_GROUPS_FROM_DM"), - preload_chat=_chat_id(env, "TELEGRAM_PRELOAD_CHAT"), + preload_chat=_chat_id(env, "TELEGRAM_IMAGE_PRELOAD_CHATID"), holding_news_time=env.get("SHARESIGHT_HOLDING_NEWS_TIME", "08:00"), holding_news_recipients=recipients, portfolio_names=_names(env.get("PORTFOLIO_NAMES", "")) or frozenset(recipients), diff --git a/lib/features/charts/__init__.py b/lib/features/charts/__init__.py index 3ff206a..e4d312d 100644 --- a/lib/features/charts/__init__.py +++ b/lib/features/charts/__init__.py @@ -25,7 +25,7 @@ DEFAULT_INTERVAL = {"1d": "5m", "5d": "30m", "1mo": "1d", "3mo": "1d", "6mo": "1wk", "1y": "1wk", "2y": "1mo", "5y": "1mo", "10y": "3mo", "max": "3mo"} MAX_CHARTS, MAX_SERIES, MAX_POINTS = 5, 8, 2000 -NIGHT_FROM, NIGHT_TO = 21, 7 # the dark theme from 9 pm to 7 am (BOT_TZ), when the chart is drawn +NIGHT_FROM, NIGHT_TO = 21, 7 # the dark theme from 9 pm to 7 am (TZ), when the chart is drawn TOTAL_MAX = 24 # characters of a legend total CAPTION_HINT = 900 # Telegram captions hold 1024 characters; a longer reply continues in a text message HISTORY_TOOL = "get_historical_prices" diff --git a/lib/features/charts/breakdown.py b/lib/features/charts/breakdown.py index fe7973c..0d44419 100644 --- a/lib/features/charts/breakdown.py +++ b/lib/features/charts/breakdown.py @@ -147,7 +147,7 @@ def parsed(ticker: str, key: str, out) -> dict | None: try: data = json.loads(out) except (TypeError, ValueError): - log.warning("Fast breakdown for %s: couldn't read %s: %s", ticker, key, " ".join(str(out).split())[:200]) + log.warning("Fast breakdown for %s: couldn't read %s: %s", ticker, key, " ".join(str(out).split())) return None if not isinstance(data, dict) or data.get("available") is False: log.debug("Fast breakdown for %s: %s isn't available: %s", ticker, key, isinstance(data, dict) and data.get("message")) @@ -189,7 +189,7 @@ async def fast(ctx: Ctx, text: str, charts: Charts) -> str | None: symbol = CURRENCIES.get(str(payload.get("reporting_currency"))) factor = read_units(payload, 0.0) except (ValueError, KeyError, IndexError, TypeError): - log.warning("Fast breakdown for %s: couldn't read the %s result (%s)", ticker, TOOL, " ".join(out.split())[:200]) + log.warning("Fast breakdown for %s: couldn't read the %s result (%s)", ticker, TOOL, " ".join(out.split())) return None if not symbol or not factor: log.warning("Fast breakdown for %s: currency %r or units %r not understood", ticker, diff --git a/lib/features/charts/buttons.py b/lib/features/charts/buttons.py index d619365..9a99598 100644 --- a/lib/features/charts/buttons.py +++ b/lib/features/charts/buttons.py @@ -85,7 +85,7 @@ async def remember(ctx: Ctx, sent, args: dict) -> None: async def preload(ctx: Ctx, chat_id: int, message_id: int, args: dict) -> None: - """Draw the other periods and upload them to TELEGRAM_PRELOAD_CHAT, keeping their file IDs for the buttons.""" + """Draw the other periods and upload them to TELEGRAM_IMAGE_PRELOAD_CHATID, keeping their file IDs for the buttons.""" shown = period_of(args) for _, period in BUTTONS: if period == shown: diff --git a/lib/features/charts/pages.py b/lib/features/charts/pages.py index 10269f7..172825c 100644 --- a/lib/features/charts/pages.py +++ b/lib/features/charts/pages.py @@ -55,7 +55,7 @@ def keyboard(labels: list[str], current: int) -> InlineKeyboardMarkup: async def remember(ctx: Ctx, sent: Message, calls: list[dict], images: list[bytes], labels: list[str]) -> None: """Keep the charts' calls so a tap can draw any of them, the picture just sent as chart 0, and, with - TELEGRAM_PRELOAD_CHAT, upload the others (already drawn) so the first tap is instant too.""" + TELEGRAM_IMAGE_PRELOAD_CHATID, upload the others (already drawn) so the first tap is instant too.""" caption = getattr(sent, "caption_html", None) or "" await asyncio.to_thread(ctx.store.chart_put, sent.chat_id, sent.message_id, json.dumps({"pages": calls, "labels": labels}), caption, "#0") diff --git a/lib/features/filings.py b/lib/features/filings.py index 1c819eb..3a4c3e9 100644 --- a/lib/features/filings.py +++ b/lib/features/filings.py @@ -79,7 +79,7 @@ async def fetch(ctx: Ctx, kind: Kind, ticker: str, section: str) -> str | None: return None out = await server.call(kind.tool, {"ticker": ticker, "section": section}) if out.startswith(("Tool error", "Error")): - log.warning("%s %s %s: %s", kind.tool, ticker, section, " ".join(out.split())[:200]) + log.warning("%s %s %s: %s", kind.tool, ticker, section, " ".join(out.split())) return None try: data = json.loads(out) @@ -87,7 +87,7 @@ async def fetch(ctx: Ctx, kind: Kind, ticker: str, section: str) -> str | None: return out # markdown, the usual answer if isinstance(data, dict) and data.get("available") is False: return str(data.get("message") or "") or None - log.warning("%s %s %s: unexpected JSON result: %s", kind.tool, ticker, section, " ".join(out.split())[:200]) + log.warning("%s %s %s: unexpected JSON result: %s", kind.tool, ticker, section, " ".join(out.split())) return None diff --git a/lib/features/periods/backfill.py b/lib/features/periods/backfill.py index 074b90e..a64eefb 100644 --- a/lib/features/periods/backfill.py +++ b/lib/features/periods/backfill.py @@ -110,11 +110,11 @@ async def worker() -> None: BAD.add((pid, *key(w))) nodata += 1 if nodata <= SAMPLE_NODATA: # enough of them to see what Sharesight says, not one line per empty day - log.debug("Period backfill: %s %s..%s has no data: %s", name, w[1], w[2], got.text[:150]) + log.debug("Period backfill: %s %s..%s has no data: %s", name, w[1], w[2], " ".join(got.text.split())) else: # a blip says nothing about the window itself, so it is tried again next pass streak[0] += 1 transient += 1 - log.warning("Period backfill: %s %s..%s failed: %s", name, w[1], w[2], got.text[:150]) + log.warning("Period backfill: %s %s..%s failed: %s", name, w[1], w[2], " ".join(got.text.split())) if streak[0] >= MAX_FAILS and not stopped: stopped = True log.warning("Period backfill: %d fetches in a row failed; stopping this pass", streak[0]) diff --git a/lib/mcp/server.py b/lib/mcp/server.py index bad50e3..4b6ade5 100644 --- a/lib/mcp/server.py +++ b/lib/mcp/server.py @@ -43,11 +43,11 @@ class MCPServer: entered and exited in the same task.""" def __init__(self, label: str, cfg: dict, *, timeout: float, max_output: int, - on_down: Callable[[MCPServer], Awaitable[None]] | None = None, log_width: int = 80): + on_down: Callable[[MCPServer], Awaitable[None]] | None = None): self.label = re.sub(r"[^A-Za-z0-9_-]", "_", label) self.cfg = cfg self.description = cfg.get("description", "") - self.timeout, self.max_output, self.on_down, self.log_width = timeout, max_output, on_down, log_width + self.timeout, self.max_output, self.on_down = timeout, max_output, on_down self.session: ClientSession | None = None self.error: str | None = None # why the server is down, if it is self.tools: list[ToolDef] = [] @@ -148,9 +148,9 @@ async def _load_tools(self, session: ClientSession) -> None: self._warn_command_tools(listed) enabled = sorted(self.fn_names.values()) skipped = sorted(t.name for t in listed if t.name not in enabled) - log_names(f"MCP {self.label}: {len(enabled)} tools:", enabled, self.log_width) + log_names(f"MCP {self.label}: {len(enabled)} tools:", enabled) if skipped: - log_names(f"MCP {self.label}: {len(skipped)} skipped:", skipped, self.log_width) + log_names(f"MCP {self.label}: {len(skipped)} skipped:", skipped) def _warn_command_tools(self, listed) -> None: """A chat command whose tool isn't enabled can't run: say why (a typo, or a tool that isn't read-only and @@ -306,7 +306,7 @@ def __init__(self, servers: list[MCPServer]): @classmethod def load(cls, path: str, *, timeout: float, max_output: int, - on_down: Callable[[MCPServer], Awaitable[None]] | None = None, log_width: int = 80) -> Registry: + on_down: Callable[[MCPServer], Awaitable[None]] | None = None) -> Registry: """Servers from a JSON file ({"mcpServers": {...}} or Claude Desktop style); a missing file means no servers.""" try: @@ -315,8 +315,7 @@ def load(cls, path: str, *, timeout: float, max_output: int, except FileNotFoundError: return cls([]) cfg = cfg.get("mcpServers", cfg) - return cls([MCPServer(label, c, timeout=timeout, max_output=max_output, on_down=on_down, - log_width=log_width) + return cls([MCPServer(label, c, timeout=timeout, max_output=max_output, on_down=on_down) for label, c in cfg.items() if not c.get("disabled")]) def start(self) -> None: @@ -347,18 +346,9 @@ def lookup(self, fn_name: str) -> tuple[MCPServer, str] | None: return None -def log_names(head: str, names: list[str], width: int = 80) -> None: - """Log a list of tool names on one line of at most `width` characters (0 = no limit). A list that - doesn't fit is cut at a whole name and ends in "…" (one character).""" - line = f"{head} {', '.join(names)}" - if width and len(line) > width: - kept = f"{head} " - for i, name in enumerate(names): - if len(kept) + len(name) + len(", …") > width and i: - break - kept += f"{name}, " - line = kept + "…" - log.info("%s", line) +def log_names(head: str, names: list[str]) -> None: + """Log a list of tool names on one line.""" + log.info("%s %s", head, ", ".join(names)) def _leaf_errors(e: BaseException) -> list[BaseException]: diff --git a/lib/msgtext.py b/lib/msgtext.py index ab94404..a219cb0 100644 --- a/lib/msgtext.py +++ b/lib/msgtext.py @@ -10,6 +10,12 @@ ) +def log_preview(text: str, words: int = 2) -> str: + """User input for the log: its first `words` words, then "…" (one character) if there were more.""" + parts = (text or "").split() + return " ".join(parts[:words]) + ("…" if len(parts) > words else "") + + def sender_name(msg: Message) -> str: """'Full Name (@username)'; must match how a bot's own replies are labelled in history.""" if msg.from_user: diff --git a/lib/store.py b/lib/store.py index b3ae888..525883c 100644 --- a/lib/store.py +++ b/lib/store.py @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS charts ( bot_id INTEGER, chat_id INTEGER, message_id INTEGER, spec TEXT, caption TEXT, period TEXT, ts INTEGER, PRIMARY KEY (bot_id, chat_id, message_id)); --- Chart images already on Telegram (uploaded to TELEGRAM_PRELOAD_CHAT, or sent as the reply): reused by the period buttons. +-- Chart images already on Telegram (uploaded to TELEGRAM_IMAGE_PRELOAD_CHATID, or sent as the reply): reused by the period buttons. CREATE TABLE IF NOT EXISTS chart_files ( bot_id INTEGER, chat_id INTEGER, message_id INTEGER, period TEXT, file_id TEXT, caption TEXT, ts INTEGER, PRIMARY KEY (bot_id, chat_id, message_id, period)); @@ -128,20 +128,21 @@ def forget_chat(self, chat_id: int) -> None: def chats(self) -> list[tuple[int, str]]: return [(r[0], r[1]) for r in self._run("SELECT chat_id, title FROM chats")] - def history(self, chat_id: int, limit: int) -> list[HistoryRow]: - """Recent messages, oldest first. + def history(self, chat_id: int, max_rows: int) -> list[HistoryRow]: + """Recent messages, oldest first, never more than `max_rows`. A plain "last N" window drops its oldest line on every message, changing the start of - the prompt and defeating prompt caching. Instead the window's start only moves every - limit/2 messages: it holds between `limit` and 1.5 x `limit` messages and in between - only grows at the end, so everything before the new messages stays cached.""" + the prompt and defeating prompt caching. Instead, once the chat is longer than `max_rows`, + the window's start only moves every max_rows/2 messages: it holds between max_rows/2 + 1 + and `max_rows` messages and in between only grows at the end, so everything before the new + messages stays cached.""" if is_dm(chat_id): table, where, args = "dm_messages", "bot_id = ? AND chat_id = ?", (self._own_id(), chat_id) else: table, where, args = "messages", "chat_id = ?", (chat_id,) total = self._run(f"SELECT COUNT(*) FROM {table} WHERE {where}", args)[0][0] - step = max(1, limit // 2) - size = total - max(0, (total - limit) // step * step) + step = max(1, max_rows // 2) + size = total - max(0, -(-(total - max_rows) // step) * step) rows = self._run( f"SELECT message_id, sender, text, ts, reply_to FROM {table} " f"WHERE {where} ORDER BY message_id DESC LIMIT ?", (*args, size)) diff --git a/lib/telegram/commands.py b/lib/telegram/commands.py index 6384a4b..7595f3d 100644 --- a/lib/telegram/commands.py +++ b/lib/telegram/commands.py @@ -16,6 +16,7 @@ from ..features import filings from ..features.periods import backfill from ..features.periods.windows import describe_scope, flush_scope +from ..msgtext import log_preview from ..store import is_dm from ..textfmt import is_parse_error, plain_text from .access import is_admin @@ -32,7 +33,7 @@ async def _gate(ctx: Ctx, update: Update) -> bool: user = update.effective_user if user and await is_admin(ctx, user.id): return True - log.info("Ignored %s from user %s: not an admin", update.effective_message.text, user and user.id) + log.info("Ignored %s from user %s: not an admin", log_preview(update.effective_message.text), user and user.id) return False diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py index 785f604..f3cbf74 100644 --- a/lib/telegram/handlers.py +++ b/lib/telegram/handlers.py @@ -26,7 +26,7 @@ from ..history import compact, format_rows from ..llm.base import BUSY_REPLY, is_busy from ..llm.gate import Route, asks_tools, is_portfolio_question, route -from ..msgtext import describe, sender_name +from ..msgtext import describe, log_preview, sender_name from ..prompts import MOVERS_EXPLAIN, PRESET_PROMPT, capabilities_note, chat_prompt, down_note from ..store import is_dm from ..textfmt import md_to_html, strip_disclaimer @@ -215,7 +215,7 @@ async def _gate_text(self, msg: Message, trig: Trigger) -> str: async def _chat_text(self, msg: Message, trig: Trigger, middle: str = "", fresh: bool = False, no_history: bool = False) -> str: ctx, st = self.ctx, self.ctx.st - rows = [] if no_history else await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_limit) + rows = [] if no_history else await asyncio.to_thread(ctx.store.history, msg.chat_id, st.history_max) if trig.nocontext: # only the message that tagged the bot rows = [r._replace(text=NO_CONTEXT.sub("", r.text).strip(), reply_to=None) for r in rows if r.message_id == msg.message_id] @@ -231,8 +231,7 @@ async def _answer(self, msg: Message, trig: Trigger, typing: asyncio.Task | None ctx, st = self.ctx, self.ctx.st who = f"{sender_name(msg)} {msg.from_user.id if msg.from_user else None}" where = "" if trig.private else f" @ {msg.chat_id}" - preview = " ".join(trig.text.split()) - preview = preview[:60].rstrip() + " ..." if len(preview) > 60 else preview + preview = log_preview(trig.text) log.info("[%s%s]%s %s", who, where, " (movers list)" if trig.movers else "", preview or f"[{describe(msg) or 'no text'}]") diff --git a/lib/telegram/send.py b/lib/telegram/send.py index 67be769..49b767b 100644 --- a/lib/telegram/send.py +++ b/lib/telegram/send.py @@ -42,7 +42,7 @@ async def reply_html(msg: Message, text: str, *, quote: bool = True, reply_marku if not is_parse_error(e): raise log.warning("Bad HTML from the model, sending as plain text: %s", e) - log.debug("The text that failed: %r", text[:600]) + log.debug("The text that failed: %r", text) return await msg.reply_text( plain_text(text), do_quote=quote, disable_notification=True, link_preview_options=NO_PREVIEW, reply_markup=reply_markup) @@ -88,5 +88,5 @@ async def _send_photo(msg: Message, photo: bytes, caption: str, quote: bool, mar if not html_ok or not is_parse_error(e): raise log.warning("Bad HTML from the model, sending the caption as plain text: %s", e) - log.debug("The caption that failed: %r", text[:600]) + log.debug("The caption that failed: %r", text) raise RuntimeError("unreachable") diff --git a/tests/test_charts.py b/tests/test_charts.py index 11c89c6..48dc6f8 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -442,7 +442,7 @@ async def settle(): await asyncio.gather(*list(buttons._tasks)) -PRELOAD = {"TELEGRAM_PRELOAD_CHAT": "-900"} +PRELOAD = {"TELEGRAM_IMAGE_PRELOAD_CHATID": "-900"} async def test_other_periods_are_uploaded_to_the_preload_chat_and_reused_by_the_buttons(env, store): @@ -499,10 +499,10 @@ async def test_a_preload_chat_the_bot_cannot_post_in_stops_after_one_warning(env def test_the_preload_chat_must_be_a_number(env): - assert config.load({**env, "TELEGRAM_PRELOAD_CHAT": " -1001234 "}).preload_chat == -1001234 + assert config.load({**env, "TELEGRAM_IMAGE_PRELOAD_CHATID": " -1001234 "}).preload_chat == -1001234 assert config.load(env).preload_chat is None - with pytest.raises(config.ConfigError, match="TELEGRAM_PRELOAD_CHAT must be a Telegram chat ID"): - config.load({**env, "TELEGRAM_PRELOAD_CHAT": "mychat"}) + with pytest.raises(config.ConfigError, match="TELEGRAM_IMAGE_PRELOAD_CHATID must be a Telegram chat ID"): + config.load({**env, "TELEGRAM_IMAGE_PRELOAD_CHATID": "mychat"}) async def test_an_upload_is_reused_for_an_hour_but_the_1d_chart_for_15_minutes(env, store, monkeypatch): diff --git a/tests/test_config.py b/tests/test_config.py index 3cbce36..2f13911 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -67,13 +67,6 @@ def test_holding_news_and_movers_switch_on_by_listing_names(env): assert not config.load({**env, "MOVERS_BOTS": ","}).movers_explain -def test_log_width_defaults_to_80(env): - assert config.load(env).log_width == 80 - assert config.load({**env, "LOG_WIDTH": "120"}).log_width == 120 - assert config.load({**env, "LOG_WIDTH": "0"}).log_width == 0 # 0 = no limit - assert config.load({**env, "LOG_WIDTH": ""}).log_width == 80 - - def test_token_limits(): assert config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a"}).max_tokens == 3000 assert config.MAX_TOOL_OUTPUT == 12_000 and config.HISTORY_OWN_LINE_MAX < config.HISTORY_LINE_MAX diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 79cb7c5..833c137 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -497,3 +497,22 @@ def test_numbered_source_links_survive_the_reply_formatting(): out = format_answer(text, movers_bots=frozenset(), summary=False) assert '[1]' in out and '[2]' in out assert "Strong." in out and "Sources:" not in out + + +def test_log_preview_keeps_two_words_and_one_ellipsis_character(): + from lib.msgtext import log_preview + assert log_preview("") == "" and log_preview("hi") == "hi" and log_preview("/segments $nvda") == "/segments $nvda" + assert log_preview("@stockbot how is NVDA?") == "@stockbot how…" + assert log_preview(" @stockbot \n how is\tit ") == "@stockbot how…" + assert log_preview("one two three four", words=3) == "one two three…" + + +async def test_the_log_shows_only_the_first_two_words_of_a_message(env, store, caplog): + import logging + ctx, backend, h = make_ctx(env, store, [step("ok")]) + m = user_msg(ctx.bot, "@stockbot my secret portfolio plan is to buy NVDA") + with caplog.at_level(logging.INFO, logger="bot"): + await run(h, m) + lines = [r.getMessage() for r in caplog.records] + assert any(line.endswith("@stockbot my…") for line in lines) + assert not any("secret" in line or "portfolio" in line for line in lines) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 0dc4d7d..98b6521 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -105,31 +105,15 @@ def test_log_names_keeps_exact_names(caplog): assert "get_analyst_estimates" in caplog.text and "get_stock_quote" in caplog.text -def test_log_names_is_one_line_cut_at_a_whole_name(caplog): +def test_log_names_is_one_line_and_never_cut(caplog): import logging from lib.mcp.server import log_names - names = ["compare_companies", "get_company", "get_company_metrics", "get_data_notes", "get_events"] - head = "MCP t: 5 tools:" + names = [f"get_a_rather_long_tool_name_{i}" for i in range(12)] with caplog.at_level(logging.INFO, logger="bot"): - log_names(head, names, 95) - log_names(head, names, 60) - log_names(head, names, 0) - log_names(head, names, 5) - short, cut, unlimited, tiny = (r.getMessage() for r in caplog.records) - assert short == f"{head} {', '.join(names)}" and len(short) <= 95 - assert cut == f"{head} compare_companies, get_company, …" and len(cut) <= 60 - assert unlimited == short - assert tiny == f"{head} compare_companies, …" # the first name is always shown - - -def test_the_ellipsis_is_one_character_so_it_costs_one_character_of_the_width(caplog): - import logging - - from lib.mcp.server import log_names - with caplog.at_level(logging.INFO, logger="bot"): - log_names("H:", ["aaaa", "bbbb", "cccc"], 17) - assert caplog.records[0].getMessage() == "H: aaaa, bbbb, …" # "..." would have room for one name only + log_names("MCP t: 12 tools:", names) + [line] = [r.getMessage() for r in caplog.records] + assert line == f"MCP t: 12 tools: {', '.join(names)}" and len(line) > 300 and "…" not in line async def test_call_slims_and_caches_identical_concurrent_calls(): diff --git a/tests/test_store.py b/tests/test_store.py index 658b5f2..e5dd645 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -37,19 +37,29 @@ def test_dm_before_bot_id_fails_clearly(tmp_path): s.save(make_msg(1, "group fine", chat_id=-5)) # groups need no bot id -def test_history_window_is_stepped(store): +def test_history_window_is_stepped_and_never_over_the_maximum(store): for i in range(1, 46): store.save(make_msg(i, f"m{i}", chat_id=-1, ts=i)) rows = store.history(-1, 20) - assert len(rows) == 25 and rows[0].message_id == 21 and rows[-1].message_id == 45 - # the start only moves every limit/2 messages + assert len(rows) == 15 and rows[0].message_id == 31 and rows[-1].message_id == 45 + # the start only moves every max/2 messages starts = set() for i in range(46, 66): store.save(make_msg(i, f"m{i}", chat_id=-1, ts=i)) rows = store.history(-1, 20) - assert 20 <= len(rows) <= 29 + assert 11 <= len(rows) <= 20 and rows[-1].message_id == i starts.add(rows[0].message_id) assert len(starts) <= 3 + assert [r.message_id for r in store.history(-1, 2)] == [64, 65] + assert len(store.history(-1, 1000)) == 65 # a short chat is returned whole + + +def test_a_history_maximum_of_ten_gives_six_to_ten_messages(store): + for i in range(1, 40): + store.save(make_msg(i, f"m{i}", chat_id=-2, ts=i)) + assert len(store.history(-2, 10)) <= 10 + if i > 10: + assert len(store.history(-2, 10)) >= 6 def test_kv_users_news_and_usage(store): From 02f8cbb460198e7b6fb87e55dad0444f7a82bcd9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:04:33 +0000 Subject: [PATCH 08/30] Docs: ADMIN_ONLY says what on and off do; configuration tables get a Default column Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/configuration.md | 56 +++++++++++++++++++++---------------------- lib/config.py | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index aca091a..2a0d316 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -5,38 +5,38 @@ ### Essential Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than one, or none, the bot refuses to start. -| Variable | Meaning | -|---|---| -| `OPENROUTER_API_KEY` | OpenRouter key; selects the OpenRouter provider (default model `z-ai/glm-5.3-flash`). | -| `TELEGRAM_BOT_TOKEN` | Bot token from @BotFather (required). | -| `XAI_API_KEY` | xAI key; selects the xAI provider (default model `grok-4.3`). | +| Variable | Default | Meaning | +|---|---|---| +| `OPENROUTER_API_KEY` | none | OpenRouter key; selects the OpenRouter provider (its default model is `z-ai/glm-5.3-flash`). | +| `TELEGRAM_BOT_TOKEN` | none (required) | Bot token from @BotFather (required). | +| `XAI_API_KEY` | none | xAI key; selects the xAI provider (its default model is `grok-4.3`). | ### Common settings -| Variable | Meaning | -|---|---| -| `TZ` | Time zone for timestamps, e.g. Australia/Melbourne (default UTC). | -| `MODEL` | Model ID (default grok-4.3 on xAI, z-ai/glm-5.3-flash on OpenRouter). | -| `REASONING` | Reasoning effort: low, medium or high; empty = the model's default. | -| `SEARCH` | on|off. Web search (and X search on xAI). Default on. | -| `DARKMODE` | Chart theme: `off` (the default) draws light charts, `on` draws dark ones, `auto` draws dark ones from 9 pm to 7 am in `TZ`. Before this setting existed, charts were always `auto`. | +| Variable | Default | Meaning | +|---|---|---| +| `TZ` | `UTC` | Time zone for timestamps, e.g. `Australia/Melbourne`. | +| `MODEL` | `grok-4.3` on xAI, `z-ai/glm-5.3-flash` on OpenRouter | Model ID. | +| `REASONING` | the model's own | Reasoning effort: `low`, `medium` or `high`. | +| `SEARCH` | `on` | `on` or `off`. Web search (and X search on xAI). | +| `DARKMODE` | `off` | Chart theme: `off` draws light charts, `on` draws dark ones, `auto` draws dark ones from 9 pm to 7 am in `TZ`. Before this setting existed, charts were always `auto`. | ### Advanced and optional -| Variable | Meaning | -|---|---| -| `ADMIN_CHAT_IDS` | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/tokens`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. | -| `ADMIN_ONLY` | on|off. Only admins may DM the bot or get Sharesight data (default on). | -| `DATA_DIR` | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. Default: systemd's `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)), else the working directory. In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. | -| `HISTORY_MAX` | The most messages of chat history in the prompt (default 10). Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. | -| `MAX_TOKENS` | Reply cap in tokens, reasoning included (default 3000). | -| `MOVERS_BOTS` | Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off. | -| `PORTFOLIO_NAMES` | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | -| `PERIOD_BACKFILL` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. Default `all`; list names to limit it, or `off` to disable. See [Caching Sharesight history](caching.md#caching-sharesight-history). | -| `POST_TO_GROUPS_FROM_DM` | on|off. A group admin can have the bot post in that group from a DM (default off). | -| `SEARCH_MODEL` | OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online). | -| `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off. | -| `SHARESIGHT_HOLDING_NEWS_TIME` | HH:MM (TZ) for the daily holding-news check (default 08:00). | -| `TELEGRAM_DM_BUTTONS` | on|off. Preset-prompt buttons in private chats (default off). | -| `TELEGRAM_IMAGE_PRELOAD_CHATID` | Chat ID of a private chat the bot can post in (a private group it is in, or a DM with you). The other periods of a chart are uploaded there in the background and reused when a period button is tapped, so the tap swaps the picture without fetching or drawing. Empty (default) = off, and a tap fetches and draws. See [Charts](features.md#charts). | +| Variable | Default | Meaning | +|---|---|---| +| `ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/tokens`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. | +| `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/tokens` and `/flushcache` stay admin-only. Who counts as an admin: `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). | +| `DATA_DIR` | systemd's `STATE_DIRECTORY`, else the working directory | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. With systemd the default is its `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)). In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. | +| `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. | +| `MAX_TOKENS` | `3000` | Reply cap in tokens, reasoning included. | +| `MOVERS_BOTS` | empty (feature off) | Usernames of bots whose end-of-day big-movers lists get explained. | +| `PORTFOLIO_NAMES` | the holding-news recipients' portfolio names | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | +| `PERIOD_BACKFILL` | `all` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. List names to limit it, or `off` to disable. See [Caching Sharesight history](caching.md#caching-sharesight-history). | +| `POST_TO_GROUPS_FROM_DM` | `off` | `on` or `off`. A group admin can have the bot post in that group from a DM. | +| `SEARCH_MODEL` | `xiaomi/mimo-v2.6-flash:online` | OpenRouter only: model that runs the searches. | +| `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | empty (daily DM off) | Comma list of portfolio:telegram_username pairs to notify. | +| `SHARESIGHT_HOLDING_NEWS_TIME` | `08:00` | HH:MM (in `TZ`) for the daily holding-news check. | +| `TELEGRAM_DM_BUTTONS` | `off` | `on` or `off`. Preset-prompt buttons in private chats. | +| `TELEGRAM_IMAGE_PRELOAD_CHATID` | empty (off) | Chat ID of a private chat the bot can post in (a private group it is in, or a DM with you). The other periods of a chart are uploaded there in the background and reused when a period button is tapped, so the tap swaps the picture without fetching or drawing. Empty = off, and a tap fetches and draws. See [Charts](features.md#charts). | ## Choosing a model Figures from search results and Artificial Analysis at the time of writing (October 2026), so diff --git a/lib/config.py b/lib/config.py index 5ef6991..1f578b8 100644 --- a/lib/config.py +++ b/lib/config.py @@ -24,7 +24,7 @@ "DARKMODE": "Chart theme: off (default, light), on (dark) or auto (dark from 9 pm to 7 am in TZ).", # Advanced and optional "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /tokens; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.", - "ADMIN_ONLY": "on|off. Only admins may DM the bot or get Sharesight data (default on).", + "ADMIN_ONLY": "on|off. On (default): only admins may DM the bot or get Sharesight data. Off: everyone may. /tokens and /flushcache stay admin-only.", "DATA_DIR": "Directory for the bot's data (the database); Default: systemd's STATE_DIRECTORY, else the working directory.", "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.", "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).", From cfd93f8c28dc343304881b5bc0dc87a449163508 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:11:30 +0000 Subject: [PATCH 09/30] OpenRouter default model: deepseek/deepseek-v4.1-flash-1017 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/configuration.md | 7 ++++--- lib/config.py | 4 ++-- tests/test_config.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2a0d316..065363b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,7 +7,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | Variable | Default | Meaning | |---|---|---| -| `OPENROUTER_API_KEY` | none | OpenRouter key; selects the OpenRouter provider (its default model is `z-ai/glm-5.3-flash`). | +| `OPENROUTER_API_KEY` | none | OpenRouter key; selects the OpenRouter provider (its default model is `deepseek/deepseek-v4.1-flash-1017`). | | `TELEGRAM_BOT_TOKEN` | none (required) | Bot token from @BotFather (required). | | `XAI_API_KEY` | none | xAI key; selects the xAI provider (its default model is `grok-4.3`). | @@ -15,7 +15,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | Variable | Default | Meaning | |---|---|---| | `TZ` | `UTC` | Time zone for timestamps, e.g. `Australia/Melbourne`. | -| `MODEL` | `grok-4.3` on xAI, `z-ai/glm-5.3-flash` on OpenRouter | Model ID. | +| `MODEL` | `grok-4.3` on xAI, `deepseek/deepseek-v4.1-flash-1017` on OpenRouter | Model ID. | | `REASONING` | the model's own | Reasoning effort: `low`, `medium` or `high`. | | `SEARCH` | `on` | `on` or `off`. Web search (and X search on xAI). | | `DARKMODE` | `off` | Chart theme: `off` draws light charts, `on` draws dark ones, `auto` draws dark ones from 9 pm to 7 am in `TZ`. Before this setting existed, charts were always `auto`. | @@ -46,7 +46,8 @@ treat them as rough; speed varies by provider. |---|---|---|---| | `xiaomi/mimo-v2.6-pro` | 0.43 / 0.87 | ~28-46 tok/s | 46 (top open-weight model) | | `xiaomi/mimo-v2.6-flash` | 0.14 / 0.28 | ~56 tok/s | 38 | -| `z-ai/glm-5.3-flash` (default on OpenRouter) | 0.15 / 0.50 | ~50 tok/s (other hosts up to ~270) | 57 | +| `deepseek/deepseek-v4.1-flash-1017` (default on OpenRouter) | not measured | not measured | not measured | +| `z-ai/glm-5.3-flash` (the previous default) | 0.15 / 0.50 | ~50 tok/s (other hosts up to ~270) | 57 | | `grok-4.3` / `x-ai/grok-4.3` (default on xAI) | 1.25 / 2.50 | ~105-146 tok/s | 25 (at high reasoning) | | `xiaomi/mimo-v2.5-pro` | 0.30 / 0.61 | ~29-46 tok/s | unreliable (retires 21 Oct 2026) | | `xiaomi/mimo-v2.5` | 0.12 / 0.24 | ~44-58 tok/s | not found (retires 21 Oct 2026) | diff --git a/lib/config.py b/lib/config.py index 1f578b8..b626816 100644 --- a/lib/config.py +++ b/lib/config.py @@ -18,7 +18,7 @@ "XAI_API_KEY": "xAI key; selects the xAI provider. Set exactly one of the three keys.", # Common settings "TZ": "Time zone for timestamps, e.g. Australia/Melbourne (default UTC).", - "MODEL": "Model ID (default grok-4.3 on xAI, z-ai/glm-5.3-flash on OpenRouter).", + "MODEL": "Model ID (default grok-4.3 on xAI, deepseek/deepseek-v4.1-flash-1017 on OpenRouter).", "REASONING": "Reasoning effort: low, medium or high; empty = the model's default.", "SEARCH": "on|off. Web search (and X search on xAI). Default on.", "DARKMODE": "Chart theme: off (default, light), on (dark) or auto (dark from 9 pm to 7 am in TZ).", @@ -39,7 +39,7 @@ "TELEGRAM_IMAGE_PRELOAD_CHATID": "Chat ID of a private chat the bot can post in. Chart periods are uploaded there ahead of time and reused when a period button is tapped. Empty (default) = off.", } -DEFAULT_MODELS = {"xai": "grok-4.3", "openrouter": "z-ai/glm-5.3-flash"} +DEFAULT_MODELS = {"xai": "grok-4.3", "openrouter": "deepseek/deepseek-v4.1-flash-1017"} # Tuning constants (not configurable). MAX_TOOL_ROUNDS = 6 # model <-> tool round trips per answer diff --git a/tests/test_config.py b/tests/test_config.py index 2f13911..436b7d6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -48,7 +48,7 @@ def test_provider_defaults(): x = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "XAI_API_KEY": "a"}) assert (x.provider, x.model, x.api_key) == ("xai", "grok-4.3", "a") o = config.load({"TELEGRAM_BOT_TOKEN": "1:x", "OPENROUTER_API_KEY": "b"}) - assert (o.provider, o.model) == ("openrouter", "z-ai/glm-5.3-flash") + assert (o.provider, o.model) == ("openrouter", "deepseek/deepseek-v4.1-flash-1017") def test_model_and_reasoning(env): From 1412dc115400934db89f68418f5b350a07e74ec4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:18:02 +0000 Subject: [PATCH 10/30] SEARCH_MODEL defaults to MODEL with :online added Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/configuration.md | 2 +- docs/how-it-works.md | 2 +- lib/config.py | 9 +++++++-- tests/test_config.py | 8 ++++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 065363b..9ce6adb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -32,7 +32,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | `PORTFOLIO_NAMES` | the holding-news recipients' portfolio names | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | | `PERIOD_BACKFILL` | `all` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. List names to limit it, or `off` to disable. See [Caching Sharesight history](caching.md#caching-sharesight-history). | | `POST_TO_GROUPS_FROM_DM` | `off` | `on` or `off`. A group admin can have the bot post in that group from a DM. | -| `SEARCH_MODEL` | `xiaomi/mimo-v2.6-flash:online` | OpenRouter only: model that runs the searches. | +| `SEARCH_MODEL` | `MODEL` with `:online` added | OpenRouter only: model that runs the searches (`:online` is what gives an OpenRouter model web search). Set but empty = no search model, so search is off. | | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | empty (daily DM off) | Comma list of portfolio:telegram_username pairs to notify. | | `SHARESIGHT_HOLDING_NEWS_TIME` | `08:00` | HH:MM (in `TZ`) for the daily holding-news check. | | `TELEGRAM_DM_BUTTONS` | `off` | `on` or `off`. Preset-prompt buttons in private chats. | diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 37325ad..99b83ef 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -41,7 +41,7 @@ ## Search - **xAI:** server-side `web_search` and `x_search` (X/Twitter) tools. - **OpenRouter:** OpenRouter's `web_search` tool; when it hands a search call back, the bot runs the - query through `SEARCH_MODEL` (default `xiaomi/mimo-v2.6-flash:online`) and returns the write-up; that call's cost is added to the answer's cost. + query through `SEARCH_MODEL` (default: `MODEL` with `:online` added) and returns the write-up; that call's cost is added to the answer's cost. Some models print a search call as text instead of making it; the bot recovers those queries too. The pages the search cited come back as `url_citation` annotations. In a chat answer each cited sentence gets a `[1]` after it (the number links to the page, the same page keeps its number); the text is marked before any other diff --git a/lib/config.py b/lib/config.py index b626816..1125bc3 100644 --- a/lib/config.py +++ b/lib/config.py @@ -32,7 +32,7 @@ "PORTFOLIO_NAMES": "Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names).", "PERIOD_BACKFILL": "Comma list of Sharesight portfolio names (or all) whose daily/weekly/monthly/quarterly/yearly returns are fetched into the local cache in the background, so period questions answer instantly. Default all; list names to limit it, or off to disable.", "POST_TO_GROUPS_FROM_DM": "on|off. Lets an admin of a group the bot is in make it post there from a DM (default off).", - "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default xiaomi/mimo-v2.6-flash:online).", + "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default: MODEL with :online added).", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.", "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (TZ) for the daily holding-news check (default 08:00).", "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).", @@ -151,6 +151,11 @@ def _backfill(text: str) -> frozenset[str]: return frozenset() if names & {"off", "none", "false", "0"} else names +def online(model: str) -> str: + """The model with OpenRouter's ":online" suffix, which gives it web search.""" + return model if model.endswith(":online") else f"{model}:online" + + def load(env: Mapping[str, str] | None = None) -> Settings: """Build Settings from `env` (default: os.environ). Raises ConfigError if unusable.""" env = os.environ if env is None else env @@ -167,7 +172,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings: raise ConfigError("Set XAI_API_KEY (xAI) or OPENROUTER_API_KEY (OpenRouter).") provider = given[0] model = env.get("MODEL", "").strip() or DEFAULT_MODELS[provider] - search_model = env.get("SEARCH_MODEL", "xiaomi/mimo-v2.6-flash:online").strip() + search_model = env.get("SEARCH_MODEL", online(model)).strip() # set but empty = no search model, so no search search = env.get("SEARCH", "on").strip().lower() != "off" if provider == "openrouter" and not search_model: search = False # nothing to run the searches with diff --git a/tests/test_config.py b/tests/test_config.py index 436b7d6..95bcc1e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -120,3 +120,11 @@ def test_darkmode_defaults_to_off_and_takes_on_and_auto_in_any_case(): assert [config.load({**base, "DARKMODE": v}).darkmode for v in ("on", "AUTO", " Off ", "")] == ["on", "auto", "off", "off"] with pytest.raises(config.ConfigError, match="DARKMODE must be off, on or auto"): config.load({**base, "DARKMODE": "night"}) + + +def test_the_search_model_defaults_to_the_model_with_online_added(env): + base = {"TELEGRAM_BOT_TOKEN": "1:x", "OPENROUTER_API_KEY": "b"} + assert config.load(base).search_model == "deepseek/deepseek-v4.1-flash-1017:online" + assert config.load({**base, "MODEL": "a/b"}).search_model == "a/b:online" + assert config.load({**base, "MODEL": "a/b:online"}).search_model == "a/b:online" # not added twice + assert config.load({**base, "MODEL": "a/b", "SEARCH_MODEL": "c/d:online"}).search_model == "c/d:online" From 07a06242ee4bdae66b508b8beb575b3a3255cb7c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:22:58 +0000 Subject: [PATCH 11/30] OpenRouter: debug-log what the citations carry and how many [n] markers were placed Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/openrouter.py | 10 +++++++++- tests/test_llm_backends.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/llm/openrouter.py b/lib/llm/openrouter.py index 0680459..f0698fa 100644 --- a/lib/llm/openrouter.py +++ b/lib/llm/openrouter.py @@ -114,6 +114,10 @@ def _read_delta(self, delta, extra: dict) -> None: fields = getattr(delta, "model_extra", None) or {} merge_reasoning(extra.setdefault("details", []), fields.get("reasoning_details") or []) extra["searches"] = extra.get("searches", 0) + len(fields.get("annotations") or []) + if fields.get("annotations"): + log.debug("OpenRouter citations: %s", [ + (a.get("type"), {k: (len(v) if k == "content" else v) for k, v in (a.get("url_citation") or {}).items()}) + if isinstance(a, dict) else repr(a) for a in fields["annotations"]]) for a in fields.get("annotations") or []: c = a.get("url_citation") if isinstance(a, dict) and a.get("type") == "url_citation" else None if c and isinstance(c.get("end_index"), int) and str(c.get("url") or "").startswith(("http://", "https://")): @@ -122,7 +126,11 @@ def _read_delta(self, delta, extra: dict) -> None: have += [c for c in citations(fields.get("annotations") or []) if c[1] not in {u for _, u in have}] def _cite(self, text: str, extra: dict) -> str: - return mark_citations(text, extra.get("cites", [])) + marked = mark_citations(text, extra.get("cites", [])) + log.debug("Citations: %d with a position, %d [n] markers placed in %d characters (%d sources)", + len(extra.get("cites", [])), marked.count("]") - text.count("]"), len(text), + len(extra.get("sources", []))) + return marked def _assistant_fields(self, extra: dict) -> dict: return {"reasoning_details": extra["details"]} if extra.get("details") else {} diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 9ef26aa..c8535dc 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -303,3 +303,17 @@ def chunks(): assert step.raw["content"] == "Micron guided up." # replayed to the model unmarked plain = req(search=True) assert (await b.step(b.start(plain), plain, tool_choice=None)).text == "Micron guided up." + + +async def test_openrouter_logs_what_its_citations_carry(env, caplog): + import logging + ann = {"type": "url_citation", "url_citation": {"url": "https://a.example/", "title": "A", "content": "x" * 500, + "start_index": 0, "end_index": 0}} + b = OpenRouterBackend(config.load(env), FakeChatClient(Stream([ + chat_chunk("Micron up.", extra={"annotations": [ann]}), chat_chunk(finish="stop")]))) + r = req(search=True, cite=True) + with caplog.at_level(logging.DEBUG, logger="bot"): + step = await b.step(b.start(r), r, tool_choice=None) + text = caplog.text + assert "'end_index': 0" in text and "'content': 500" in text and "x" * 50 not in text # the snippet's length only + assert "0 [n] markers placed" in text and step.sources == [("A", "https://a.example/")] From f23ca7f475b6991eb50fe90543ed873dbc9c72aa Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:26:41 +0000 Subject: [PATCH 12/30] Cite web-search sources as [n] after the sentence each snippet fits best OpenRouter's url_citation positions are 0 and the search runs in an earlier round than the answer, so the position-based marker is replaced by matching the answer's sentences to each source's snippet (figures weigh most, words every snippet shares count for less). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/how-it-works.md | 10 +++-- lib/llm/base.py | 16 +++++-- lib/llm/chat.py | 7 +--- lib/llm/cite.py | 86 ++++++++++++++++++++++++++++++++++++++ lib/llm/openrouter.py | 52 ++++------------------- lib/llm/runner.py | 13 +++--- lib/textfmt.py | 6 +-- tests/test_cite.py | 43 +++++++++++++++++++ tests/test_handlers.py | 3 +- tests/test_llm_backends.py | 65 ++++++---------------------- tests/test_runner.py | 15 +++---- 11 files changed, 187 insertions(+), 129 deletions(-) create mode 100644 lib/llm/cite.py create mode 100644 tests/test_cite.py diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 99b83ef..d34f96e 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -43,10 +43,12 @@ - **OpenRouter:** OpenRouter's `web_search` tool; when it hands a search call back, the bot runs the query through `SEARCH_MODEL` (default: `MODEL` with `:online` added) and returns the write-up; that call's cost is added to the answer's cost. Some models print a search call as text instead of making it; the bot recovers those queries too. - The pages the search cited come back as `url_citation` annotations. In a chat answer each cited sentence gets a - `[1]` after it (the number links to the page, the same page keeps its number); the text is marked before any other - formatting, because the positions are in the raw reply. If no marker could be placed and the answer links nothing - itself, it ends with a `Sources:` line of up to 3 links instead. + The pages the search cited come back as `url_citation` annotations, each with the snippet of the page the model was + given; OpenRouter does not say which sentence a page backs (the positions are 0, and the search is often in an + earlier round than the answer). So in a chat answer the bot puts `[1]` after the sentence each snippet fits best + (the number links to the page): the figures and distinctive words of the sentence also appear in the snippet, and a + sentence that fits no snippet, such as an opinion, gets none. If no snippet fits anything and the answer links + nothing itself, it ends with a `Sources:` line of up to 3 links instead. `DEBUG` logs show each source's best score. - At most 5 searches per round; extras get a "skipped" note. - `SEARCH=off` turns search off (the prompt then says the model has no web search). - **Forced search:** movers lists, DM preset buttons and the holding-news digest *require* a diff --git a/lib/llm/base.py b/lib/llm/base.py index dc530d0..ef5c3ca 100644 --- a/lib/llm/base.py +++ b/lib/llm/base.py @@ -4,7 +4,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, NamedTuple from ..mcp.schema import ToolDef @@ -48,6 +48,14 @@ class Call: arguments: str +class Source(NamedTuple): + """A web page a search used: its title, URL and the snippet of it the model was given.""" + + title: str + url: str + content: str = "" + + @dataclass class Step: """One model response.""" @@ -58,7 +66,7 @@ class Step: finish: str | None = None searches: int = 0 # server-side searches the provider ran for this step raw: Any = None # provider payload add_results() needs to replay this step - sources: list[tuple[str, str]] = field(default_factory=list) # (title, url) the search cited + sources: list[Source] = field(default_factory=list) # the pages the search cited @dataclass @@ -79,7 +87,7 @@ class Request: no_tools_system: str = "" # system prompt for the retry without tools local: dict[str, Callable[[dict], Awaitable[str]]] = field(default_factory=dict) # tools run in the bot max_tokens: int | None = None # reply cap incl. reasoning; None = the provider's setting - cite: bool = False # mark cited sentences with [n] links to their sources, where the provider says which + cite: bool = False # put [n] links to the pages a search used after the sentences they back @dataclass @@ -88,7 +96,7 @@ class Answer: usage: Usage rounds: int tool_calls: int - sources: list[tuple[str, str]] = field(default_factory=list) # (title, url) the search cited + sources: list[Source] = field(default_factory=list) # the pages the search cited class Backend: diff --git a/lib/llm/chat.py b/lib/llm/chat.py index f6bacf2..6bed859 100644 --- a/lib/llm/chat.py +++ b/lib/llm/chat.py @@ -46,10 +46,6 @@ def _kwargs(self, req: Request, tool_choice: str | None) -> dict: def _read_delta(self, delta, extra: dict) -> None: """Pick provider-specific fields out of a streamed delta.""" - def _cite(self, text: str, extra: dict) -> str: - """The reply with [n] source links after the sentences the provider says it cited.""" - return text - def _assistant_fields(self, extra: dict) -> dict: """Extra fields the next request must carry on this step's assistant message.""" return {} @@ -100,8 +96,7 @@ async def step(self, conv: dict, req: Request, *, tool_choice: str | None) -> St tokens_out=getattr(u, "completion_tokens", 0) or 0, cost=self._cost(u, req.model), ) - shown = self._cite(text, extra) if req.cite else text # the model's own turn is replayed unmarked - return Step(shown.strip(), calls, usage, finish, extra.get("searches", 0), raw=assistant, + return Step(text.strip(), calls, usage, finish, extra.get("searches", 0), raw=assistant, sources=extra.get("sources", [])) def add_results(self, conv: dict, step: Step, results: list[str]) -> None: diff --git a/lib/llm/cite.py b/lib/llm/cite.py new file mode 100644 index 0000000..9403d01 --- /dev/null +++ b/lib/llm/cite.py @@ -0,0 +1,86 @@ +"""Putting [n] source links after the sentences of an answer that a web search backed.""" + +from __future__ import annotations + +import html +import logging +import re + +from .base import Source + +log = logging.getLogger("bot") + +NUMBER_RE = re.compile(r"\d[\d.,]*\d|\d") +WORD_RE = re.compile(r"[^\W\d_]{6,}") +SENTENCE_END_RE = re.compile(r"(?<=[.!?])(?=\s)|\n+") +NUMBER_WEIGHT = 3.0 # a figure shared with a source says more than a word does +MIN_SCORE = 4.0 # below this a sentence is not taken to come from the source +MARKER_RE = re.compile(r'\[\d+\]') + + +def mark_citations(text: str, cites: list[tuple[int, str]]) -> str: + """Put "[n]" after each cited sentence, n a link to the page (the same page keeps its number). `cites` are + (end position in `text`, url); a position inside a tag or a link is skipped, and one right before the + sentence's closing punctuation moves past it.""" + numbers: dict[str, int] = {} + at: dict[int, list[int]] = {} + for end, url in sorted(cites): + pos = min(end, len(text)) + while pos < len(text) and text[pos] in ".!?": + pos += 1 + before = text[:pos] + if (pos <= 0 or before.rfind("<") > before.rfind(">") or before.count("") + or before.rfind("](") > before.rfind(")")): + continue + n = numbers.setdefault(url, len(numbers) + 1) + if n not in at.setdefault(pos, []): + at[pos].append(n) + urls = {n: url for url, n in numbers.items()} + for pos in sorted(at, reverse=True): + marks = "".join(f'[{n}]' for n in at[pos]) + text = text[:pos] + marks + text[pos:] + return text + + +def _tokens(text: str) -> tuple[set[str], set[str]]: + plain = re.sub(r"<[^>]+>", " ", text) + return ({n.replace(",", "") for n in NUMBER_RE.findall(plain) if len(n) > 1}, + {w.lower() for w in WORD_RE.findall(plain)}) + + +def mark_by_overlap(text: str, sources: list[Source]) -> str: + """Put "[n]" after the sentence of `text` that each source's snippet fits best, when it fits well enough. + + OpenRouter says which pages a search used but not which sentence each backs (its positions are 0), so the + link goes where the figures and the distinctive words of the sentence are also in the page's snippet. + Words every snippet shares count for less. A source that fits no sentence gets no marker.""" + if MARKER_RE.search(text) or not sources: + return text + snippets = [_tokens(s.content) for s in sources if s.content] + if not snippets: + return text + spread: dict[str, int] = {} + for _, words in snippets: + for w in words: + spread[w] = spread.get(w, 0) + 1 + ends, start = [], 0 + for m in SENTENCE_END_RE.finditer(text): + ends.append((start, m.start())) + start = m.end() + ends.append((start, len(text))) + cites = [] + for source in sources: + numbers, words = _tokens(source.content) + best, best_end = 0.0, None + for lo, hi in ends: + sentence_numbers, sentence_words = _tokens(text[lo:hi]) + score = NUMBER_WEIGHT * len(sentence_numbers & numbers) + score += sum(1 - 0.6 * (spread[w] - 1) / max(1, len(snippets) - 1) for w in sentence_words & words) + if score > best: + best, best_end = score, hi + if best_end is not None and best >= MIN_SCORE: + cites.append((best_end, source.url)) + log.debug("Citation %s fits the sentence ending at %d (score %.1f)", source.url, best_end, best) + else: + log.debug("Citation %s fits no sentence (best score %.1f)", source.url, best) + return mark_citations(text, cites) diff --git a/lib/llm/openrouter.py b/lib/llm/openrouter.py index f0698fa..00d7ea9 100644 --- a/lib/llm/openrouter.py +++ b/lib/llm/openrouter.py @@ -2,7 +2,6 @@ from __future__ import annotations -import html import logging import httpx @@ -10,7 +9,7 @@ from .. import config from ..config import Settings -from .base import Request, Usage +from .base import Request, Source, Usage from .chat import ChatBackend log = logging.getLogger("bot") @@ -23,41 +22,17 @@ ) -def citations(annotations: list) -> list[tuple[str, str]]: - """(title, url) of each web page a message cites, in order, http(s) only, one per URL.""" - out: dict[str, str] = {} +def citations(annotations: list) -> list[Source]: + """The web pages a message cites, in order, http(s) only, one per URL.""" + out: dict[str, Source] = {} for a in annotations or []: if not isinstance(a, dict) or a.get("type") != "url_citation": continue c = a.get("url_citation") or {} url = str(c.get("url") or "") if url.startswith(("http://", "https://")): - out.setdefault(url, str(c.get("title") or "").strip()) - return [(title, url) for url, title in out.items()] - - -def mark_citations(text: str, cites: list[tuple[int, str]]) -> str: - """Put "[n]" after each cited sentence, n a link to the page (the same page keeps its number). `cites` are - (end position in `text`, url); a position inside a tag or a link is skipped, and one right before the - sentence's closing punctuation moves past it.""" - numbers: dict[str, int] = {} - at: dict[int, list[int]] = {} - for end, url in sorted(cites): - pos = min(end, len(text)) - while pos < len(text) and text[pos] in ".!?": - pos += 1 - before = text[:pos] - if (pos <= 0 or before.rfind("<") > before.rfind(">") or before.count("") - or before.rfind("](") > before.rfind(")")): - continue - n = numbers.setdefault(url, len(numbers) + 1) - if n not in at.setdefault(pos, []): - at[pos].append(n) - urls = {n: url for url, n in numbers.items()} - for pos in sorted(at, reverse=True): - marks = "".join(f'[{n}]' for n in at[pos]) - text = text[:pos] + marks + text[pos:] - return text + out.setdefault(url, Source(str(c.get("title") or "").strip(), url, str(c.get("content") or ""))) + return list(out.values()) def merge_reasoning(details: list[dict], new: list) -> None: @@ -118,19 +93,8 @@ def _read_delta(self, delta, extra: dict) -> None: log.debug("OpenRouter citations: %s", [ (a.get("type"), {k: (len(v) if k == "content" else v) for k, v in (a.get("url_citation") or {}).items()}) if isinstance(a, dict) else repr(a) for a in fields["annotations"]]) - for a in fields.get("annotations") or []: - c = a.get("url_citation") if isinstance(a, dict) and a.get("type") == "url_citation" else None - if c and isinstance(c.get("end_index"), int) and str(c.get("url") or "").startswith(("http://", "https://")): - extra.setdefault("cites", []).append((c["end_index"], c["url"])) have = extra.setdefault("sources", []) - have += [c for c in citations(fields.get("annotations") or []) if c[1] not in {u for _, u in have}] - - def _cite(self, text: str, extra: dict) -> str: - marked = mark_citations(text, extra.get("cites", [])) - log.debug("Citations: %d with a position, %d [n] markers placed in %d characters (%d sources)", - len(extra.get("cites", [])), marked.count("]") - text.count("]"), len(text), - len(extra.get("sources", []))) - return marked + have += [c for c in citations(fields.get("annotations") or []) if c.url not in {h.url for h in have}] def _assistant_fields(self, extra: dict) -> dict: return {"reasoning_details": extra["details"]} if extra.get("details") else {} @@ -155,7 +119,7 @@ async def run_search(self, query: str, usage: Usage, args: dict) -> str | None: return "The search returned no text (the search model may have run out of tokens)." links = citations(getattr(resp.choices[0].message, "annotations", None) or []) if links: # the model only sees this text: hand it the URLs so it can cite them - text += "\nLinks: " + " ".join(url for _, url in links[:config.SOURCES_MAX]) + text += "\nLinks: " + " ".join(link.url for link in links[:config.SOURCES_MAX]) return text async def credits(self) -> str | None: diff --git a/lib/llm/runner.py b/lib/llm/runner.py index 5c8f86b..88250ce 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -14,7 +14,8 @@ from .. import config from ..mcp.server import Registry from ..textfmt import TOOL_SYNTAX_RE -from .base import Answer, Backend, Call, Request, Step, Usage +from .base import Answer, Backend, Call, Request, Source, Step, Usage +from .cite import mark_by_overlap log = logging.getLogger("bot") @@ -177,9 +178,9 @@ async def _recover_fake_calls(step: Step, backend: Backend, usage: Usage) -> str return "\n\n".join(parts) + "\n\nAnswer the question now using these results." -def _cited(have: list[tuple[str, str]], new: list[tuple[str, str]]) -> list[tuple[str, str]]: - """The sources of the latest step that cited any (the ones behind the final answer), else what was gathered.""" - return new[:] if new else have +def _cited(have: list[Source], new: list[Source]) -> list[Source]: + """The pages cited so far plus those a step adds (the search is often in an earlier round than the answer).""" + return have + [s for s in new if s.url not in {h.url for h in have}] async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: bool = True) -> Answer: @@ -188,7 +189,7 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo total, rounds, tool_calls = Usage(), 0, 0 seen: dict[tuple[str, str], int] = {} searched = 0 - cited: list[tuple[str, str]] = [] + cited: list[Source] = [] while True: final = rounds >= config.MAX_TOOL_ROUNDS choice = "none" if final else ("required" if (req.must_search or req.must_call) and rounds == 0 else None) @@ -235,4 +236,6 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo if step.finish == "length": raise RuntimeError("hit the token cap before writing an answer") raise RuntimeError(f"empty reply (finish_reason={step.finish})") + if req.cite and cited: + text = mark_by_overlap(text, cited) return Answer(text, total, rounds, tool_calls, cited) diff --git a/lib/textfmt.py b/lib/textfmt.py index ca79df2..711e1c2 100644 --- a/lib/textfmt.py +++ b/lib/textfmt.py @@ -51,8 +51,8 @@ def link(m: re.Match) -> str: return MD_CODE.sub(r"\1", text) -def with_sources(text: str, sources: list[tuple[str, str]], limit: int = SOURCES_MAX) -> str: - """An answer with a "Sources:" line of links when it has none of its own.""" +def with_sources(text: str, sources: list, limit: int = SOURCES_MAX) -> str: + """An answer with a "Sources:" line of links (sources are (title, url, ...)) when it has none of its own.""" if not sources or " str: return html.escape(name if len(name) <= 40 else name[:39].rstrip() + "…") links = " · ".join(f'{label(title, url)}' - for title, url in sources[:limit]) + for title, url, *_ in sources[:limit]) return f"{text}\n\nSources: {links}" diff --git a/tests/test_cite.py b/tests/test_cite.py new file mode 100644 index 0000000..0c6d903 --- /dev/null +++ b/tests/test_cite.py @@ -0,0 +1,43 @@ +from lib.llm.base import Source +from lib.llm.cite import mark_by_overlap, mark_citations + +A, B = "https://a.example/x?p=1&q=2", "https://b.example/" +LINK_A = '[1]' +LINK_B = '[2]' + + +def test_citations_become_numbered_links_after_the_sentence(): + text = "NVDA rose 3%. Micron guided up. Both beat." + out = mark_citations(text, [(12, A), (30, B), (41, A)]) + assert out == f"NVDA rose 3%.{LINK_A} Micron guided up.{LINK_B} Both beat.{LINK_A}" + assert mark_citations("One.", [(3, A), (3, B)]) == f"One.{LINK_A}{LINK_B.replace('2', '2')}" + assert mark_citations('See x.', [(10, A)]) == 'See x.' + assert mark_citations("Hi.", [(99, A)]).startswith("Hi.[") and mark_citations("Hi.", [(0, A)]) == "Hi." + + +PRESS = ("AMD reported third quarter revenue of $13.38 billion, up 45% year over year, with Data Center revenue of " + "$8.5 billion and non-GAAP earnings per share of $2.09. Gross margin was 56%.") +NEWS = ("AMD shares fell after hours although analysts at Morgan Stanley raised their price target to $550 and " + "JPMorgan raised its target to $575, staying neutral on the stock.") +TEXT = ("AMD's quarter: revenue $13.38b, +45% y/y, EPS $2.09. Data Center did the work, $8.5b. " + "Street is bullish, with JPMorgan and Morgan Stanley raising targets to $575 and $550 while staying neutral. " + "My read: the multiple is high.") + + +def test_each_source_is_cited_after_the_sentence_it_fits_best(): + out = mark_by_overlap(TEXT, [Source("Release", A, PRESS), Source("News", B, NEWS)]) + assert "EPS $2.09." + LINK_A in out and "$550 while staying neutral." + LINK_B in out + assert out.endswith("My read: the multiple is high.") # an opinion fits no source: no marker + + +def test_numbering_follows_the_order_in_the_text_and_a_source_that_fits_nothing_gets_no_marker(): + out = mark_by_overlap(TEXT, [Source("News", B, NEWS), Source("Release", A, PRESS), + Source("Other", "https://c.example/", "Completely unrelated gardening advice.")]) + assert "EPS $2.09.[1]" in out and 'c.example' not in out + assert out.count("]") == 2 + + +def test_nothing_is_marked_without_snippets_or_when_markers_exist_already(): + assert mark_by_overlap(TEXT, [Source("T", A)]) == TEXT and mark_by_overlap(TEXT, []) == TEXT + marked = mark_by_overlap(TEXT, [Source("Release", A, PRESS)]) + assert mark_by_overlap(marked, [Source("News", B, NEWS)]) == marked diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 833c137..201b743 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -26,8 +26,9 @@ async def run(h, msg, edited=False): async def test_a_chat_answer_that_used_web_search_ends_with_its_sources(env, store): + from lib.llm.base import Source cited = step("Micron guided higher.") - cited.sources = [("Reuters", "https://www.reuters.com/mu")] + cited.sources = [Source("Reuters", "https://www.reuters.com/mu")] ctx, backend, h = make_ctx(env, store, [cited]) m = user_msg(ctx.bot, "@stockbot what did Micron say?") await run(h, m) diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index c8535dc..98d5e04 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -246,18 +246,24 @@ async def test_xai_failed_stream_raises(): await b.step(b.start(r), r, tool_choice=None) -def cite(url, title=""): - return {"type": "url_citation", "url_citation": {"url": url, "title": title}} +def cite(url, title="", content=""): + return {"type": "url_citation", "url_citation": {"url": url, "title": title, "content": content, + "start_index": 0, "end_index": 0}} -async def test_openrouter_collects_web_search_citations(env): - chunks = [chat_chunk("a", extra={"annotations": [cite("https://a.example/x", "A"), cite("ftp://no", "N")]}), +async def test_openrouter_collects_web_search_citations_with_their_snippets(env, caplog): + import logging + + from lib.llm.base import Source + chunks = [chat_chunk("a", extra={"annotations": [cite("https://a.example/x", "A", "x" * 500), cite("ftp://no", "N")]}), chat_chunk("b", extra={"annotations": [cite("https://a.example/x", "A"), cite("https://b.example/y")]}), chat_chunk(finish="stop")] b = OpenRouterBackend(config.load(env), FakeChatClient(Stream(chunks))) r = req(search=True) - step = await b.step(b.start(r), r, tool_choice=None) - assert step.sources == [("A", "https://a.example/x"), ("", "https://b.example/y")] + with caplog.at_level(logging.DEBUG, logger="bot"): + step = await b.step(b.start(r), r, tool_choice=None) + assert step.sources == [Source("A", "https://a.example/x", "x" * 500), Source("", "https://b.example/y")] + assert "'content': 500" in caplog.text and "x" * 50 not in caplog.text # the log has the snippet's length only async def test_openrouter_run_search_hands_the_model_the_links(env): @@ -270,50 +276,3 @@ async def create(**kw): b = OpenRouterBackend(config.load(env), N(chat=N(completions=N(create=create)))) assert await b.run_search("nvda", Usage(), {}) == "- NVDA 100 [Reuters]\nLinks: https://r.example/n" - - -def cite_at(url, end): - return {"type": "url_citation", "url_citation": {"url": url, "title": "t", "start_index": 0, "end_index": end}} - - -def test_citations_become_numbered_links_after_the_sentence(): - from lib.llm.openrouter import mark_citations - a, b = "https://a.example/x?p=1&q=2", "https://b.example/" - text = "NVDA rose 3%. Micron guided up. Both beat." - out = mark_citations(text, [(12, a), (30, b), (41, a)]) - assert out == ('NVDA rose 3%.[1] Micron guided up.' - '[2] Both beat.[1]') - # two sources for one sentence; a spot inside a tag or a link, or past the end, is handled - assert mark_citations("One.", [(3, a), (3, b)]).endswith('.[1]' - '[2]') - assert mark_citations('See x.', [(10, a)]) == 'See x.' - assert mark_citations("Hi.", [(99, a)]).startswith("Hi.[") - assert mark_citations("Hi.", [(0, a)]) == "Hi." - - -async def test_openrouter_marks_cited_sentences_only_when_asked(env): - def chunks(): - return Stream([chat_chunk("Micron guided up.", extra={"annotations": [cite_at("https://m.example/", 16)]}), - chat_chunk(finish="stop")]) - - b = OpenRouterBackend(config.load(env), FakeChatClient(chunks(), chunks())) - r = req(search=True, cite=True) - step = await b.step(b.start(r), r, tool_choice=None) - assert step.text == 'Micron guided up.[1]' - assert step.raw["content"] == "Micron guided up." # replayed to the model unmarked - plain = req(search=True) - assert (await b.step(b.start(plain), plain, tool_choice=None)).text == "Micron guided up." - - -async def test_openrouter_logs_what_its_citations_carry(env, caplog): - import logging - ann = {"type": "url_citation", "url_citation": {"url": "https://a.example/", "title": "A", "content": "x" * 500, - "start_index": 0, "end_index": 0}} - b = OpenRouterBackend(config.load(env), FakeChatClient(Stream([ - chat_chunk("Micron up.", extra={"annotations": [ann]}), chat_chunk(finish="stop")]))) - r = req(search=True, cite=True) - with caplog.at_level(logging.DEBUG, logger="bot"): - step = await b.step(b.start(r), r, tool_choice=None) - text = caplog.text - assert "'end_index': 0" in text and "'content': 500" in text and "x" * 50 not in text # the snippet's length only - assert "0 [n] markers placed" in text and step.sources == [("A", "https://a.example/")] diff --git a/tests/test_runner.py b/tests/test_runner.py index 2b6d6ec..62905db 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -3,7 +3,7 @@ from openai import APIStatusError, BadRequestError from lib import config -from lib.llm.base import Overloaded, Usage +from lib.llm.base import Overloaded, Source, Usage from lib.llm.gate import route, wants_tools from lib.llm.policy import ask from lib.llm.runner import call_query, run, run_calls @@ -312,15 +312,12 @@ async def test_arguments_that_are_not_json_are_a_warning_and_a_message_for_the_m assert [r.levelname for r in caplog.records] == ["WARNING"] and not caplog.records[0].exc_info -async def test_the_sources_of_the_last_step_that_cited_any_land_on_the_answer(): +async def test_the_sources_of_every_round_land_on_the_answer_without_repeats(): m = FakeMcp() first = step("", [("c1", "yahoo__q", "{}")]) - first.sources = [("Old", "https://old.example/")] + first.sources = [Source("Old", "https://old.example/", "snippet")] last = step("Answer.") - last.sources = [("New", "https://new.example/")] + last.sources = [Source("Old", "https://old.example/"), Source("New", "https://new.example/")] answer = await run(ScriptedBackend([first, last]), registry(m), with_tools(m)) - assert answer.sources == [("New", "https://new.example/")] - first = step("", [("c1", "yahoo__q", "{}")]) - first.sources = [("Old", "https://old.example/")] - answer = await run(ScriptedBackend([first, step("Answer.")]), registry(m), with_tools(m)) - assert answer.sources == [("Old", "https://old.example/")] + assert [s.url for s in answer.sources] == ["https://old.example/", "https://new.example/"] + assert answer.sources[0].content == "snippet" From 4806c05d07b320e3d56e75d902e3e12b2b6d4458 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:29:18 +0000 Subject: [PATCH 13/30] Ask the model for inline [n] source links; drop the Sources list The chat system prompt (OpenRouter only) tells the model to cite a sentence that rests on a search result as [n]. Its links are kept only for pages the search returned and renumbered; without them the snippet-overlap markers apply. The trailing Sources line is gone. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/how-it-works.md | 12 ++++++------ lib/ask.py | 14 ++++++++------ lib/config.py | 2 +- lib/llm/base.py | 1 + lib/llm/cite.py | 18 +++++++++++++++++- lib/llm/openrouter.py | 2 ++ lib/llm/runner.py | 6 +++--- lib/prompts.py | 8 ++++---- lib/textfmt.py | 17 +---------------- tests/test_budgets.py | 7 +++++++ tests/test_cite.py | 13 +++++++++++++ tests/test_handlers.py | 21 +++++++++++---------- tests/test_textfmt.py | 11 ----------- 13 files changed, 74 insertions(+), 58 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index d34f96e..e306eac 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -43,12 +43,12 @@ - **OpenRouter:** OpenRouter's `web_search` tool; when it hands a search call back, the bot runs the query through `SEARCH_MODEL` (default: `MODEL` with `:online` added) and returns the write-up; that call's cost is added to the answer's cost. Some models print a search call as text instead of making it; the bot recovers those queries too. - The pages the search cited come back as `url_citation` annotations, each with the snippet of the page the model was - given; OpenRouter does not say which sentence a page backs (the positions are 0, and the search is often in an - earlier round than the answer). So in a chat answer the bot puts `[1]` after the sentence each snippet fits best - (the number links to the page): the figures and distinctive words of the sentence also appear in the snippet, and a - sentence that fits no snippet, such as an opinion, gets none. If no snippet fits anything and the answer links - nothing itself, it ends with a `Sources:` line of up to 3 links instead. `DEBUG` logs show each source's best score. + In a chat answer the model is told to cite a sentence that rests on a search result as `[1]`, the number linking to the + page (its exact URL from the results). The bot keeps only links to pages the search really returned and renumbers + them in order. OpenRouter's `url_citation` annotations say which pages were used but not which sentence each backs + (the positions are 0), so if the model gave no link, the bot puts `[n]` after the sentence each snippet fits best: the + figures and distinctive words of the sentence also appear in the snippet. A sentence that fits none, such as an + opinion, gets none; there is no list of sources at the end. `DEBUG` logs show each source's best score. - At most 5 searches per round; extras get a "skipped" note. - `SEARCH=off` turns search off (the prompt then says the model has no web search). - **Forced search:** movers lists, DM preset buttons and the holding-news digest *require* a diff --git a/lib/ask.py b/lib/ask.py index ed1003d..671d433 100644 --- a/lib/ask.py +++ b/lib/ask.py @@ -11,8 +11,7 @@ from .llm import policy from .llm.base import Answer, Request from .llm.gate import NEEDS_TOOLS, Route, wants_tools -from .prompts import no_tools_system, system_prompt -from .textfmt import with_sources +from .prompts import SOURCE_NOTE, no_tools_system, system_prompt log = logging.getLogger("bot") @@ -38,6 +37,11 @@ def forward(text: str) -> None: return forward +def _source_note(ctx: Ctx, kind: str) -> str: + """How to cite search results: only in chat answers, and only where the provider gives the model their URLs.""" + return (ctx.backend.source_note or SOURCE_NOTE) if kind == "chat" else SOURCE_NOTE + + def _public(servers: list) -> list: """The MCP servers anyone may use: not the ones gated on portfolio questions (Sharesight).""" return [s for s in servers if s.cfg.get("gate") != "portfolio"] @@ -60,7 +64,7 @@ async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = elif not admin: route = dataclasses.replace(route, servers=_public(route.servers)) req = Request( - system=system_prompt(ctx.first_name, route, ctx.backend.search_what), + system=system_prompt(ctx.first_name, route, ctx.backend.search_what, _source_note(ctx, kind)), parts=parts, model=st.model, tools=[t for s in route.servers for t in s.tools if only_tools is None or t.name in only_tools] @@ -79,14 +83,12 @@ async def ask(ctx: Ctx, parts: list[dict], route: Route, *, must_search: bool = log.info("Gate offered fewer tools, the model asked for more: asking again with everything on") full = Route(ctx.registry.up() if admin else _public(ctx.registry.up()), st.search) req = dataclasses.replace( - req, system=system_prompt(ctx.first_name, full, ctx.backend.search_what), + req, system=system_prompt(ctx.first_name, full, ctx.backend.search_what, _source_note(ctx, kind)), tools=[t for s in full.servers for t in s.tools] + [t for t, _ in extra_tools], search=full.search, on_text=on_text) answer = await policy.ask(ctx.backend, ctx.registry, req) await _record(ctx, req.model, kind, chat_id, answer) - if kind == "chat": # other kinds (holding news, filings) parse or format their own text - answer.text = with_sources(answer.text, answer.sources) return answer diff --git a/lib/config.py b/lib/config.py index 1125bc3..522c999 100644 --- a/lib/config.py +++ b/lib/config.py @@ -56,7 +56,7 @@ SEARCH_RESULTS = 20 SEARCH_TOKENS = 400 # cap on each search write-up (OpenRouter search runner) MAX_SEARCHES = 5 # search calls run per round -SOURCES_MAX = 3 # links in the "Sources:" line added to an answer that cites none +SOURCES_MAX = 3 # links to a search result the model is handed back as text (OpenRouter run_search) MAX_TG_MESSAGE = 4000 # Telegram's limit is 4096 HOLDING_NEWS_RETRY_EVERY = 30 * 60 # seconds between retries of a holding-news check the provider was too busy for HOLDING_NEWS_RETRY_FOR = 12 * 3600 # seconds after which it gives up diff --git a/lib/llm/base.py b/lib/llm/base.py index ef5c3ca..56ef86a 100644 --- a/lib/llm/base.py +++ b/lib/llm/base.py @@ -104,6 +104,7 @@ class Backend: name = "" search_what = "" # how prompts name the search ("the web and X (Twitter)") + source_note = "" # prompt text on how to cite search results, where the provider's results carry URLs def start(self, req: Request) -> Any: """The opaque conversation state for a new request.""" diff --git a/lib/llm/cite.py b/lib/llm/cite.py index 9403d01..4421b05 100644 --- a/lib/llm/cite.py +++ b/lib/llm/cite.py @@ -15,7 +15,7 @@ SENTENCE_END_RE = re.compile(r"(?<=[.!?])(?=\s)|\n+") NUMBER_WEIGHT = 3.0 # a figure shared with a source says more than a word does MIN_SCORE = 4.0 # below this a sentence is not taken to come from the source -MARKER_RE = re.compile(r'\[\d+\]') +MARKER_RE = re.compile(r'\[\d+\]') def mark_citations(text: str, cites: list[tuple[int, str]]) -> str: @@ -42,6 +42,22 @@ def mark_citations(text: str, cites: list[tuple[int, str]]) -> str: return text +def normalise_markers(text: str, sources: list[Source]) -> str: + """The model's own [n] links kept only where they point at a page the search really returned, numbered 1, 2, ... + in the order they appear (a link to a page it made up is dropped, marker and all).""" + known = {s.url.rstrip("/"): s.url for s in sources} + numbers: dict[str, int] = {} + + def fix(m: re.Match) -> str: + url = known.get(html.unescape(m.group(1)).rstrip("/")) + if url is None: + return "" + n = numbers.setdefault(url, len(numbers) + 1) + return f'[{n}]' + + return MARKER_RE.sub(fix, text) + + def _tokens(text: str) -> tuple[set[str], set[str]]: plain = re.sub(r"<[^>]+>", " ", text) return ({n.replace(",", "") for n in NUMBER_RE.findall(plain) if len(n) > 1}, diff --git a/lib/llm/openrouter.py b/lib/llm/openrouter.py index 00d7ea9..533ca26 100644 --- a/lib/llm/openrouter.py +++ b/lib/llm/openrouter.py @@ -52,6 +52,8 @@ def merge_reasoning(details: list[dict], new: list) -> None: class OpenRouterBackend(ChatBackend): name = "openrouter" search_what = "the web" + source_note = ("After a sentence that rests on a search result, cite it as [1], numbering the pages " + "you cite 1, 2, ... and using the page's exact URL from the results, never one you didn't get.") def __init__(self, st: Settings, client: AsyncOpenAI | None = None): super().__init__(st, client or AsyncOpenAI( diff --git a/lib/llm/runner.py b/lib/llm/runner.py index 88250ce..4c5d239 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -15,7 +15,7 @@ from ..mcp.server import Registry from ..textfmt import TOOL_SYNTAX_RE from .base import Answer, Backend, Call, Request, Source, Step, Usage -from .cite import mark_by_overlap +from .cite import mark_by_overlap, normalise_markers log = logging.getLogger("bot") @@ -236,6 +236,6 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo if step.finish == "length": raise RuntimeError("hit the token cap before writing an answer") raise RuntimeError(f"empty reply (finish_reason={step.finish})") - if req.cite and cited: - text = mark_by_overlap(text, cited) + if req.cite: + text = mark_by_overlap(normalise_markers(text, cited), cited) return Answer(text, total, rounds, tool_calls, cited) diff --git a/lib/prompts.py b/lib/prompts.py index 99204ee..0cd5234 100644 --- a/lib/prompts.py +++ b/lib/prompts.py @@ -19,9 +19,9 @@ REFRESH = (" Earlier replies of yours may be incomplete or out of date: don't imitate them, fetch " "fresh data instead of reusing their numbers, and follow these formatting rules.") +SOURCE_NOTE = "Mention a source briefly if you like, one link at most." SEARCH_NOTE = ("You can search {what}: use it for news, prices, current events and what " - "people are saying instead of claiming you lack live data. Mention a source " - "briefly if you like, one link at most.\n") + "people are saying instead of claiming you lack live data. {source}\n") NO_SEARCH_NOTE = "You have no web search: say plainly what you can't check instead of guessing.\n" SIMPLE_NOTE = ("You have no live data tools for this message. If answering needs live data " f"or news, reply with exactly {NEEDS_TOOLS} and nothing else. Never claim " @@ -44,11 +44,11 @@ def tools_note(servers: list[MCPServer]) -> str: "which and quote the error briefly.\n") -def system_prompt(bot_name: str, route: Route, search_what: str) -> str: +def system_prompt(bot_name: str, route: Route, search_what: str, source_note: str = SOURCE_NOTE) -> str: if route.simple: note = SIMPLE_NOTE else: - note = (SEARCH_NOTE.format(what=search_what) if route.search else NO_SEARCH_NOTE) \ + note = (SEARCH_NOTE.format(what=search_what, source=source_note) if route.search else NO_SEARCH_NOTE) \ + tools_note(route.servers) + (PARTIAL_NOTE if route.partial else "") return SYSTEM.format(bot_name=bot_name, tools_note=note + "\n", refresh=REFRESH) diff --git a/lib/textfmt.py b/lib/textfmt.py index 711e1c2..fa1cffa 100644 --- a/lib/textfmt.py +++ b/lib/textfmt.py @@ -5,7 +5,7 @@ import html import re -from .config import MAX_TG_MESSAGE, SOURCES_MAX +from .config import MAX_TG_MESSAGE # [label](url), where label may itself be bracketed, as in [[1]](url) citations MD_LINK = re.compile(r"\[(\[[^\]]*\]|[^\[\]]+)\]\((https?://[^\s)]+)\)") @@ -51,21 +51,6 @@ def link(m: re.Match) -> str: return MD_CODE.sub(r"\1", text) -def with_sources(text: str, sources: list, limit: int = SOURCES_MAX) -> str: - """An answer with a "Sources:" line of links (sources are (title, url, ...)) when it has none of its own.""" - if not sources or " str: - host = re.sub(r"^www\.", "", url.split("/")[2]) if url.count("/") >= 2 else url - name = title if title else host - return html.escape(name if len(name) <= 40 else name[:39].rstrip() + "…") - - links = " · ".join(f'{label(title, url)}' - for title, url, *_ in sources[:limit]) - return f"{text}\n\nSources: {links}" - - def plain_text(text: str) -> str: """Telegram HTML as plain text: links become "label (url)", other tags are dropped.""" return html.unescape(ANY_TAG_RE.sub("", A_TAG_RE.sub(r"\2 (\1)", text))) diff --git a/tests/test_budgets.py b/tests/test_budgets.py index 9152c1c..86a648a 100644 --- a/tests/test_budgets.py +++ b/tests/test_budgets.py @@ -52,3 +52,10 @@ def test_system_prompt_explains_cut_lines(): def test_compact_text_helper(): assert compact('PME') == "PME" + + +def test_the_openrouter_citation_note_is_short(): + from lib.llm.openrouter import OpenRouterBackend + plain = report.tokens(system_prompt("Stock", Route([], True), "the web")) + cited = report.tokens(system_prompt("Stock", Route([], True), "the web", OpenRouterBackend.source_note)) + assert cited - plain < 60, cited - plain diff --git a/tests/test_cite.py b/tests/test_cite.py index 0c6d903..255bde5 100644 --- a/tests/test_cite.py +++ b/tests/test_cite.py @@ -41,3 +41,16 @@ def test_nothing_is_marked_without_snippets_or_when_markers_exist_already(): assert mark_by_overlap(TEXT, [Source("T", A)]) == TEXT and mark_by_overlap(TEXT, []) == TEXT marked = mark_by_overlap(TEXT, [Source("Release", A, PRESS)]) assert mark_by_overlap(marked, [Source("News", B, NEWS)]) == marked + + +def test_the_models_own_links_are_kept_only_for_pages_the_search_returned_and_renumbered(): + from lib.llm.cite import normalise_markers + srcs = [Source("A", A), Source("B", B)] + model = ('One.[3] Two.[2] ' + 'Three.[1][3]') + out = normalise_markers(model, srcs) + # b.example (given without its slash) is 1, the invented page is gone, a.example is 2 and b.example stays 1 + assert out == ('One.[1] Two. ' + 'Three.[2][1]') + assert normalise_markers(model, []) == "One. Two. Three." # no search happened: nothing to cite + assert mark_by_overlap(out, [Source("Release", A, PRESS)]) == out # the model's markers are enough diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 201b743..e0635a8 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -25,16 +25,6 @@ async def run(h, msg, edited=False): await asyncio.sleep(0) -async def test_a_chat_answer_that_used_web_search_ends_with_its_sources(env, store): - from lib.llm.base import Source - cited = step("Micron guided higher.") - cited.sources = [Source("Reuters", "https://www.reuters.com/mu")] - ctx, backend, h = make_ctx(env, store, [cited]) - m = user_msg(ctx.bot, "@stockbot what did Micron say?") - await run(h, m) - assert m.replies[0]["text"].endswith('Sources: Reuters') - - async def test_group_mention_is_answered_with_linked_tickers_and_logged(env, store): ctx, backend, h = make_ctx(env, store, [step("NVDA looks strong, SQX.AX too")]) m = user_msg(ctx.bot, "@stockbot how is NVDA?") @@ -517,3 +507,14 @@ async def test_the_log_shows_only_the_first_two_words_of_a_message(env, store, c lines = [r.getMessage() for r in caplog.records] assert any(line.endswith("@stockbot my…") for line in lines) assert not any("secret" in line or "portfolio" in line for line in lines) + + +async def test_only_chat_answers_get_the_citation_instruction(env, store): + from lib.ask import ask + from lib.llm.gate import Route + ctx, backend, h = make_ctx(env, store, [step("a"), step("b")]) + backend.source_note = "CITE-LIKE-THIS" + await ask(ctx, [{"type": "text", "text": "q"}], Route([], True)) + await ask(ctx, [{"type": "text", "text": "q"}], Route([], True), kind="holding-news") + assert "CITE-LIKE-THIS" in backend.seen[0]["system"] and "CITE-LIKE-THIS" not in backend.seen[1]["system"] + assert "one link at most" in backend.seen[1]["system"] diff --git a/tests/test_textfmt.py b/tests/test_textfmt.py index c4f8d36..e7d8ba4 100644 --- a/tests/test_textfmt.py +++ b/tests/test_textfmt.py @@ -52,14 +52,3 @@ def test_split_html_edge_cases(): assert all(len(c) <= 4096 for c in cs) cs = t.split_html("" * 20 + "x " * 3000 + "" * 20) assert len(cs) > 1 and all(len(c) <= 4096 and _balanced(c) for c in cs) - - -def test_with_sources_adds_links_only_when_the_answer_has_none(): - src = [("Reuters: NVDA & co", "https://www.reuters.com/a?x=1&y=2"), ("", "https://www.ft.com/b/c"), - ("x" * 60, "https://c.example/"), ("D", "https://d.example/")] - out = t.with_sources("NVDA rose.", src) - assert out == ('NVDA rose.\n\nSources: Reuters: NVDA & co' - ' · ft.com · ' + "x" * 39 + "…") - assert t.with_sources('See a.', src) == 'See a.' - assert t.with_sources("See [a](https://a.b).", src) == "See [a](https://a.b)." - assert t.with_sources("Plain.", []) == "Plain." From dbfe3334ef06b5deda10dfa2d758b14702064c8b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:37:03 +0000 Subject: [PATCH 14/30] Cite: debug-log why no [n] markers were added and which model links were dropped Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- lib/llm/cite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/llm/cite.py b/lib/llm/cite.py index 4421b05..ea5742e 100644 --- a/lib/llm/cite.py +++ b/lib/llm/cite.py @@ -51,6 +51,7 @@ def normalise_markers(text: str, sources: list[Source]) -> str: def fix(m: re.Match) -> str: url = known.get(html.unescape(m.group(1)).rstrip("/")) if url is None: + log.debug("Dropped the model's link to %s: not a page the search returned", html.unescape(m.group(1))) return "" n = numbers.setdefault(url, len(numbers) + 1) return f'[{n}]' @@ -71,9 +72,11 @@ def mark_by_overlap(text: str, sources: list[Source]) -> str: link goes where the figures and the distinctive words of the sentence are also in the page's snippet. Words every snippet shares count for less. A source that fits no sentence gets no marker.""" if MARKER_RE.search(text) or not sources: + log.debug("No [n] markers added: %s", "the model's own are there" if sources else "no sources") return text snippets = [_tokens(s.content) for s in sources if s.content] if not snippets: + log.debug("No [n] markers added: none of the %d sources came with a snippet", len(sources)) return text spread: dict[str, int] = {} for _, words in snippets: From be66672b88d282d8c096e13dd0310ff612f8d6fd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:42:33 +0000 Subject: [PATCH 15/30] NOT_TICKERS env var: extra words never linked as tickers Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/configuration.md | 1 + lib/app.py | 3 ++- lib/config.py | 3 +++ lib/tickers.py | 5 +++++ tests/test_tickers.py | 15 +++++++++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9ce6adb..8503be9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -29,6 +29,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. | | `MAX_TOKENS` | `3000` | Reply cap in tokens, reasoning included. | | `MOVERS_BOTS` | empty (feature off) | Usernames of bots whose end-of-day big-movers lists get explained. | +| `NOT_TICKERS` | empty | Extra capitalised words, comma or space separated, that are never linked as stock tickers, e.g. `GAAP GM`. Added to a built-in list (`EPS`, `GDP`, `GPU`, ...), so it only needs the words that still get linked. Words that are real tickers (`GM`, `UPS`) then stop linking everywhere. | | `PORTFOLIO_NAMES` | the holding-news recipients' portfolio names | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | | `PERIOD_BACKFILL` | `all` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. List names to limit it, or `off` to disable. See [Caching Sharesight history](caching.md#caching-sharesight-history). | | `POST_TO_GROUPS_FROM_DM` | `off` | `on` or `off`. A group admin can have the bot post in that group from a DM. | diff --git a/lib/app.py b/lib/app.py index 3694bb4..1dc9ac4 100644 --- a/lib/app.py +++ b/lib/app.py @@ -32,7 +32,7 @@ filters, ) -from . import config +from . import config, tickers from .context import Ctx from .features import chat_commands, dm_buttons, holding_news, post from .features.charts import buttons as chart_buttons @@ -163,6 +163,7 @@ async def register_menu(ctx: Ctx) -> None: def build_ctx(st: config.Settings) -> Ctx: ctx = Ctx(st, Store(st.db_path), build_backend(st), Registry([])) + tickers.exclude(st.not_tickers) async def on_down(server: MCPServer) -> None: await alert_down(ctx, server) diff --git a/lib/config.py b/lib/config.py index 522c999..103e29f 100644 --- a/lib/config.py +++ b/lib/config.py @@ -29,6 +29,7 @@ "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.", "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).", "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.", + "NOT_TICKERS": "Extra capitalised words (comma/space separated) that must not be linked as stock tickers, e.g. GAAP GM; added to the built-in list.", "PORTFOLIO_NAMES": "Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names).", "PERIOD_BACKFILL": "Comma list of Sharesight portfolio names (or all) whose daily/weekly/monthly/quarterly/yearly returns are fetched into the local cache in the background, so period questions answer instantly. Default all; list names to limit it, or off to disable.", "POST_TO_GROUPS_FROM_DM": "on|off. Lets an admin of a group the bot is in make it post there from a DM (default off).", @@ -89,6 +90,7 @@ class Settings: holding_news_time: str holding_news_recipients: dict[str, str] portfolio_names: frozenset[str] + not_tickers: frozenset[str] # extra words never linked as tickers period_backfill: frozenset[str] @property @@ -204,5 +206,6 @@ def load(env: Mapping[str, str] | None = None) -> Settings: holding_news_time=env.get("SHARESIGHT_HOLDING_NEWS_TIME", "08:00"), holding_news_recipients=recipients, portfolio_names=_names(env.get("PORTFOLIO_NAMES", "")) or frozenset(recipients), + not_tickers=frozenset(w.upper() for w in env.get("NOT_TICKERS", "").replace(",", " ").split()), period_backfill=_backfill(env.get("PERIOD_BACKFILL", "all")), ) diff --git a/lib/tickers.py b/lib/tickers.py index 5bcb075..6f4e831 100644 --- a/lib/tickers.py +++ b/lib/tickers.py @@ -31,6 +31,11 @@ "Still", "Plus", "Even", "Now", "Overall", "However", "Elsewhere"} +def exclude(words) -> None: + """Add words (upper case) that are never to be linked as tickers: the NOT_TICKERS setting.""" + NOT_TICKERS.update(words) + + def is_ticker(word: str) -> bool: base, _, suffix = word.partition(".") if suffix in EXCHANGE_SUFFIXES and base not in NOT_TICKERS: diff --git a/tests/test_tickers.py b/tests/test_tickers.py index c922e70..c7ee99f 100644 --- a/tests/test_tickers.py +++ b/tests/test_tickers.py @@ -70,3 +70,18 @@ def test_link_labelled_with_its_own_url_is_numbered_but_named_links_and_code_are assert link_tickers('https://y.com/p… ok') == '[1] ok' assert link_tickers('Reuters') == 'Reuters' assert link_tickers("https://x.com") == "https://x.com" + + +def test_not_tickers_setting_adds_words_that_are_never_linked(env): + from lib import config, tickers + st = config.load({**env, "NOT_TICKERS": "gaap, GM ups"}) + assert st.not_tickers == {"GAAP", "GM", "UPS"} and config.load(env).not_tickers == frozenset() + before = set(tickers.NOT_TICKERS) + try: + assert "GAAP" in tickers.link_tickers("non-GAAP EPS at GM") + tickers.exclude(st.not_tickers) + out = tickers.link_tickers("non-GAAP EPS at GM, UPS and NVDA") + assert "GAAP" in out and ">GAAP<" not in out and ">GM<" not in out and ">UPS<" not in out and ">NVDA<" in out + finally: + tickers.NOT_TICKERS.clear() + tickers.NOT_TICKERS.update(before) From b371c23623257ecde60435edddd8df938e885480 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:43:14 +0000 Subject: [PATCH 16/30] Docs: configuration.md ends after the Advanced and optional table Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/configuration.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8503be9..f545fa1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -38,19 +38,3 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | `SHARESIGHT_HOLDING_NEWS_TIME` | `08:00` | HH:MM (in `TZ`) for the daily holding-news check. | | `TELEGRAM_DM_BUTTONS` | `off` | `on` or `off`. Preset-prompt buttons in private chats. | | `TELEGRAM_IMAGE_PRELOAD_CHATID` | empty (off) | Chat ID of a private chat the bot can post in (a private group it is in, or a DM with you). The other periods of a chart are uploaded there in the background and reused when a period button is tapped, so the tap swaps the picture without fetching or drawing. Empty = off, and a tap fetches and draws. See [Charts](features.md#charts). | - -## Choosing a model -Figures from search results and Artificial Analysis at the time of writing (October 2026), so -treat them as rough; speed varies by provider. - -| Model | $ per 1M in / out | Output speed | Quality index | -|---|---|---|---| -| `xiaomi/mimo-v2.6-pro` | 0.43 / 0.87 | ~28-46 tok/s | 46 (top open-weight model) | -| `xiaomi/mimo-v2.6-flash` | 0.14 / 0.28 | ~56 tok/s | 38 | -| `deepseek/deepseek-v4.1-flash-1017` (default on OpenRouter) | not measured | not measured | not measured | -| `z-ai/glm-5.3-flash` (the previous default) | 0.15 / 0.50 | ~50 tok/s (other hosts up to ~270) | 57 | -| `grok-4.3` / `x-ai/grok-4.3` (default on xAI) | 1.25 / 2.50 | ~105-146 tok/s | 25 (at high reasoning) | -| `xiaomi/mimo-v2.5-pro` | 0.30 / 0.61 | ~29-46 tok/s | unreliable (retires 21 Oct 2026) | -| `xiaomi/mimo-v2.5` | 0.12 / 0.24 | ~44-58 tok/s | not found (retires 21 Oct 2026) | - -Most of the delay is reasoning time before the first answer token, not typing speed: use `REASONING=low`. From 6a4b748a370c6b9b7e57beeb54aa653ab084cb40 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 22:57:57 +0000 Subject: [PATCH 17/30] Cite: add [n] for the pages the model did not cite, then number all in order The model cited 3 of ~10 pages the search returned; the overlap pass now runs for the rest even when the model wrote markers of its own. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- docs/how-it-works.md | 10 +++++----- lib/llm/cite.py | 21 ++++++++++++++------- lib/llm/runner.py | 4 ++-- tests/test_cite.py | 18 +++++++++++++++--- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index e306eac..313b46b 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -44,11 +44,11 @@ query through `SEARCH_MODEL` (default: `MODEL` with `:online` added) and returns the write-up; that call's cost is added to the answer's cost. Some models print a search call as text instead of making it; the bot recovers those queries too. In a chat answer the model is told to cite a sentence that rests on a search result as `[1]`, the number linking to the - page (its exact URL from the results). The bot keeps only links to pages the search really returned and renumbers - them in order. OpenRouter's `url_citation` annotations say which pages were used but not which sentence each backs - (the positions are 0), so if the model gave no link, the bot puts `[n]` after the sentence each snippet fits best: the - figures and distinctive words of the sentence also appear in the snippet. A sentence that fits none, such as an - opinion, gets none; there is no list of sources at the end. `DEBUG` logs show each source's best score. + page (its exact URL from the results). The bot keeps only links to pages the search really returned. OpenRouter's + `url_citation` annotations say which pages were used but not which sentence each backs (the positions are 0), so each + page the model did not cite gets a `[n]` after the sentence its snippet fits best: the figures and distinctive words + of the sentence also appear in the snippet. A sentence that fits no snippet, such as an opinion, gets none. All the + numbers follow the order they appear in; there is no list of sources at the end. `DEBUG` logs show each page's best score. - At most 5 searches per round; extras get a "skipped" note. - `SEARCH=off` turns search off (the prompt then says the model has no web search). - **Forced search:** movers lists, DM preset buttons and the holding-news digest *require* a diff --git a/lib/llm/cite.py b/lib/llm/cite.py index ea5742e..b49ed7f 100644 --- a/lib/llm/cite.py +++ b/lib/llm/cite.py @@ -65,18 +65,25 @@ def _tokens(text: str) -> tuple[set[str], set[str]]: {w.lower() for w in WORD_RE.findall(plain)}) -def mark_by_overlap(text: str, sources: list[Source]) -> str: +def cite_answer(text: str, sources: list[Source]) -> str: + """The answer with [n] links for the pages a search used: the model's own, where they point at a page the search + returned, then one after the sentence each remaining page's snippet fits best. Numbered in order of appearance.""" + text = normalise_markers(text, sources) + cited = {html.unescape(url).rstrip("/") for url in MARKER_RE.findall(text)} + rest = [s for s in sources if s.url.rstrip("/") not in cited] + return normalise_markers(mark_by_overlap(text, rest, among=sources), sources) + + +def mark_by_overlap(text: str, sources: list[Source], among: list[Source] | None = None) -> str: """Put "[n]" after the sentence of `text` that each source's snippet fits best, when it fits well enough. OpenRouter says which pages a search used but not which sentence each backs (its positions are 0), so the link goes where the figures and the distinctive words of the sentence are also in the page's snippet. - Words every snippet shares count for less. A source that fits no sentence gets no marker.""" - if MARKER_RE.search(text) or not sources: - log.debug("No [n] markers added: %s", "the model's own are there" if sources else "no sources") - return text - snippets = [_tokens(s.content) for s in sources if s.content] + Words every snippet shares count for less (`among`: the snippets compared, default `sources`). A source that fits + no sentence gets no marker.""" + snippets = [_tokens(s.content) for s in (among or sources) if s.content] if not snippets: - log.debug("No [n] markers added: none of the %d sources came with a snippet", len(sources)) + log.debug("No [n] markers added by overlap: %d sources left without a snippet", len(sources)) return text spread: dict[str, int] = {} for _, words in snippets: diff --git a/lib/llm/runner.py b/lib/llm/runner.py index 4c5d239..82acfe2 100644 --- a/lib/llm/runner.py +++ b/lib/llm/runner.py @@ -15,7 +15,7 @@ from ..mcp.server import Registry from ..textfmt import TOOL_SYNTAX_RE from .base import Answer, Backend, Call, Request, Source, Step, Usage -from .cite import mark_by_overlap, normalise_markers +from .cite import cite_answer log = logging.getLogger("bot") @@ -237,5 +237,5 @@ async def run(backend: Backend, registry: Registry, req: Request, *, dedupe: boo raise RuntimeError("hit the token cap before writing an answer") raise RuntimeError(f"empty reply (finish_reason={step.finish})") if req.cite: - text = mark_by_overlap(normalise_markers(text, cited), cited) + text = cite_answer(text, cited) return Answer(text, total, rounds, tool_calls, cited) diff --git a/tests/test_cite.py b/tests/test_cite.py index 255bde5..0c0c9aa 100644 --- a/tests/test_cite.py +++ b/tests/test_cite.py @@ -37,10 +37,22 @@ def test_numbering_follows_the_order_in_the_text_and_a_source_that_fits_nothing_ assert out.count("]") == 2 -def test_nothing_is_marked_without_snippets_or_when_markers_exist_already(): +def test_nothing_is_marked_without_snippets(): assert mark_by_overlap(TEXT, [Source("T", A)]) == TEXT and mark_by_overlap(TEXT, []) == TEXT - marked = mark_by_overlap(TEXT, [Source("Release", A, PRESS)]) - assert mark_by_overlap(marked, [Source("News", B, NEWS)]) == marked + + +def test_pages_the_model_did_not_cite_are_added_after_their_sentences_and_all_are_numbered_in_order(): + from lib.llm.cite import cite_answer + srcs = [Source("Release", A, PRESS), Source("News", B, NEWS)] + model = TEXT.replace("EPS $2.09.", 'EPS $2.09.[7]') # it cited News for the figures + out = cite_answer(model, srcs) + # the model's own link stays where it put it (number 1, though it wrote 7); Release, which it did not cite, goes + # after the Data Center sentence it fits (number 2); News is not cited a second time on the Street sentence + assert out == ( + "AMD's quarter: revenue $13.38b, +45% y/y, EPS $2.09." + LINK_B.replace(">2<", ">1<") + " Data Center did the work, " + "$8.5b." + LINK_A.replace(">1<", ">2<") + " Street is bullish, with JPMorgan and Morgan Stanley raising " + "targets to $575 and $550 while staying neutral. My read: the multiple is high.") + assert cite_answer(TEXT, []) == TEXT def test_the_models_own_links_are_kept_only_for_pages_the_search_returned_and_renumbered(): From 934757927649c01cbcf6aafc9772864333c293f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 5 Nov 2026 06:32:42 +0000 Subject: [PATCH 18/30] Ticker links: only marked forms and known tickers, never bare words Bare capitalised words (GAAP, EBITDA, AFR, GM, UPS) kept getting Yahoo links and the stoplist never ended; ASX:NXT linked to the US NXT. Replies now link only $NVDA, NXT.AX, ASX:NXT (mapped to Yahoo's suffix), Name (TICKER), Sharesight's NXT (ASX), ^GSPC and BTC-USD, plus bare words in a known set: the S&P 500 and NASDAQ-100 (from Wikipedia) and the US Sharesight holdings, refreshed daily in the background and kept in kv, never fetched while answering. A bare code in brackets links only if known, since it names no exchange. Holding news links the holdings' codes and a movers explanation the listed codes with the list's exchange. The prompt asks for these forms. The NOT_TICKERS env var is gone; the built-in list now only serves the tool gate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw --- CLAUDE.md | 4 +- docs/configuration.md | 1 - docs/how-it-works.md | 19 +++-- lib/app.py | 6 +- lib/config.py | 11 ++- lib/features/holding_news.py | 32 ++------ lib/features/known_tickers.py | 150 ++++++++++++++++++++++++++++++++++ lib/features/movers.py | 18 ++++ lib/holdings.py | 36 ++++++++ lib/prompts.py | 2 +- lib/telegram/handlers.py | 12 ++- lib/tickers.py | 150 +++++++++++++++++++++++++++------- lib/tickers_data.py | 22 ++++- tests/test_features.py | 16 ++++ tests/test_fixtures.py | 8 +- tests/test_handlers.py | 8 +- tests/test_known_tickers.py | 90 ++++++++++++++++++++ tests/test_tickers.py | 134 ++++++++++++++++++------------ 18 files changed, 586 insertions(+), 133 deletions(-) create mode 100644 lib/features/known_tickers.py create mode 100644 lib/holdings.py create mode 100644 tests/test_known_tickers.py diff --git a/CLAUDE.md b/CLAUDE.md index bbff8fc..b9d1a02 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,14 +19,14 @@ MCP data tools and web search. User-facing docs are in `README.md` (quick start) - `store.py`: SQLite `Store`: history (`messages` shared by groups, `dm_messages` per bot), users, kv, holding-news state, `usage` ledger, `text_cache` (finished /narrative and /events messages, 24 h), `period_cache` (finished Sharesight periods, kept until flushed). Thread-safe; call through `asyncio.to_thread`. - `msgtext.py` sender_name/describe. `history.py` transcript lines. `textfmt.py` HTML helpers - (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo links (+ `tickers_data.py`). + (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo links on marked tickers (`$X`, `X.AX`, `ASX:X`, `Name (X)`, `X (ASX)`) and the known set `tickers.KNOWN`; bare words otherwise never link (+ `tickers_data.py`: exchange prefixes -> Yahoo suffixes, the gate's NOT_TICKERS). `holdings.py` current Sharesight holdings (holding news, known tickers). - `mcp/`: `server.py` (MCPServer lifecycle, TTL cache, `Registry`), `schema.py` (neutral `ToolDef`, schema diet), `results.py` (result slimming). - `llm/`: `base.py` (Request/Step/Answer/Backend), `chat.py` (shared chat-completions loop), `xai.py` + `openrouter.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs). - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun. - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming - drafts), `commands.py` (/tokens), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model). + drafts), `commands.py` (/tokens), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `known_tickers.py` (daily background refresh of `tickers.KNOWN`: S&P 500 + NASDAQ-100 from Wikipedia, US Sharesight holdings; kept in kv), `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model). - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot). - `docs/plans/go-recode.md`: saved plan for a Go rewrite (not started); its key requirement is a modular design with self-contained features. - `deploy/systemd/`: the example unit `grokbot.service` and the optional `grokbot.service.d/hardening.conf` drop-in (docs/running.md). diff --git a/docs/configuration.md b/docs/configuration.md index f545fa1..23df421 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -29,7 +29,6 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o | `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. | | `MAX_TOKENS` | `3000` | Reply cap in tokens, reasoning included. | | `MOVERS_BOTS` | empty (feature off) | Usernames of bots whose end-of-day big-movers lists get explained. | -| `NOT_TICKERS` | empty | Extra capitalised words, comma or space separated, that are never linked as stock tickers, e.g. `GAAP GM`. Added to a built-in list (`EPS`, `GDP`, `GPU`, ...), so it only needs the words that still get linked. Words that are real tickers (`GM`, `UPS`) then stop linking everywhere. | | `PORTFOLIO_NAMES` | the holding-news recipients' portfolio names | Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names). | | `PERIOD_BACKFILL` | `all` | Comma list of Sharesight portfolio names (or `all`) whose returns per day, week, month, quarter and year are fetched into the local cache in the background (about 1,650 Sharesight calls per portfolio the first time, a few hours), so "best day since inception" answers instantly. List names to limit it, or `off` to disable. See [Caching Sharesight history](caching.md#caching-sharesight-history). | | `POST_TO_GROUPS_FROM_DM` | `off` | `on` or `off`. A group admin can have the bot post in that group from a DM. | diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 313b46b..667276f 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -104,12 +104,19 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf ## Formatting and ticker links - Replies are Telegram HTML. Markdown the model slips in (`**bold**`, links, backticks) is converted; trailing "not advice"/"NFA"/"DYOR" disclaimers are stripped. -- **Yahoo Finance links:** the model writes plain tickers (with the exchange suffix for non-US - listings: `SQX.AX`, `000660.KS`) and the bot links each one to its Yahoo Finance page. The visible text drops the suffix (`SQX.AX` shows as `SQX`, `BRK.B` stays) while the URL - keeps it; crypto gets `-USD` in the URL (`BTC` -> `BTC-USD`). Words like CEO, ETF, FY26 and Q3 are - not linked, a lone letter only counts next to a price or move (`F 5.2`), and text already inside a - link, `` or `
` is left alone. A company name written as `Name (TICKER)` is bold,
-  name and ticker together (`Micron (MU)`), unless it is already bold.
+- **Yahoo Finance links:** only a ticker written in a marked form is linked to its Yahoo Finance page: `$NVDA`,
+  an exchange suffix (`NXT.AX`, `000660.KS`), an exchange prefix (`ASX:NXT`, `NASDAQ: NVDA`, mapped to Yahoo's suffix:
+  `ASX:NXT` -> `NXT.AX`), `Name (TICKER)` (bold, name and ticker together, unless already bold), Sharesight's
+  `NXT (ASX)`, `^GSPC` and `BTC-USD`. The system prompt asks the model for these forms. A bare word is linked only
+  when it is a known ticker: the S&P 500 and NASDAQ-100 constituents (read from their Wikipedia pages) and the US
+  Sharesight holdings, refreshed once a day in the background and kept in the database, so a reply never waits for
+  them. Any other capitalised word stays plain (`GAAP`, `EBITDA`, `AFR`, `M3`), so there is no stoplist to maintain;
+  single letters and index tickers that are everyday words (`IT`, `ALL`, `NOW`) need `$` or `Name (TICKER)`. A bare
+  code in brackets names no exchange, so `Goodman Group (GMG)` is linked only if `GMG` is known (it is `GMG.AX`).
+  A holding-news digest also links the holdings' own codes, and a movers explanation the listed codes with the list's
+  exchange. The visible text drops `$`, the exchange prefix and the suffix (`NXT.AX` shows as `NXT`; `BRK.B` and
+  number codes such as `9988` stay as they are) while the URL keeps it; crypto gets `-USD` in the URL. Text inside a
+  link, `` or `
` is left alone.
 - **Citations:** a raw link in a reply (a bare URL, or a link labelled with its own URL) becomes a
   numbered link, `[1]`, `[2]`, in order of appearance, the same URL keeping its number. Links with a
   real label, and anything in `` or `
`, are left alone.
diff --git a/lib/app.py b/lib/app.py
index 1dc9ac4..37a0f7f 100644
--- a/lib/app.py
+++ b/lib/app.py
@@ -32,9 +32,9 @@
     filters,
 )
 
-from . import config, tickers
+from . import config
 from .context import Ctx
-from .features import chat_commands, dm_buttons, holding_news, post
+from .features import chat_commands, dm_buttons, holding_news, known_tickers, post
 from .features.charts import buttons as chart_buttons
 from .features.charts import pages as chart_pages
 from .features.periods import backfill
@@ -163,7 +163,6 @@ async def register_menu(ctx: Ctx) -> None:
 
 def build_ctx(st: config.Settings) -> Ctx:
     ctx = Ctx(st, Store(st.db_path), build_backend(st), Registry([]))
-    tickers.exclude(st.not_tickers)
 
     async def on_down(server: MCPServer) -> None:
         await alert_down(ctx, server)
@@ -185,6 +184,7 @@ async def post_init(app: Application) -> None:
         await register_menu(ctx)
         ctx.registry.start()  # connects in the background; requests see whichever servers are up
         background.append(asyncio.create_task(log_tool_size(ctx)))
+        background.append(asyncio.create_task(known_tickers.loop(ctx)))
         if st.holding_news:
             background.append(asyncio.create_task(holding_news.daily_loop(ctx)))
         if st.dm_buttons:
diff --git a/lib/config.py b/lib/config.py
index 103e29f..4be4471 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -29,7 +29,6 @@
     "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.",
     "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).",
     "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.",
-    "NOT_TICKERS": "Extra capitalised words (comma/space separated) that must not be linked as stock tickers, e.g. GAAP GM; added to the built-in list.",
     "PORTFOLIO_NAMES": "Comma list of Sharesight portfolio names; a message naming one is treated as a portfolio question (default: the recipients' portfolio names).",
     "PERIOD_BACKFILL": "Comma list of Sharesight portfolio names (or all) whose daily/weekly/monthly/quarterly/yearly returns are fetched into the local cache in the background, so period questions answer instantly. Default all; list names to limit it, or off to disable.",
     "POST_TO_GROUPS_FROM_DM": "on|off. Lets an admin of a group the bot is in make it post there from a DM (default off).",
@@ -57,6 +56,14 @@
 SEARCH_RESULTS = 20
 SEARCH_TOKENS = 400  # cap on each search write-up (OpenRouter search runner)
 MAX_SEARCHES = 5  # search calls run per round
+# Index constituents linked bare in replies (features/known_tickers.py): (name, Wikipedia page, fewest and most rows
+# that look right). Fetched in the background once a day, never while answering.
+INDEX_LISTS = (
+    ("S&P 500", "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies", 450, 550),
+    ("NASDAQ-100", "https://en.wikipedia.org/wiki/Nasdaq-100", 90, 110),
+)
+KNOWN_TICKERS_REFRESH = 24 * 3600  # seconds between refreshes of the indexes and the US holdings
+KNOWN_TICKERS_START_DELAY = 60  # seconds after startup before a due refresh, so the MCP servers are up
 SOURCES_MAX = 3  # links to a search result the model is handed back as text (OpenRouter run_search)
 MAX_TG_MESSAGE = 4000  # Telegram's limit is 4096
 HOLDING_NEWS_RETRY_EVERY = 30 * 60  # seconds between retries of a holding-news check the provider was too busy for
@@ -90,7 +97,6 @@ class Settings:
     holding_news_time: str
     holding_news_recipients: dict[str, str]
     portfolio_names: frozenset[str]
-    not_tickers: frozenset[str]  # extra words never linked as tickers
     period_backfill: frozenset[str]
 
     @property
@@ -206,6 +212,5 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
         holding_news_time=env.get("SHARESIGHT_HOLDING_NEWS_TIME", "08:00"),
         holding_news_recipients=recipients,
         portfolio_names=_names(env.get("PORTFOLIO_NAMES", "")) or frozenset(recipients),
-        not_tickers=frozenset(w.upper() for w in env.get("NOT_TICKERS", "").replace(",", " ").split()),
         period_backfill=_backfill(env.get("PERIOD_BACKFILL", "all")),
     )
diff --git a/lib/features/holding_news.py b/lib/features/holding_news.py
index 8cb3032..1216647 100644
--- a/lib/features/holding_news.py
+++ b/lib/features/holding_news.py
@@ -5,7 +5,6 @@
 
 import asyncio
 import html
-import json
 import logging
 import re
 from datetime import datetime, timedelta
@@ -18,14 +17,14 @@
 from .. import config
 from ..ask import ask
 from ..context import Ctx
+from ..holdings import code_and_market, current_holdings
 from ..llm.base import BUSY_REPLY, is_busy
 from ..llm.gate import Route
-from ..mcp.results import table_records
 from ..prompts import HOLDING_NEWS_PROMPT
 from ..telegram.draft import Draft
 from ..telegram.send import send_html
 from ..textfmt import ANY_TAG_RE, md_to_html, split_html, strip_disclaimer
-from ..tickers import link_tickers
+from ..tickers import link_tickers, yahoo_symbol
 
 log = logging.getLogger("bot")
 UNSUBSCRIBE_BUTTON = InlineKeyboardMarkup([[InlineKeyboardButton("🔕 Unsubscribe…", callback_data="hn:menu")]])
@@ -46,6 +45,12 @@ def as_bullets(text: str) -> str:
     return "\n".join(f"• {ln}" for ln in lines if ln)
 
 
+def known_symbols(holdings: dict[str, str]) -> dict[str, str]:
+    """The holdings' codes as Yahoo symbols ({"NXT": "NXT.AX"}), so the digest links them however the model writes them."""
+    pairs = (code_and_market(key) for key in holdings)
+    return {code: yahoo_symbol(code, market) or code for code, market in pairs}
+
+
 def news_codes(news: str, holdings: dict[str, str]) -> list[str]:
     """Holding codes that appear as bold tickers or Yahoo links in a news message."""
     held = {k.split(" ")[0].upper() for k in holdings}
@@ -68,25 +73,6 @@ def undo_button(code: str) -> InlineKeyboardMarkup:
         [[InlineKeyboardButton(f"↩️ Undo (unsubscribed from {what})", callback_data=f"hn:undo:{code}")]])
 
 
-async def current_holdings(server, portfolio_names: list[str]) -> dict[str, str]:
-    """Current holdings across the named portfolios, as {"CODE (MARKET)": name}."""
-    data = json.loads(await server.call("list_portfolios", {}))
-    portfolios = {p["name"].lower(): p for p in data.get("portfolios", [])}
-    holdings: dict[str, str] = {}
-    for name in portfolio_names:
-        p = portfolios.get(name)
-        if not p:
-            log.warning("Holding news: no Sharesight portfolio called %r (have: %s)",
-                        name, ", ".join(sorted(portfolios)))
-            continue
-        report = json.loads(await server.call(
-            "get_performance_report", {"portfolio_id": p["id"], "include_sales": False})).get("report") or {}
-        for h in table_records(report.get("holdings")):
-            if h.get("code"):
-                holdings[f"{h['code']} ({h.get('market', '?')})"] = h.get("name") or h["code"]
-    return holdings
-
-
 async def holding_news_text(ctx: Ctx, username: str, holdings: dict[str, str]) -> str | None:
     """Ask the model for major news about these holdings; None if there's nothing."""
     since = int((datetime.now() - timedelta(days=3)).timestamp())
@@ -101,7 +87,7 @@ async def holding_news_text(ctx: Ctx, username: str, holdings: dict[str, str]) -
     raw = strip_disclaimer(answer.text)
     if is_nothing(raw):
         return None
-    return link_tickers(as_bullets(md_to_html(raw)))
+    return link_tickers(as_bullets(md_to_html(raw)), known=known_symbols(holdings))
 
 
 async def send_holding_news(ctx: Ctx, chat_id: int, news: str, codes: list[str]) -> None:
diff --git a/lib/features/known_tickers.py b/lib/features/known_tickers.py
new file mode 100644
index 0000000..de1173c
--- /dev/null
+++ b/lib/features/known_tickers.py
@@ -0,0 +1,150 @@
+"""Tickers a reply may name bare and still get a link: the S&P 500, the NASDAQ-100 and the US Sharesight holdings.
+
+Refreshed in the background once a day and kept in the kv table, so a restart has them at once; replies only read
+tickers.KNOWN and never wait for a fetch."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import time
+from html.parser import HTMLParser
+
+import httpx
+
+from .. import config, tickers
+from ..context import Ctx
+from ..holdings import code_and_market, current_holdings
+
+log = logging.getLogger("bot")
+KV_KEY = "known_tickers"
+SYMBOL_RE = re.compile(r"^[A-Z]{1,5}(?:-[A-Z])?$")
+HEADERS = {"User-Agent": "grokbot/1.0 (Telegram stock-chat bot; daily index list)"}
+
+
+class _Tables(HTMLParser):
+    """Every  on a page as {"id": ..., "rows": [[cell text, ...], ...]}, nested ones included."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.tables: list[dict] = []
+        self._open: list[dict] = []
+        self._cell: list[str] | None = None
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "table":
+            table = {"id": dict(attrs).get("id") or "", "rows": []}
+            self.tables.append(table)
+            self._open.append(table)
+        elif tag == "tr" and self._open:
+            self._open[-1]["rows"].append([])
+        elif tag in ("td", "th") and self._open and self._open[-1]["rows"]:
+            self._cell = []
+            self._open[-1]["rows"][-1].append(self._cell)
+
+    def handle_endtag(self, tag):
+        if tag == "table" and self._open:
+            self._open.pop()
+        elif tag in ("td", "th"):
+            self._cell = None
+
+    def handle_data(self, data):
+        if self._cell is not None:
+            self._cell.append(data)
+
+
+def symbols_from_html(page: str) -> list[str]:
+    """The ticker column of a Wikipedia index page: the table with id "constituents", else the first table with a
+    "Symbol" or "Ticker" header. Yahoo writes share classes with a hyphen (BRK.B -> BRK-B)."""
+    parser = _Tables()
+    parser.feed(page)
+    tables = sorted(parser.tables, key=lambda t: t["id"] != "constituents")
+    for table in tables:
+        rows = [["".join(cell).strip() for cell in row] for row in table["rows"]]
+        if not rows:
+            continue
+        header = [h.lower() for h in rows[0]]
+        col = next((i for i, h in enumerate(header) if h in ("symbol", "ticker", "ticker symbol")), None)
+        if col is None:
+            continue
+        found = [row[col].replace(".", "-") for row in rows[1:] if len(row) > col]
+        return [s for s in found if SYMBOL_RE.match(s)]
+    return []
+
+
+async def fetch_index(client: httpx.AsyncClient, name: str, url: str, low: int, high: int) -> list[str] | None:
+    """An index's tickers, or None if the page couldn't be read or doesn't look right."""
+    try:
+        r = await client.get(url, headers=HEADERS, follow_redirects=True)
+        r.raise_for_status()
+    except httpx.HTTPError as e:
+        log.warning("Known tickers: couldn't fetch the %s list: %s", name, e)
+        return None
+    found = symbols_from_html(r.text)
+    if not low <= len(found) <= high:
+        log.warning("Known tickers: the %s page gave %d tickers (expected %d-%d); keeping the last list",
+                    name, len(found), low, high)
+        return None
+    return found
+
+
+async def us_holdings(ctx: Ctx) -> list[str] | None:
+    """The codes of the US-listed Sharesight holdings, or None if Sharesight isn't up or isn't set up."""
+    server = next((s for s in ctx.registry.up() if s.cfg.get("gate") == "portfolio"), None)
+    if server is None or not ctx.st.portfolio_names:
+        return None
+    try:
+        held = await current_holdings(server, sorted(ctx.st.portfolio_names))
+    except Exception as e:  # the MCP server's own errors, bad JSON
+        log.warning("Known tickers: couldn't read the holdings: %s", e)
+        return None
+    pairs = (code_and_market(key) for key in held)
+    return sorted(code for code, market in pairs if tickers.yahoo_symbol(code, market) == code)
+
+
+def apply(saved: dict) -> None:
+    """Make the saved lists the ones replies link."""
+    codes = {c for name, _, _, _ in config.INDEX_LISTS for c in saved.get("indexes", {}).get(name, [])}
+    tickers.set_known({c: c for c in codes | set(saved.get("held", []))})
+
+
+def load(ctx: Ctx) -> dict:
+    """The saved lists (empty on a first start), applied."""
+    try:
+        saved = json.loads(ctx.store.kv_get(KV_KEY) or "{}")
+    except ValueError:
+        saved = {}
+    apply(saved)
+    return saved
+
+
+async def refresh(ctx: Ctx, saved: dict, client: httpx.AsyncClient) -> dict:
+    """Fetch the indexes and the holdings; a part that fails keeps its last list."""
+    indexes = dict(saved.get("indexes", {}))
+    for name, url, low, high in config.INDEX_LISTS:
+        found = await fetch_index(client, name, url, low, high)
+        if found is not None:
+            indexes[name] = found
+    held = await us_holdings(ctx)
+    new = {"indexes": indexes, "held": saved.get("held", []) if held is None else held, "ts": int(time.time())}
+    await asyncio.to_thread(ctx.store.kv_set, KV_KEY, json.dumps(new))
+    apply(new)
+    log.info("Known tickers: %s, %d held, %d in all", ", ".join(f"{len(v)} {k}" for k, v in indexes.items()) or
+             "no index lists", len(new["held"]), len(tickers.KNOWN))
+    return new
+
+
+async def loop(ctx: Ctx) -> None:
+    saved = await asyncio.to_thread(load, ctx)
+    log.info("Known tickers: %d loaded", len(tickers.KNOWN))
+    async with httpx.AsyncClient(timeout=30) as client:
+        while True:
+            due = saved.get("ts", 0) + config.KNOWN_TICKERS_REFRESH - time.time()
+            await asyncio.sleep(max(due, config.KNOWN_TICKERS_START_DELAY))  # overdue at startup: once the MCP is up
+            try:
+                saved = await refresh(ctx, saved, client)
+            except Exception:
+                log.exception("Known tickers: refresh failed")
+                saved = {**saved, "ts": int(time.time())}  # try again tomorrow, not in a tight loop
diff --git a/lib/features/movers.py b/lib/features/movers.py
index f19d0da..0fb0d5f 100644
--- a/lib/features/movers.py
+++ b/lib/features/movers.py
@@ -6,9 +6,13 @@
 
 from telegram import Message, MessageEntity
 
+from ..tickers import yahoo_symbol
+
 # The bold header, e.g. "≥ 5.0% at close (ASX):" or "≤ -5% at close (NASDAQ, NYSE):"
 HEADER = re.compile(r"(?:≥|≤|>=?|<=?)\s*[-−]?\s*\d+(?:\.\d+)?\s*%.*\bclose\b", re.IGNORECASE)
 PERCENT = re.compile(r"\d+(?:\.\d+)?\s*%")
+EXCHANGES = re.compile(r"\(([A-Z][A-Z ,/]*)\)")  # "(ASX)", "(NASDAQ, NYSE)" in the header
+STOCK_LINE = re.compile(r"^\W*([A-Z][A-Z0-9]{0,5})\b.*\d+(?:\.\d+)?\s*%")
 
 
 def is_movers_list(msg: Message, bots: frozenset[str]) -> bool:
@@ -42,3 +46,17 @@ def strip_summary(text: str) -> str:
     if len(paras) > 1 and " dict[str, str]:
+    """The listed codes as Yahoo symbols ({"DRO": "DRO.AX"} under an "(ASX)" header), so the explanation links them
+    bare. A header naming exchanges with different suffixes, or none we know, gives nothing: a guess could be wrong."""
+    lines = re.sub(r"<[^>]+>", "", listing_html).splitlines()
+    head = next((ln for ln in lines if HEADER.search(ln)), "")
+    found = EXCHANGES.search(head)
+    names = [n.strip() for n in re.split(r"[,/]", found.group(1))] if found else []
+    symbols = {yahoo_symbol("X", n) for n in names}
+    if len(symbols) != 1 or None in symbols:
+        return {}
+    codes = (m.group(1) for ln in lines if ln is not head and (m := STOCK_LINE.match(ln)))
+    return {code: yahoo_symbol(code, names[0]) or code for code in codes}
diff --git a/lib/holdings.py b/lib/holdings.py
new file mode 100644
index 0000000..9fdb6a3
--- /dev/null
+++ b/lib/holdings.py
@@ -0,0 +1,36 @@
+"""Current Sharesight holdings across named portfolios, shared by holding news and the known-ticker list."""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+
+from .mcp.results import table_records
+
+log = logging.getLogger("bot")
+KEY_RE = re.compile(r"^(\S+) \((.*)\)$")
+
+
+async def current_holdings(server, portfolio_names: list[str]) -> dict[str, str]:
+    """Current holdings across the named portfolios, as {"CODE (MARKET)": name}."""
+    data = json.loads(await server.call("list_portfolios", {}))
+    portfolios = {p["name"].lower(): p for p in data.get("portfolios", [])}
+    holdings: dict[str, str] = {}
+    for name in portfolio_names:
+        p = portfolios.get(name)
+        if not p:
+            log.warning("No Sharesight portfolio called %r (have: %s)", name, ", ".join(sorted(portfolios)))
+            continue
+        report = json.loads(await server.call(
+            "get_performance_report", {"portfolio_id": p["id"], "include_sales": False})).get("report") or {}
+        for h in table_records(report.get("holdings")):
+            if h.get("code"):
+                holdings[f"{h['code']} ({h.get('market', '?')})"] = h.get("name") or h["code"]
+    return holdings
+
+
+def code_and_market(key: str) -> tuple[str, str]:
+    """"NXT (ASX)" -> ("NXT", "ASX")."""
+    m = KEY_RE.match(key)
+    return (m.group(1), m.group(2)) if m else (key, "")
diff --git a/lib/prompts.py b/lib/prompts.py
index 0cd5234..2197c5b 100644
--- a/lib/prompts.py
+++ b/lib/prompts.py
@@ -12,7 +12,7 @@
 
 {tools_note}Each reply is final: never say you'll check later. Do the whole job in one go, with all lookups made first (several at once), and only ask a question if the request is genuinely ambiguous.{refresh}
 
-Format as Telegram HTML, never Markdown:       
 ; plain newlines, no headings or list tags; write & < > as & < >. If you mention a listed company, write its ticker as plain text, with the exchange suffix for non-US listings (SQX.AX, 000660.KS): links are added automatically. In a list, one company per line: Name (TICKER) metric. Shorten company names: drop Ltd, Inc, Corp, Co, plc, ADR, Holdings and a trailing Technology/Technologies (Micron Technology is Micron, DUG Technology is DUG Tech).
+Format as Telegram HTML, never Markdown:       
 ; plain newlines, no headings or list tags; write & < > as & < >. Write a listed company as Name (TICKER), or a bare ticker as $NVDA, with the Yahoo suffix for non-US listings (DUG Tech (DUG.AX), $SQX.AX, 000660.KS): only these forms get links, acronyms stay plain. In a list, one company per line: Name (TICKER) metric. Shorten company names: drop Ltd, Inc, Corp, Co, plc, ADR, Holdings and a trailing Technology/Technologies (Micron Technology is Micron, DUG Technology is DUG Tech).
 
 No disclaimers, no moralising. If your confidence is low, say so."""
 
diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py
index f3cbf74..c3c340d 100644
--- a/lib/telegram/handlers.py
+++ b/lib/telegram/handlers.py
@@ -368,6 +368,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None, char
         preset = dm_buttons.preset_for(trig.text) if trig.private and st.dm_buttons else None
         on_text = draft.update if draft else None
         chat_id = msg.chat_id
+        known = None
         if preset:  # a standalone request: no transcript, which would only add tokens
             prompt = PRESET_PROMPT.format(sender=sender_name(msg), label=trig.text.strip(), ask=preset,
                                           now=ctx.now(), what=ctx.backend.search_what)
@@ -377,6 +378,7 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None, char
                                kind="preset", chat_id=chat_id)
         elif trig.movers:
             listing = msg.text_html if msg.text else (msg.caption_html or "")
+            known = movers.listed_codes(listing)
             text = await self._chat_text(msg, trig, MOVERS_EXPLAIN.format(listing=listing), no_history=True)
             parts = [{"type": "text", "text": text}]
             answer = await ask(ctx, parts, route("", st, ctx.registry, force_full=True),
@@ -413,13 +415,15 @@ async def _generate(self, msg: Message, trig: Trigger, draft: Draft | None, char
             answer = await ask(ctx, parts, chosen,
                                cache_key=f"chat-{chat_id}", on_text=on_text, chat_id=chat_id, extra_tools=extra,
                                admin=admin)
-        return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers)
+        return format_answer(answer.text, movers_bots=st.movers_bots, summary=trig.movers, known=known)
 
 
-def format_answer(text: str, *, movers_bots: frozenset[str], summary: bool) -> str:
+def format_answer(text: str, *, movers_bots: frozenset[str], summary: bool,
+                  known: dict[str, str] | None = None) -> str:
     """The model's text as Telegram HTML: Markdown fallback, no disclaimer, no tags of movers
-    bots, Yahoo links on tickers (and, for a movers list, no closing wrap-up paragraph)."""
-    out = link_tickers(movers.untag(md_to_html(strip_disclaimer(text)), movers_bots))
+    bots, Yahoo links on tickers (`known`: codes to link bare as well, such as a movers list's;
+    and, for a movers list, no closing wrap-up paragraph)."""
+    out = link_tickers(movers.untag(md_to_html(strip_disclaimer(text)), movers_bots), known)
     if summary:
         out = movers.strip_summary(out)
     return out or NO_RESPONSE
diff --git a/lib/tickers.py b/lib/tickers.py
index 6f4e831..ad75a6c 100644
--- a/lib/tickers.py
+++ b/lib/tickers.py
@@ -1,12 +1,16 @@
-"""Reply filter: Yahoo links on tickers (the label hides the exchange suffix), bold "Name (TICKER)", raw links as [n]."""
+"""Reply filter: Yahoo links on marked tickers ($NVDA, NXT.AX, ASX:NXT, Name (NVDA)) and known ones, raw links as [n]."""
 
 from __future__ import annotations
 
 import re
+from collections.abc import Mapping
+from functools import lru_cache
 from urllib.parse import quote
 
-from .tickers_data import CRYPTO, EXCHANGE_SUFFIXES, NOT_TICKERS
+from .tickers_data import CRYPTO, EXCHANGE_PREFIXES, EXCHANGE_SUFFIXES, NOT_TICKERS, WORD_TICKERS
 
+# TICKER_RE, ONE_LETTER_RE and is_ticker read the user's text (llm/gate.py, charts/breakdown.py); replies link
+# only the marked forms below and known tickers.
 TICKER_RE = re.compile(r"\b([A-Z0-9]{1,6}\.[A-Z]{1,2}|[A-Z][A-Z0-9]{1,5})\b")  # 9988.HK, SQX.AX, ACMR
 # Single-letter tickers (U, F, X, T) only count next to a move or a price, so "vitamin C"
 # and "A big move" are left alone.
@@ -22,20 +26,11 @@
 _CITE = r'\[\d+\]'
 CITE_ONLY_RUN_RE = re.compile(rf"\s*\n\s*({_CITE}(?:\s+{_CITE})*)[ \t]*(?=\n|$)")  # lines holding only citations
 RAW_ANCHOR_RE =re.compile(r'\s*https?://[^<]*')
-# "Micron (MU)": up to four capitalised words (or a number) right before a parenthesised ticker.
-_WORD = r"[A-Z0-9][\w&'’-]*(?:\.[\w&'’-]+)*"
-NAME_TICKER_RE = re.compile(
-    rf"(? None:
-    """Add words (upper case) that are never to be linked as tickers: the NOT_TICKERS setting."""
-    NOT_TICKERS.update(words)
-
-
 def is_ticker(word: str) -> bool:
     base, _, suffix = word.partition(".")
     if suffix in EXCHANGE_SUFFIXES and base not in NOT_TICKERS:
@@ -54,20 +49,116 @@ def ticker_label(symbol: str) -> str:
     return base if suffix in EXCHANGE_SUFFIXES else symbol
 
 
-def bold_names(text: str) -> str:
-    """Bold "Name (TICKER)" (name and parenthesised ticker, one ), e.g. "Micron (MU)"."""
-    def wrap(m: re.Match) -> str:
-        name, symbol = m.group(1), m.group(2)
-        if len(symbol) > 1 and not is_ticker(symbol):
-            return m.group(0)
+# Known tickers a reply may name bare (code -> Yahoo symbol): the S&P 500, the NASDAQ-100 and US holdings, refreshed
+# daily by features/known_tickers.py and read from here, never fetched while answering.
+KNOWN: dict[str, str] = {}
+
+
+def set_known(codes: Mapping[str, str]) -> None:
+    KNOWN.clear()
+    KNOWN.update(codes)
+
+
+def yahoo_symbol(code: str, market: str) -> str | None:
+    """Yahoo's symbol for a code on an exchange ("NXT", "ASX" -> "NXT.AX"), or None for an exchange we don't know."""
+    suffix = EXCHANGE_PREFIXES.get(market.upper())
+    if suffix is None:
+        return None
+    if market.upper() == "TSE" and code.isdigit():
+        suffix = "T"  # Tokyo; Toronto's codes are letters
+    if suffix == "HK" and code.isdigit():
+        code = code.zfill(4)
+    return f"{code}.{suffix}" if suffix else code
+
+
+def _alt(words) -> str:
+    return "|".join(re.escape(w) for w in sorted(words, key=len, reverse=True))
+
+
+_CODE = r"(?:[A-Z][A-Z0-9]{0,5}(?:-[A-Z]{1,2})?|\d{3,6})"
+_PREFIX = _alt(EXCHANGE_PREFIXES)
+_SUFFIX = _alt(EXCHANGE_SUFFIXES)
+_WORD = r"[A-Z0-9][\w&'’-]*(?:\.[\w&'’-]+)*"
+_MARKERS = (
+    # "Micron (MU)", "NextDC (ASX:NXT)", "DUG Tech (DUG.AX)", "NXT (ASX)": up to four capitalised words, then a paren
+    rf"(?(?:{_WORD}[ ]){{0,3}}{_WORD})[ ]\((?P[A-Z0-9][A-Z0-9.:-]*(?:[ ][A-Z0-9.-]+)?)\)"
+    rf"|(?{_PREFIX}):[ ]?(?P{_CODE})"  # ASX:NXT, NASDAQ: NVDA
+    r"|\$(?P[A-Z][A-Z0-9]{1,5}(?:[.-][A-Z]{1,2}){0,2})"  # $NVDA, $SQX.AX, $BRK-B (never $A, $600, $1.2B)
+    rf"|(?P(?:[A-Z][A-Z0-9]{{1,5}}|\d{{3,6}})(?:-[A-Z]{{1,2}})?\.(?:{_SUFFIX}))"  # NXT.AX, 9988.HK, RCI-B.TO
+    r"|\^(?P[A-Z0-9]{1,8})"  # ^GSPC
+    r"|(?P[A-Z]{2,6})-USD"  # BTC-USD
+    r"{known})(?![\w-]|\.\w)"
+)
+
+
+@lru_cache(maxsize=16)
+def _link_re(codes: frozenset[str]) -> re.Pattern:
+    known = rf"|\b(?P{_alt(codes)})\b" if codes else ""
+    return re.compile(_MARKERS.replace("{known}", known))
+
+
+def _link(symbol: str, label: str) -> str:
+    return f'{label}'
+
+
+def _paren(paren: str, known: Mapping[str, str]) -> str | None:
+    """The Yahoo symbol of what "Name (...)" holds: EXCH:CODE, CODE.SUFFIX or a known ticker; None for anything else."""
+    exch, colon, code = paren.partition(":")
+    if colon:
+        code = code.strip()
+        return yahoo_symbol(code, exch) if re.fullmatch(_CODE, code) else None
+    base, _, suffix = paren.rpartition(".")
+    if base and suffix in EXCHANGE_SUFFIXES and re.fullmatch(_CODE, base) and _code_ok(base):
+        return paren
+    # A bare code says nothing about its exchange ("Goodman Group (GMG)" is GMG.AX), so only a known one is linked.
+    return known.get(paren)
+
+
+def _code_ok(base: str) -> bool:
+    """A code before an exchange suffix: numbers are fine (9988.HK), FY2026 and Q3 are not."""
+    return base.isdigit() or not (NOT_TICKER_RE.match(base) or base in NOT_TICKERS)
+
+
+def _wrap(m: re.Match, bold: bool, known: Mapping[str, str]) -> str:
+    if m.group("name"):
+        name, paren = m.group("name"), m.group("paren")
         words = name.split(" ")
+        if paren in EXCHANGE_PREFIXES:  # Sharesight's "NXT (ASX)": link the code, keep the exchange as text
+            code = words[-1]
+            symbol = yahoo_symbol(code, paren) if re.fullmatch(_CODE, code) else None
+            if symbol is None or (not code.isdigit() and not is_ticker(code) and code not in known):
+                return m.group(0)
+            return " ".join(words[:-1] + [_link(symbol, code)]) + f" ({paren})"
+        symbol = _paren(paren, known)
+        if symbol is None:
+            return m.group(0)
+        label = paren.partition(":")[2].strip() or ticker_label(paren)
         skip = 0
         while skip < len(words) - 1 and words[skip] in SENTENCE_OPENERS:
             skip += 1
         lead = " ".join(words[:skip]) + " " if skip else ""
-        return f"{lead}{' '.join(words[skip:])} ({symbol})"
-
-    return NAME_TICKER_RE.sub(wrap, text)
+        body = f"{' '.join(words[skip:])} ({_link(symbol, label)})"
+        return lead + (body if bold else f"{body}")
+    if m.group("exch"):
+        code = m.group("ecode")
+        symbol = yahoo_symbol(code, m.group("exch"))
+        return _link(symbol, code) if symbol else m.group(0)
+    if m.group("cash"):
+        code = m.group("cash")
+        base = re.split(r"[.-]", code)[0]
+        if code not in known and not is_ticker(base):
+            return m.group(0)  # $AUD, $US
+        return _link(known.get(code, code), ticker_label(code))
+    if m.group("suff"):
+        code = m.group("suff")
+        base = code.rsplit(".", 1)[0]
+        return _link(code, ticker_label(code)) if _code_ok(base) else m.group(0)
+    if m.group("idx"):
+        return _link("^" + m.group("idx"), "^" + m.group("idx"))
+    if m.group("cry"):
+        return _link(m.group("cry") + "-USD", m.group("cry"))
+    code = m.group("known")
+    return _link(known[code], code)
 
 
 def _rewrite(text: str, fn) -> str:
@@ -106,14 +197,11 @@ def cite(m: re.Match) -> str:
     return CITE_ONLY_RUN_RE.sub(lambda m: re.sub(r"(?<=)\s+", "", m.group(1)), text)
 
 
-def link_tickers(text: str) -> str:
-    """Number raw links, wrap ticker-shaped words in Yahoo links and bold "Name (TICKER)", leaving
-    tags and anything inside ,  or 
 alone (so a link the model already wrote is
-    never doubled), and adding no bold inside text that is already bold."""
-    def wrap(m: re.Match) -> str:
-        word = m.group(0)
-        if len(word) > 1 and not is_ticker(word):
-            return word
-        return f'{ticker_label(word)}'
-
-    return _rewrite(number_links(text), lambda seg, bold: COMBINED_RE.sub(wrap, seg if bold else bold_names(seg)))
+def link_tickers(text: str, known: Mapping[str, str] | None = None) -> str:
+    """Number raw links, then link the tickers written in a marked form ($NVDA, NXT.AX, ASX:NXT, Name (NVDA), NXT (ASX),
+    ^GSPC, BTC-USD) and the bare words that are known tickers (KNOWN, plus `known`: code -> Yahoo symbol), bolding
+    "Name (TICKER)" unless it is already bold. Any other capitalised word stays plain, so acronyms are never linked.
+    Tags and anything inside ,  or 
 are left alone, so a link the model wrote is never doubled."""
+    codes = {**KNOWN, **(known or {})}
+    pattern = _link_re(frozenset(c for c in codes if len(c) > 1 and c not in NOT_TICKERS | WORD_TICKERS))
+    return _rewrite(number_links(text), lambda seg, bold: pattern.sub(lambda m: _wrap(m, bold, codes), seg))
diff --git a/lib/tickers_data.py b/lib/tickers_data.py
index 9549d69..8e86c4a 100644
--- a/lib/tickers_data.py
+++ b/lib/tickers_data.py
@@ -1,6 +1,7 @@
 """Data for ticker linking: words that look like tickers but aren't, exchange suffixes, crypto."""
 
-# Words shaped like tickers that aren't tickers.
+# Words shaped like tickers that aren't tickers. Replies never link a bare word unless it is a known ticker
+# (tickers.KNOWN), so this list only keeps the tool gate, the /segments fast path and "Name (WORD)" honest.
 NOT_TICKERS = {
     "AI", "AM", "PM", "AND", "THE", "NOT", "BUT", "FOR", "ALL", "NEW", "OLD", "OK",
     "US", "USA", "UK", "EU", "AU", "CN", "UTC", "AEST", "AEDT", "GMT",
@@ -13,6 +14,23 @@
     "NOTE", "EDIT", "TLDR", "FYI", "IMO", "IMHO", "AKA", "ETA", "VS", "PS",
     "VR", "AR", "XR", "API", "GPU", "CPU", "TPU", "HBM", "DRAM", "NAND", "EUV", "OS",
     "U.S", "U.K", "P.A", "E.G", "I.E", "SMSF", "SUPER", "PNL",
+    "GAAP", "EBITDA", "CAGR", "ARPU", "AFR", "NAV", "AUM", "DPS", "VWAP", "SMA", "EMA", "MACD", "LLM", "AGI",
+    "WSJ", "NZD", "CAD", "CHF",
+}
+
+# Index tickers that are everyday words in capitals ("IT spending", "ALL in"): linked only as $IT or Name (IT).
+WORD_TICKERS = {"IT", "ALL", "ON", "NOW", "KEY", "LOW", "ARE", "DAY", "HAS", "WELL", "BALL", "FAST", "CAT", "AMP"}
+
+# An exchange as the model, the press or Sharesight writes it ("ASX:NXT", "NXT (ASX)") -> Yahoo's suffix ("" = US,
+# none). TSE is Toronto for letters and Tokyo for numbers (tickers.yahoo_symbol); HK numbers are padded to 4 digits.
+EXCHANGE_PREFIXES = {
+    "ASX": "AX", "NZX": "NZ", "LSE": "L", "LON": "L", "TSX": "TO", "TSE": "TO", "TSXV": "V", "CVE": "V",
+    "HKEX": "HK", "HKG": "HK", "SEHK": "HK", "TYO": "T", "JPX": "T", "KRX": "KS", "KOSDAQ": "KQ",
+    "XETRA": "DE", "ETR": "DE", "FRA": "F", "EPA": "PA", "AMS": "AS", "EBR": "BR", "SWX": "SW", "SIX": "SW",
+    "BIT": "MI", "BME": "MC", "STO": "ST", "OSL": "OL", "CPH": "CO", "HEL": "HE", "SGX": "SI", "SHA": "SS",
+    "SHE": "SZ", "TPE": "TW", "BOM": "BO", "NSE": "NS", "BVMF": "SA", "BMV": "MX", "JSE": "JO", "TLV": "TA",
+    "NYSE": "", "NASDAQ": "", "NYSEARCA": "", "NYSEAMERICAN": "", "AMEX": "", "OTC": "", "OTCMKTS": "",
+    "CBOE": "", "BATS": "",
 }
 
 # Yahoo needs the exchange suffix in the URL, but it's noise in the chat: link SQX.AX, show SQX.
@@ -20,7 +38,7 @@
 EXCHANGE_SUFFIXES = {
     "AX", "L", "TO", "V", "NZ", "HK", "SS", "SZ", "T", "KS", "DE",
     "PA", "AS", "MI", "MC", "ST", "OL", "SI", "BO", "NS", "SA", "MX",
-}
+} | {suffix for suffix in EXCHANGE_PREFIXES.values() if suffix}
 
 # Yahoo quotes crypto as BTC-USD, not BTC.
 CRYPTO = {
diff --git a/tests/test_features.py b/tests/test_features.py
index dbf78dd..0fe79ec 100644
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -67,6 +67,22 @@ async def test_daily_digest_dms_news_with_links_and_records_it(env, store):
     assert ctx.backend.seen[0]["tools"] == [] and ctx.backend.seen[0]["search"] is True  # forced search only
 
 
+async def test_digest_links_held_codes_however_the_model_writes_them(env, store):
+    ctx = holding_ctx(env, store, [step("• Nvidia (NVDA): beat estimates\n• AMD fell, GAAP loss")])
+    text, codes = (await holding_news.run_holding_news(ctx))["bob_llama"]
+    assert 'Nvidia (NVDA)' in text
+    assert 'quote/AMD"' in text and "quote/GAAP" not in text and codes == ["AMD", "NVDA"]
+
+
+def test_known_symbols_and_listed_codes_give_yahoo_symbols():
+    held = {"NXT (ASX)": "NextDC", "NVDA (NASDAQ)": "Nvidia", "X (MOON)": "x"}
+    assert holding_news.known_symbols(held) == {"NXT": "NXT.AX", "NVDA": "NVDA", "X": "X"}
+    assert movers.listed_codes("≥ 5.0% at close (ASX):\nDRO +12.1%\nNXT +5.5%") == {"DRO": "DRO.AX", "NXT": "NXT.AX"}
+    assert movers.listed_codes("≤ -5% at close (NASDAQ, NYSE):\nNVDA -6.1%") == {"NVDA": "NVDA"}
+    assert movers.listed_codes("≥ 5% at close (ASX, NYSE):\nBHP +6%") == {}  # which exchange? no guess
+    assert movers.listed_codes("≥ 5% at close:\nBHP +6%") == {}
+
+
 async def test_digest_says_nothing_when_nothing_and_respects_mutes(env, store):
     ctx = holding_ctx(env, store, [step("NOTHING")])
     assert (await holding_news.run_holding_news(ctx)) == {"bob_llama": None} and ctx.bot.sent == []
diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py
index bf3cc43..b690752 100644
--- a/tests/test_fixtures.py
+++ b/tests/test_fixtures.py
@@ -7,7 +7,7 @@
 import pytest
 
 from lib import config
-from lib.features import holding_news
+from lib import holdings as sharesight_holdings
 from lib.mcp.results import slim_result, table_records
 from lib.mcp.schema import compact_description, compact_schema
 
@@ -67,12 +67,12 @@ async def call(self, tool, args):
 
 
 async def test_holding_news_reads_current_holdings_from_real_shaped_reports():
-    holdings = await holding_news.current_holdings(
+    holdings = await sharesight_holdings.current_holdings(
         SharesightFixtures("sharesight", tools=("list_portfolios",)), ["bobsmsf", "alicesmsf"])
     assert holdings["GNP (ASX)"] == "Genusplus Group" and "NVDA (NASDAQ)" in holdings
-    only_alice = await holding_news.current_holdings(SharesightFixtures("sharesight"), ["alicesmsf"])
+    only_alice = await sharesight_holdings.current_holdings(SharesightFixtures("sharesight"), ["alicesmsf"])
     assert set(only_alice) < set(holdings) and "GNP (ASX)" not in only_alice
-    assert await holding_news.current_holdings(SharesightFixtures("sharesight"), ["no such portfolio"]) == {}
+    assert await sharesight_holdings.current_holdings(SharesightFixtures("sharesight"), ["no such portfolio"]) == {}
 
 
 def test_sharesight_tool_definitions_hide_the_parameters_the_model_never_needs():
diff --git a/tests/test_handlers.py b/tests/test_handlers.py
index e0635a8..c8e7ad1 100644
--- a/tests/test_handlers.py
+++ b/tests/test_handlers.py
@@ -26,13 +26,14 @@ async def run(h, msg, edited=False):
 
 
 async def test_group_mention_is_answered_with_linked_tickers_and_logged(env, store):
-    ctx, backend, h = make_ctx(env, store, [step("NVDA looks strong, SQX.AX too")])
+    ctx, backend, h = make_ctx(env, store, [step("$NVDA looks strong, SQX.AX too, GAAP not")])
     m = user_msg(ctx.bot, "@stockbot how is NVDA?")
     await run(h, m)
     assert len(m.replies) == 1
     text = m.replies[0]["text"]
     assert 'NVDA' in text
     assert 'SQX' in text and "NVDA" not in text
+    assert "quote/GAAP" not in text  # a bare acronym is never linked
     assert m.replies[0]["do_quote"] is True and m.replies[0]["disable_notification"] is True
     rows = store.history(-100, 20)
     assert [r.text for r in rows][0] == "@stockbot how is NVDA?" and len(rows) == 2
@@ -222,12 +223,15 @@ async def test_movers_list_off_by_default_then_answered_once_without_tagging(env
         assert len(m.replies) == expect
     assert backend.seen[0]["tools"] == [] and backend.seen[0]["search"] is True  # forced search only
     assert "Small caps" not in m.replies[0]["text"]  # closing wrap-up paragraph dropped
+    assert 'quote/NVDA.AX"' in m.replies[0]["text"]  # the list's "(ASX)" header gives the codes their suffix
 
 
 def test_format_answer():
-    out = format_answer("**NVDA** up, ping @finbotibot", movers_bots=frozenset({"finbotibot"}), summary=False)
+    out = format_answer("**$NVDA** up, ping @finbotibot", movers_bots=frozenset({"finbotibot"}), summary=False)
     assert "@finbotibot" not in out and "finbotibot" in out and 'quote/NVDA"' in out
     assert format_answer("", movers_bots=frozenset(), summary=False) == "(no response)"
+    assert "quote/" not in format_answer("NVDA up", movers_bots=frozenset(), summary=False)
+    assert 'quote/NVDA"' in format_answer("NVDA up", movers_bots=frozenset(), summary=False, known={"NVDA": "NVDA"})
 
 
 def test_mentions():
diff --git a/tests/test_known_tickers.py b/tests/test_known_tickers.py
new file mode 100644
index 0000000..00a4330
--- /dev/null
+++ b/tests/test_known_tickers.py
@@ -0,0 +1,90 @@
+import json
+
+import httpx
+
+from lib import config, tickers
+from lib.context import Ctx
+from lib.features import known_tickers as kt
+
+from .conftest import FakeBot
+from .fakes import FakeMcp, ScriptedBackend, registry
+
+
+def page(rows, table_id="constituents", header="Symbol"):
+    body = "".join(f"
" for s in rows)
+    return (f"
{s}Co {s}
Founded
"
+            f"{body}
{header}Security
")
+
+
+def test_the_ticker_column_is_read_and_share_classes_use_a_hyphen():
+    assert kt.symbols_from_html(page(["NVDA", "BRK.B", "bad one", "GOOGL"])) == ["NVDA", "BRK-B", "GOOGL"]
+    assert kt.symbols_from_html(page(["AAPL"], table_id="", header="Ticker")) == ["AAPL"]  # no id: by its header
+    assert kt.symbols_from_html("
Name
x
") == []
+
+
+class Sharesight(FakeMcp):
+    async def call(self, tool, args):
+        if tool == "list_portfolios":
+            return json.dumps({"portfolios": [{"name": "Bob", "id": 7}]})
+        rows = [["GM", "NYSE", "General Motors"], ["NXT", "ASX", "NextDC"], ["ZZZ", "NASDAQ", "Zed"]]
+        return json.dumps({"report": {"holdings": {"columns": ["code", "market", "name"], "rows": rows}}})
+
+
+def client(pages: dict[str, str]) -> httpx.AsyncClient:
+    def reply(request: httpx.Request) -> httpx.Response:
+        text = pages.get(str(request.url))
+        return httpx.Response(200, text=text) if text is not None else httpx.Response(503)
+    return httpx.AsyncClient(transport=httpx.MockTransport(reply))
+
+
+async def test_refresh_merges_the_indexes_and_the_us_holdings_and_keeps_a_list_that_fails(env, store, monkeypatch):
+    (sp, sp_url, _, _), (ndx, ndx_url, _, _) = config.INDEX_LISTS
+    monkeypatch.setattr(config, "INDEX_LISTS", ((sp, sp_url, 2, 5), (ndx, ndx_url, 2, 5)))
+    st = config.load({**env, "PORTFOLIO_NAMES": "Bob"})
+    ctx = Ctx(st, store, ScriptedBackend([]), registry(Sharesight("sharesight", gate="portfolio")), FakeBot())
+    before = dict(tickers.KNOWN)
+    try:
+        saved = await kt.refresh(ctx, {}, client({sp_url: page(["NVDA", "BRK.B"]), ndx_url: page(["NVDA", "AMD"])}))
+        assert saved["indexes"] == {sp: ["NVDA", "BRK-B"], ndx: ["NVDA", "AMD"]} and saved["held"] == ["GM", "ZZZ"]
+        assert set(tickers.KNOWN) == {"NVDA", "BRK-B", "AMD", "GM", "ZZZ"}  # NXT is ASX: not a bare US code
+        # the next day the S&P page is down and the NASDAQ page looks wrong: both keep yesterday's list
+        again = await kt.refresh(ctx, saved, client({ndx_url: page(["ONLY"])}))
+        assert again["indexes"] == saved["indexes"]
+        tickers.set_known({})
+        assert kt.load(ctx) == again and "BRK-B" in tickers.KNOWN  # a restart has them from the kv table at once
+    finally:
+        tickers.set_known(before)
+
+
+async def test_without_sharesight_the_held_list_stays_as_it_was(env, store):
+    ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), FakeBot())
+    assert await kt.us_holdings(ctx) is None
+
+
+async def test_the_loop_waits_for_the_mcp_when_overdue_and_otherwise_until_due(env, store, monkeypatch):
+    import asyncio
+    import time
+    ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), FakeBot())
+    waits, refreshed = [], []
+
+    async def sleep(seconds):
+        waits.append(seconds)
+        if len(waits) > 1:
+            raise asyncio.CancelledError
+
+    async def refresh(ctx, saved, client):
+        refreshed.append(saved)
+        return {**saved, "ts": int(time.time())}
+
+    monkeypatch.setattr(kt.asyncio, "sleep", sleep)
+    monkeypatch.setattr(kt, "refresh", refresh)
+    before = dict(tickers.KNOWN)
+    try:
+        try:
+            await kt.loop(ctx)
+        except asyncio.CancelledError:
+            pass
+    finally:
+        tickers.set_known(before)
+    assert waits[0] == config.KNOWN_TICKERS_START_DELAY and refreshed == [{}]  # nothing saved yet: refresh soon
+    assert config.KNOWN_TICKERS_REFRESH - 5 < waits[1] <= config.KNOWN_TICKERS_REFRESH  # then a day later
diff --git a/tests/test_tickers.py b/tests/test_tickers.py
index c7ee99f..dcd4a91 100644
--- a/tests/test_tickers.py
+++ b/tests/test_tickers.py
@@ -1,54 +1,101 @@
-from lib.tickers import link_tickers, ticker_label, yahoo_url
+from lib import tickers
+from lib.tickers import link_tickers, ticker_label, yahoo_symbol, yahoo_url
 
+Y = "https://finance.yahoo.com/quote/"
 
-def test_link_only_no_bold():
-    out = link_tickers("NVDA up 3%")
-    assert out == 'NVDA up 3%'
-    assert "" not in out
 
+def a(symbol, label):
+    return f'{label}'
 
-def test_label_hides_suffix_url_keeps_it():
-    out = link_tickers("SQX.AX and BRK.B and 9988.HK")
-    assert 'SQX' in out
-    assert 'BRK.B' in out
-    assert '9988' in out
+
+def test_bare_words_are_never_linked_so_acronyms_need_no_stoplist():
+    for text in ("non-GAAP EBITDA; AFR says GM, UPS, M3, VWAP, XJO, AT&T, S&P 500", "NVDA up 3%",
+                 "AI and CEO, Q3 FY26", "F 5.2 today and vitamin C", "BTC and LINK bare", "I.T. staff in the U.S.",
+                 "FY2026.T and Q3.AX", "NXT:ASX and NXT.ASX"):
+        assert link_tickers(text) == text
+
+
+def test_dollar_amounts_and_currencies_are_not_cashtags():
+    for text in ("$600, $12.AX, $30.5T, US$1.2T, $A1.2bn and $US2bn", "the $A fell; $AUD and $US weak"):
+        assert link_tickers(text) == text
+
+
+def test_cashtags_link_without_the_dollar_or_bold():
+    assert link_tickers("$NVDA up 3%") == f"{a('NVDA', 'NVDA')} up 3%"
+    assert link_tickers("$SQX.AX and $BRK-B") == f"{a('SQX.AX', 'SQX')} and {a('BRK-B', 'BRK-B')}"
+    assert link_tickers("$BTC rallied") == f"{a('BTC-USD', 'BTC')} rallied" and yahoo_url("BTC").endswith("BTC-USD")
+
+
+def test_exchange_suffix_links_with_the_suffix_hidden():
+    out = link_tickers("SQX.AX and BRK.B and 9988.HK and 000660.KS")
+    assert out == f"{a('SQX.AX', 'SQX')} and BRK.B and {a('9988.HK', '9988')} and {a('000660.KS', '000660')}"
+    assert link_tickers("RCI-B.TO") == a("RCI-B.TO", "RCI-B")
     assert ticker_label("BRK.B") == "BRK.B" and ticker_label("SQX.AX") == "SQX"
 
 
-def test_crypto_gets_usd_in_url_only():
-    assert yahoo_url("BTC").endswith("BTC-USD")
-    assert '>BTC' in link_tickers("BTC rallied")
+def test_exchange_prefix_maps_to_the_yahoo_suffix():
+    assert link_tickers("up on ASX:NXT.") == f"up on {a('NXT.AX', 'NXT')}."
+    assert link_tickers("(ASX: NXT)") == f"({a('NXT.AX', 'NXT')})"
+    assert link_tickers("NASDAQ: NVDA, LON:BP, NYSE:F") == f"{a('NVDA', 'NVDA')}, {a('BP.L', 'BP')}, {a('F', 'F')}"
+    assert link_tickers("TSE:7203 TSE:SHOP HKG:700") == f"{a('7203.T', '7203')} {a('SHOP.TO', 'SHOP')} {a('0700.HK', '700')}"
+    assert yahoo_symbol("NXT", "asx") == "NXT.AX" and yahoo_symbol("NVDA", "NASDAQ") == "NVDA"
+    assert yahoo_symbol("X", "MOON") is None
 
 
-def test_protected_markup_is_left_alone():
-    assert link_tickers('NVDA and AI') == 'NVDA and AI'
-    assert link_tickers("
NVDA 12.3
 AMD").startswith("
NVDA 12.3
 NVDA") == "NVDA"
-    # a ticker the model bolded still gets linked (inside the bold)
-    assert link_tickers("AMD").startswith("Micron ({a('MU', 'MU')}) +3.1%" in out and f"1. Trade Desk ({a('TTD', 'TTD')}) -5%" in out
+    assert link_tickers("DUG Tech (DUG.AX) fell") == f"DUG Tech ({a('DUG.AX', 'DUG')}) fell"
+    assert link_tickers("NextDC (ASX:NXT) +5%") == f"NextDC ({a('NXT.AX', 'NXT')}) +5%"
+    assert link_tickers("Nvidia (NASDAQ: NVDA)") == f"Nvidia ({a('NVDA', 'NVDA')})"
+    assert link_tickers("SK Hynix (000660.KS)") == f"SK Hynix ({a('000660.KS', '000660')})"
+    assert link_tickers("Ford (F) fell", known) == f"Ford ({a('F', 'F')}) fell"
 
-def test_stoplist_and_one_letter():
-    assert link_tickers("AI and CEO, Q3 FY26") == "AI and CEO, Q3 FY26"
-    assert 'quote/F"' in link_tickers("F 5.2 today")
-    assert link_tickers("vitamin C helps") == "vitamin C helps"
 
+def test_a_bare_code_in_brackets_links_only_when_known_since_it_names_no_exchange():
+    text = "Goodman Group (GMG) and Digital Realty (DLR)"
+    assert link_tickers(text) == text  # GMG is GMG.AX: a US link would be wrong
+    assert link_tickers(text, {"DLR": "DLR"}) == f"Goodman Group (GMG) and Digital Realty ({a('DLR', 'DLR')})"
+    for text in ("the Fed (RBA) held and vitamin C (CEO)", "Australian Financial Review (AFR) reports"):
+        assert link_tickers(text, {"MU": "MU"}) == text
 
-def test_company_name_and_ticker_are_bolded_together():
-    out = link_tickers("• Micron (MU) +3.1%\n1. Trade Desk (TTD) -5%")
-    assert '• Micron (MU) +3.1%' in out
-    assert '1. Trade Desk (TTD) -5%' in out
-    ext = link_tickers("DUG Tech (DUG.AX) fell")
-    assert ext.startswith("") and ">DUG) fell" in ext and 'quote/DUG.AX"' in ext
 
+def test_bolding_leaves_sentence_openers_and_existing_bold_alone():
+    known = {"MU": "MU", "NVDA": "NVDA"}
+    assert link_tickers("Today Micron (MU) ripped", known).startswith("Today Micron (")
+    assert link_tickers("Micron (MU) up", known) == f"Micron ({a('MU', 'MU')}) up"
+    assert link_tickers("Micron (MU)", known) == "Micron (MU)"
+    assert link_tickers("Micron (MU)\nNvidia (NVDA)", known).count("") == 2  # a line break ends a name
 
-def test_bolding_leaves_sentence_openers_existing_bold_and_non_tickers_alone():
-    assert link_tickers("Today Micron (MU) ripped").startswith("Today Micron (")
-    assert link_tickers("Micron (MU) up").count("") == 1
-    assert link_tickers("Micron (MU)").count("") == 1
-    assert "" not in link_tickers("the Fed (RBA) held and vitamin C (CEO)")
-    assert "" not in link_tickers("Micron (MU)")
-    assert link_tickers("Micron (MU)\nNvidia (NVDA)").count("") == 2  # a line break ends a name
+
+def test_sharesight_code_and_market_links_the_code_and_keeps_the_exchange():
+    assert link_tickers("• NVDA (NASDAQ): beat") == f"• {a('NVDA', 'NVDA')} (NASDAQ): beat"
+    assert link_tickers("• NXT (ASX): halt") == f"• {a('NXT.AX', 'NXT')} (ASX): halt"
+    assert link_tickers("Nvidia (NASDAQ) fell") == "Nvidia (NASDAQ) fell"
+
+
+def test_known_tickers_link_bare_but_not_single_letters_or_everyday_words():
+    known = {"NVDA": "NVDA", "GM": "GM", "IT": "IT", "A": "A", "ALL": "ALL"}
+    out = link_tickers("NVDA up, GM too; IT spending, A big move, ALL in", known)
+    assert out == f"{a('NVDA', 'NVDA')} up, {a('GM', 'GM')} too; IT spending, A big move, ALL in"
+    before = dict(tickers.KNOWN)
+    try:
+        tickers.set_known({"NVDA": "NVDA"})
+        assert link_tickers("NVDA and AMD") == f"{a('NVDA', 'NVDA')} and AMD"  # the daily list, with no `known` passed
+    finally:
+        tickers.set_known(before)
+
+
+def test_protected_markup_is_left_alone():
+    assert link_tickers('NVDA and $AMD') == f'NVDA and {a("AMD", "AMD")}'
+    assert link_tickers("
$NVDA 12.3
 $AMD").startswith("
$NVDA 12.3
 $NVDA") == "$NVDA"
+    assert link_tickers("$AMD") == f"{a('AMD', 'AMD')}"
 
 
 def test_raw_links_go_behind_citation_numbers():
@@ -70,18 +117,3 @@ def test_link_labelled_with_its_own_url_is_numbered_but_named_links_and_code_are
     assert link_tickers('https://y.com/p… ok') == '[1] ok'
     assert link_tickers('Reuters') == 'Reuters'
     assert link_tickers("https://x.com") == "https://x.com"
-
-
-def test_not_tickers_setting_adds_words_that_are_never_linked(env):
-    from lib import config, tickers
-    st = config.load({**env, "NOT_TICKERS": "gaap, GM  ups"})
-    assert st.not_tickers == {"GAAP", "GM", "UPS"} and config.load(env).not_tickers == frozenset()
-    before = set(tickers.NOT_TICKERS)
-    try:
-        assert "GAAP" in tickers.link_tickers("non-GAAP EPS at GM")
-        tickers.exclude(st.not_tickers)
-        out = tickers.link_tickers("non-GAAP EPS at GM, UPS and NVDA")
-        assert "GAAP" in out and ">GAAP<" not in out and ">GM<" not in out and ">UPS<" not in out and ">NVDA<" in out
-    finally:
-        tickers.NOT_TICKERS.clear()
-        tickers.NOT_TICKERS.update(before)

From 3e75889d51c9de2180599cd93f8fa15337fe50d7 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 06:49:56 +0000
Subject: [PATCH 19/30] Ticker links: fixes from an adversarial review

- An exchange suffix links on its own: ALL.AX (Aristocrat), ASX.AX, AI.PA were left plain because
  the code is also on the acronym list.
- A bracket that isn't a ticker ("NVDA (Q3 FY26)") no longer swallows the ticker before it.
- Share classes: BRK.B as the model writes it links to Yahoo's BRK-B.
- "AT&T (T)" and "Procter & Gamble (PG)" are bolded whole.
- Numeric holding codes (700 on HKEX) never link bare numbers, and keep their unsubscribe button.
- Index tickers that are also finance or tech acronyms (GM, UPS, IR, HD, USB, RF...) link only in a
  marked form, as single letters already did.
- A refresh part that fails (a page down, Sharesight not up yet) is tried again in 30 minutes,
  not 24 hours; a footnote in the table header no longer hides the ticker column.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 docs/how-it-works.md          |  6 ++++--
 lib/config.py                 |  1 +
 lib/features/holding_news.py  | 12 +++++++++---
 lib/features/known_tickers.py | 23 +++++++++++++++--------
 lib/tickers.py                | 34 +++++++++++++++++++++-------------
 lib/tickers_data.py           |  9 +++++++--
 tests/test_features.py        |  3 +++
 tests/test_known_tickers.py   |  8 +++++++-
 tests/test_tickers.py         | 32 +++++++++++++++++++++++++++++---
 9 files changed, 96 insertions(+), 32 deletions(-)

diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index 667276f..ef011fd 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -110,8 +110,10 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf
   `NXT (ASX)`, `^GSPC` and `BTC-USD`. The system prompt asks the model for these forms. A bare word is linked only
   when it is a known ticker: the S&P 500 and NASDAQ-100 constituents (read from their Wikipedia pages) and the US
   Sharesight holdings, refreshed once a day in the background and kept in the database, so a reply never waits for
-  them. Any other capitalised word stays plain (`GAAP`, `EBITDA`, `AFR`, `M3`), so there is no stoplist to maintain;
-  single letters and index tickers that are everyday words (`IT`, `ALL`, `NOW`) need `$` or `Name (TICKER)`. A bare
+  them. Any other capitalised word stays plain (`GAAP`, `EBITDA`, `AFR`, `M3`), so there is no stoplist to maintain. An
+  exchange suffix settles it on its own (`ALL.AX` is Aristocrat);
+  single letters and index tickers that are also everyday words or acronyms (`IT`, `ALL`, `GM`, `UPS`, `HD`) need `$`,
+  an exchange prefix or `Name (TICKER)`. A part of the refresh that fails is tried again in half an hour. A bare
   code in brackets names no exchange, so `Goodman Group (GMG)` is linked only if `GMG` is known (it is `GMG.AX`).
   A holding-news digest also links the holdings' own codes, and a movers explanation the listed codes with the list's
   exchange. The visible text drops `$`, the exchange prefix and the suffix (`NXT.AX` shows as `NXT`; `BRK.B` and
diff --git a/lib/config.py b/lib/config.py
index 4be4471..3479ce7 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -63,6 +63,7 @@
     ("NASDAQ-100", "https://en.wikipedia.org/wiki/Nasdaq-100", 90, 110),
 )
 KNOWN_TICKERS_REFRESH = 24 * 3600  # seconds between refreshes of the indexes and the US holdings
+KNOWN_TICKERS_RETRY = 30 * 60  # seconds before a failed part (a page down, Sharesight not up yet) is tried again
 KNOWN_TICKERS_START_DELAY = 60  # seconds after startup before a due refresh, so the MCP servers are up
 SOURCES_MAX = 3  # links to a search result the model is handed back as text (OpenRouter run_search)
 MAX_TG_MESSAGE = 4000  # Telegram's limit is 4096
diff --git a/lib/features/holding_news.py b/lib/features/holding_news.py
index 1216647..8b7211c 100644
--- a/lib/features/holding_news.py
+++ b/lib/features/holding_news.py
@@ -52,12 +52,18 @@ def known_symbols(holdings: dict[str, str]) -> dict[str, str]:
 
 
 def news_codes(news: str, holdings: dict[str, str]) -> list[str]:
-    """Holding codes that appear as bold tickers or Yahoo links in a news message."""
-    held = {k.split(" ")[0].upper() for k in holdings}
+    """Holding codes that appear as bold tickers or Yahoo links in a news message (as the holdings write them: an HK
+    holding "700" is linked as 0700.HK)."""
+    held = {_plain_code(k.split(" ")[0]): k.split(" ")[0].upper() for k in holdings}
     found = {m.upper() for m in re.findall(r"([^<]{1,12})", news)}
     found |= {m.split(".")[0].upper()
               for m in re.findall(r"finance\.yahoo\.com/quote/([\w.\-^]+)", news)}
-    return sorted(held & found)
+    return sorted({held[c] for c in map(_plain_code, found) if c in held})
+
+
+def _plain_code(code: str) -> str:
+    code = code.upper()
+    return code.lstrip("0") or "0" if code.isdigit() else code
 
 
 def unsubscribe_menu(codes: list[str]) -> InlineKeyboardMarkup:
diff --git a/lib/features/known_tickers.py b/lib/features/known_tickers.py
index de1173c..043e78c 100644
--- a/lib/features/known_tickers.py
+++ b/lib/features/known_tickers.py
@@ -65,7 +65,7 @@ def symbols_from_html(page: str) -> list[str]:
         rows = [["".join(cell).strip() for cell in row] for row in table["rows"]]
         if not rows:
             continue
-        header = [h.lower() for h in rows[0]]
+        header = [re.sub(r"\[.*?\]", "", h).strip().lower() for h in rows[0]]  # "Ticker[12]": a footnote
         col = next((i for i, h in enumerate(header) if h in ("symbol", "ticker", "ticker symbol")), None)
         if col is None:
             continue
@@ -105,9 +105,10 @@ async def us_holdings(ctx: Ctx) -> list[str] | None:
 
 
 def apply(saved: dict) -> None:
-    """Make the saved lists the ones replies link."""
+    """Make the saved lists the ones replies link. Yahoo writes share classes BRK-B; the model may write BRK.B."""
     codes = {c for name, _, _, _ in config.INDEX_LISTS for c in saved.get("indexes", {}).get(name, [])}
-    tickers.set_known({c: c for c in codes | set(saved.get("held", []))})
+    codes |= {c.replace(".", "-") for c in saved.get("held", [])}
+    tickers.set_known({**{c: c for c in codes}, **{c.replace("-", "."): c for c in codes if "-" in c}})
 
 
 def load(ctx: Ctx) -> dict:
@@ -121,14 +122,20 @@ def load(ctx: Ctx) -> dict:
 
 
 async def refresh(ctx: Ctx, saved: dict, client: httpx.AsyncClient) -> dict:
-    """Fetch the indexes and the holdings; a part that fails keeps its last list."""
-    indexes = dict(saved.get("indexes", {}))
+    """Fetch the indexes and the holdings; a part that fails keeps its last list and is tried again sooner
+    (KNOWN_TICKERS_RETRY) than the daily refresh."""
+    indexes, failed = dict(saved.get("indexes", {})), False
     for name, url, low, high in config.INDEX_LISTS:
         found = await fetch_index(client, name, url, low, high)
-        if found is not None:
+        if found is None:
+            failed = True
+        else:
             indexes[name] = found
     held = await us_holdings(ctx)
-    new = {"indexes": indexes, "held": saved.get("held", []) if held is None else held, "ts": int(time.time())}
+    failed |= held is None and bool(ctx.st.portfolio_names) and any(
+        s.cfg.get("gate") == "portfolio" for s in ctx.registry.servers.values())
+    ts = int(time.time()) - (config.KNOWN_TICKERS_REFRESH - config.KNOWN_TICKERS_RETRY if failed else 0)
+    new = {"indexes": indexes, "held": saved.get("held", []) if held is None else held, "ts": ts}
     await asyncio.to_thread(ctx.store.kv_set, KV_KEY, json.dumps(new))
     apply(new)
     log.info("Known tickers: %s, %d held, %d in all", ", ".join(f"{len(v)} {k}" for k, v in indexes.items()) or
@@ -147,4 +154,4 @@ async def loop(ctx: Ctx) -> None:
                 saved = await refresh(ctx, saved, client)
             except Exception:
                 log.exception("Known tickers: refresh failed")
-                saved = {**saved, "ts": int(time.time())}  # try again tomorrow, not in a tight loop
+                saved = {**saved, "ts": int(time.time()) - config.KNOWN_TICKERS_REFRESH + config.KNOWN_TICKERS_RETRY}
diff --git a/lib/tickers.py b/lib/tickers.py
index ad75a6c..76fad47 100644
--- a/lib/tickers.py
+++ b/lib/tickers.py
@@ -78,10 +78,10 @@ def _alt(words) -> str:
 _CODE = r"(?:[A-Z][A-Z0-9]{0,5}(?:-[A-Z]{1,2})?|\d{3,6})"
 _PREFIX = _alt(EXCHANGE_PREFIXES)
 _SUFFIX = _alt(EXCHANGE_SUFFIXES)
-_WORD = r"[A-Z0-9][\w&'’-]*(?:\.[\w&'’-]+)*"
+_WORD = r"[A-Z0-9](?:&|[\w&'’-])*(?:\.(?:&|[\w&'’-])+)*"  # AT&T, S&P
 _MARKERS = (
     # "Micron (MU)", "NextDC (ASX:NXT)", "DUG Tech (DUG.AX)", "NXT (ASX)": up to four capitalised words, then a paren
-    rf"(?(?:{_WORD}[ ]){{0,3}}{_WORD})[ ]\((?P[A-Z0-9][A-Z0-9.:-]*(?:[ ][A-Z0-9.-]+)?)\)"
+    rf"(?(?:(?:{_WORD}|&)[ ]){{0,3}}{_WORD})[ ]\((?P[A-Z0-9][A-Z0-9.:-]*(?:[ ][A-Z0-9.-]+)?)\)"
     rf"|(?{_PREFIX}):[ ]?(?P{_CODE})"  # ASX:NXT, NASDAQ: NVDA
     r"|\$(?P[A-Z][A-Z0-9]{1,5}(?:[.-][A-Z]{1,2}){0,2})"  # $NVDA, $SQX.AX, $BRK-B (never $A, $600, $1.2B)
     rf"|(?P(?:[A-Z][A-Z0-9]{{1,5}}|\d{{3,6}})(?:-[A-Z]{{1,2}})?\.(?:{_SUFFIX}))"  # NXT.AX, 9988.HK, RCI-B.TO
@@ -111,27 +111,31 @@ def _paren(paren: str, known: Mapping[str, str]) -> str | None:
     if base and suffix in EXCHANGE_SUFFIXES and re.fullmatch(_CODE, base) and _code_ok(base):
         return paren
     # A bare code says nothing about its exchange ("Goodman Group (GMG)" is GMG.AX), so only a known one is linked.
-    return known.get(paren)
+    return known.get(paren) or known.get(paren.replace(".", "-"))
 
 
 def _code_ok(base: str) -> bool:
-    """A code before an exchange suffix: numbers are fine (9988.HK), FY2026 and Q3 are not."""
-    return base.isdigit() or not (NOT_TICKER_RE.match(base) or base in NOT_TICKERS)
+    """A code before an exchange suffix: the suffix settles it (ALL.AX is Aristocrat), but FY2026 and Q3 are not codes."""
+    return base.isdigit() or not NOT_TICKER_RE.match(base)
 
 
-def _wrap(m: re.Match, bold: bool, known: Mapping[str, str]) -> str:
+def _wrap(m: re.Match, bold: bool, known: Mapping[str, str], pattern: re.Pattern) -> str:
     if m.group("name"):
         name, paren = m.group("name"), m.group("paren")
         words = name.split(" ")
+
+        def plain() -> str:  # not a ticker in brackets ("NVDA (Q3 FY26)"): the words before it may still hold one
+            return pattern.sub(lambda m2: _wrap(m2, bold, known, pattern), name) + m.group(0)[len(name):]
+
         if paren in EXCHANGE_PREFIXES:  # Sharesight's "NXT (ASX)": link the code, keep the exchange as text
             code = words[-1]
             symbol = yahoo_symbol(code, paren) if re.fullmatch(_CODE, code) else None
             if symbol is None or (not code.isdigit() and not is_ticker(code) and code not in known):
-                return m.group(0)
+                return plain()
             return " ".join(words[:-1] + [_link(symbol, code)]) + f" ({paren})"
         symbol = _paren(paren, known)
         if symbol is None:
-            return m.group(0)
+            return plain()
         label = paren.partition(":")[2].strip() or ticker_label(paren)
         skip = 0
         while skip < len(words) - 1 and words[skip] in SENTENCE_OPENERS:
@@ -145,10 +149,13 @@ def _wrap(m: re.Match, bold: bool, known: Mapping[str, str]) -> str:
         return _link(symbol, code) if symbol else m.group(0)
     if m.group("cash"):
         code = m.group("cash")
-        base = re.split(r"[.-]", code)[0]
-        if code not in known and not is_ticker(base):
+        base, _, suffix = code.rpartition(".")
+        if suffix in EXCHANGE_SUFFIXES and base:
+            return _link(code, ticker_label(code)) if _code_ok(base) else m.group(0)  # $ALL.AX
+        symbol = known.get(code) or known.get(code.replace(".", "-"))
+        if symbol is None and not is_ticker(re.split(r"[.-]", code)[0]):
             return m.group(0)  # $AUD, $US
-        return _link(known.get(code, code), ticker_label(code))
+        return _link(symbol or code.replace(".", "-"), code)
     if m.group("suff"):
         code = m.group("suff")
         base = code.rsplit(".", 1)[0]
@@ -203,5 +210,6 @@ def link_tickers(text: str, known: Mapping[str, str] | None = None) -> str:
     "Name (TICKER)" unless it is already bold. Any other capitalised word stays plain, so acronyms are never linked.
     Tags and anything inside ,  or 
 are left alone, so a link the model wrote is never doubled."""
     codes = {**KNOWN, **(known or {})}
-    pattern = _link_re(frozenset(c for c in codes if len(c) > 1 and c not in NOT_TICKERS | WORD_TICKERS))
-    return _rewrite(number_links(text), lambda seg, bold: pattern.sub(lambda m: _wrap(m, bold, codes), seg))
+    bare = frozenset(c for c in codes if len(c) > 1 and not c.isdigit() and c not in NOT_TICKERS | WORD_TICKERS)
+    pattern = _link_re(bare)
+    return _rewrite(number_links(text), lambda seg, bold: pattern.sub(lambda m: _wrap(m, bold, codes, pattern), seg))
diff --git a/lib/tickers_data.py b/lib/tickers_data.py
index 8e86c4a..008435c 100644
--- a/lib/tickers_data.py
+++ b/lib/tickers_data.py
@@ -18,8 +18,13 @@
     "WSJ", "NZD", "CAD", "CHF",
 }
 
-# Index tickers that are everyday words in capitals ("IT spending", "ALL in"): linked only as $IT or Name (IT).
-WORD_TICKERS = {"IT", "ALL", "ON", "NOW", "KEY", "LOW", "ARE", "DAY", "HAS", "WELL", "BALL", "FAST", "CAT", "AMP"}
+# Index tickers that are also everyday words or acronyms in capitals ("IT spending", "GM expanded 200bp", "UPS demand",
+# "HD maps"): never linked bare, only as $GM, NYSE:GM or General Motors (GM). A fixed set: the indexes don't grow.
+WORD_TICKERS = {
+    "IT", "ALL", "ON", "NOW", "KEY", "LOW", "ARE", "DAY", "HAS", "WELL", "BALL", "FAST", "CAT", "AMP", "SO", "ED",
+    "GM", "UPS", "IP", "IR", "ICE", "PTC", "RF", "RL", "USB", "HD", "TECH", "EA", "CE", "MAR", "DD", "CF", "MS",
+    "FANG", "MSCI", "COST", "BR", "GE", "HP", "PH", "TT", "CTA",
+}
 
 # An exchange as the model, the press or Sharesight writes it ("ASX:NXT", "NXT (ASX)") -> Yahoo's suffix ("" = US,
 # none). TSE is Toronto for letters and Tokyo for numbers (tickers.yahoo_symbol); HK numbers are padded to 4 digits.
diff --git a/tests/test_features.py b/tests/test_features.py
index 0fe79ec..a38bc4a 100644
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -77,6 +77,9 @@ async def test_digest_links_held_codes_however_the_model_writes_them(env, store)
 def test_known_symbols_and_listed_codes_give_yahoo_symbols():
     held = {"NXT (ASX)": "NextDC", "NVDA (NASDAQ)": "Nvidia", "X (MOON)": "x"}
     assert holding_news.known_symbols(held) == {"NXT": "NXT.AX", "NVDA": "NVDA", "X": "X"}
+    hk = {"700 (HKG)": "Tencent"}
+    news = holding_news.link_tickers("• Tencent (700): profit up; 1,700 staff", known=holding_news.known_symbols(hk))
+    assert news.count("quote/0700.HK") == 1 and holding_news.news_codes(news, hk) == ["700"]  # not "1,700"
     assert movers.listed_codes("≥ 5.0% at close (ASX):\nDRO +12.1%\nNXT +5.5%") == {"DRO": "DRO.AX", "NXT": "NXT.AX"}
     assert movers.listed_codes("≤ -5% at close (NASDAQ, NYSE):\nNVDA -6.1%") == {"NVDA": "NVDA"}
     assert movers.listed_codes("≥ 5% at close (ASX, NYSE):\nBHP +6%") == {}  # which exchange? no guess
diff --git a/tests/test_known_tickers.py b/tests/test_known_tickers.py
index 00a4330..a5756d0 100644
--- a/tests/test_known_tickers.py
+++ b/tests/test_known_tickers.py
@@ -1,4 +1,5 @@
 import json
+import time
 
 import httpx
 
@@ -20,6 +21,8 @@ def test_the_ticker_column_is_read_and_share_classes_use_a_hyphen():
     assert kt.symbols_from_html(page(["NVDA", "BRK.B", "bad one", "GOOGL"])) == ["NVDA", "BRK-B", "GOOGL"]
     assert kt.symbols_from_html(page(["AAPL"], table_id="", header="Ticker")) == ["AAPL"]  # no id: by its header
     assert kt.symbols_from_html("
Name
x
") == []
+    footnote = "
Ticker[12]
MSFT
"
+    assert kt.symbols_from_html(footnote) == ["MSFT"]
 
 
 class Sharesight(FakeMcp):
@@ -46,10 +49,13 @@ async def test_refresh_merges_the_indexes_and_the_us_holdings_and_keeps_a_list_t
     try:
         saved = await kt.refresh(ctx, {}, client({sp_url: page(["NVDA", "BRK.B"]), ndx_url: page(["NVDA", "AMD"])}))
         assert saved["indexes"] == {sp: ["NVDA", "BRK-B"], ndx: ["NVDA", "AMD"]} and saved["held"] == ["GM", "ZZZ"]
-        assert set(tickers.KNOWN) == {"NVDA", "BRK-B", "AMD", "GM", "ZZZ"}  # NXT is ASX: not a bare US code
+        assert set(tickers.KNOWN) == {"NVDA", "BRK-B", "BRK.B", "AMD", "GM", "ZZZ"}  # NXT is ASX: not a bare US code
+        assert tickers.KNOWN["BRK.B"] == "BRK-B" and saved["ts"] > time.time() - 5  # all parts worked: due tomorrow
         # the next day the S&P page is down and the NASDAQ page looks wrong: both keep yesterday's list
         again = await kt.refresh(ctx, saved, client({ndx_url: page(["ONLY"])}))
         assert again["indexes"] == saved["indexes"]
+        due = again["ts"] + config.KNOWN_TICKERS_REFRESH - time.time()
+        assert due <= config.KNOWN_TICKERS_RETRY + 5  # a part failed: tried again in half an hour, not a day
         tickers.set_known({})
         assert kt.load(ctx) == again and "BRK-B" in tickers.KNOWN  # a restart has them from the kv table at once
     finally:
diff --git a/tests/test_tickers.py b/tests/test_tickers.py
index dcd4a91..feb41d8 100644
--- a/tests/test_tickers.py
+++ b/tests/test_tickers.py
@@ -80,9 +80,10 @@ def test_sharesight_code_and_market_links_the_code_and_keeps_the_exchange():
 
 
 def test_known_tickers_link_bare_but_not_single_letters_or_everyday_words():
-    known = {"NVDA": "NVDA", "GM": "GM", "IT": "IT", "A": "A", "ALL": "ALL"}
-    out = link_tickers("NVDA up, GM too; IT spending, A big move, ALL in", known)
-    assert out == f"{a('NVDA', 'NVDA')} up, {a('GM', 'GM')} too; IT spending, A big move, ALL in"
+    known = {"NVDA": "NVDA", "AMD": "AMD", "GM": "GM", "IT": "IT", "A": "A", "ALL": "ALL", "700": "0700.HK"}
+    out = link_tickers("NVDA up, AMD too; GM expanded, IT spending, A big move, ALL in, 1,700 staff", known)
+    assert out == f"{a('NVDA', 'NVDA')} up, {a('AMD', 'AMD')} too; GM expanded, IT spending, A big move, ALL in, 1,700 staff"
+    assert link_tickers("General Motors (GM) and $GM", known) == f"General Motors ({a('GM', 'GM')}) and {a('GM', 'GM')}"
     before = dict(tickers.KNOWN)
     try:
         tickers.set_known({"NVDA": "NVDA"})
@@ -91,6 +92,31 @@ def test_known_tickers_link_bare_but_not_single_letters_or_everyday_words():
         tickers.set_known(before)
 
 
+def test_an_exchange_suffix_links_even_a_code_that_is_also_a_word():
+    assert link_tickers("Aristocrat (ALL.AX) rose") == f"Aristocrat ({a('ALL.AX', 'ALL')}) rose"
+    assert link_tickers("ASX.AX, $ALL.AX and AI.PA") == f"{a('ASX.AX', 'ASX')}, {a('ALL.AX', 'ALL')} and {a('AI.PA', 'AI')}"
+    assert link_tickers("ALL in, $FY26.AX") == "ALL in, $FY26.AX"
+
+
+def test_a_bracket_that_is_not_a_ticker_leaves_the_words_before_it_linkable():
+    known = {"NVDA": "NVDA"}
+    assert link_tickers("NVDA (Q3 FY26) revenue", known) == f"{a('NVDA', 'NVDA')} (Q3 FY26) revenue"
+    assert link_tickers("NXT.AX (FY26) guidance") == f"{a('NXT.AX', 'NXT')} (FY26) guidance"
+
+
+def test_share_classes_written_with_a_dot_link_to_yahoos_hyphen():
+    known = {"BRK-B": "BRK-B", "BRK.B": "BRK-B"}
+    assert link_tickers("Berkshire Hathaway (BRK.B) rose", known) == f"Berkshire Hathaway ({a('BRK-B', 'BRK.B')}) rose"
+    assert link_tickers("$BRK.B and BRK.B", known) == f"{a('BRK-B', 'BRK.B')} and {a('BRK-B', 'BRK.B')}"
+
+
+def test_names_with_an_escaped_ampersand_are_bolded_whole():
+    known = {"T": "T", "PG": "PG", "SPGI": "SPGI"}
+    assert link_tickers("AT&T (T) rose", known) == f"AT&T ({a('T', 'T')}) rose"
+    assert link_tickers("Procter & Gamble (PG)", known) == f"Procter & Gamble ({a('PG', 'PG')})"
+    assert link_tickers("S&P Global (SPGI)", known) == f"S&P Global ({a('SPGI', 'SPGI')})"
+
+
 def test_protected_markup_is_left_alone():
     assert link_tickers('NVDA and $AMD') == f'NVDA and {a("AMD", "AMD")}'
     assert link_tickers("
$NVDA 12.3
 $AMD").startswith("
$NVDA 12.3
 
Date: Thu, 5 Nov 2026 07:18:59 +0000
Subject: [PATCH 20/30] Known tickers: Wikipedia's action API with a contact
 User-Agent, and the ASX 200

Wikipedia answered 403 to the page fetches: its policy refuses programs whose User-Agent names no
contact. The lists now come through the action API (action=parse) with the repo URL in the
User-Agent, and a refusal logs Wikimedia's reason. The S&P/ASX 200 is added: its codes link bare
with .AX. Holdings on any known exchange count (NXT -> NXT.AX). A code in a US index and in the
ASX 200 isn't linked bare unless a holding settles it. ASX codes that are everyday words or
acronyms (GPT, WOW, ZIP, CAR...) need a marked form.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 CLAUDE.md                     |   2 +-
 docs/how-it-works.md          |   7 ++-
 lib/config.py                 |   9 +--
 lib/features/known_tickers.py | 102 +++++++++++++++++++++++-----------
 lib/tickers_data.py           |   6 +-
 tests/test_known_tickers.py   |  58 +++++++++++++------
 tests/test_tickers.py         |   6 ++
 7 files changed, 132 insertions(+), 58 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index b9d1a02..5a31c00 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -26,7 +26,7 @@ MCP data tools and web search. User-facing docs are in `README.md` (quick start)
   `openrouter.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs).
 - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun.
 - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming
-  drafts), `commands.py` (/tokens), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `known_tickers.py` (daily background refresh of `tickers.KNOWN`: S&P 500 + NASDAQ-100 from Wikipedia, US Sharesight holdings; kept in kv), `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model).
+  drafts), `commands.py` (/tokens), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `known_tickers.py` (daily background refresh of `tickers.KNOWN`: S&P 500 + NASDAQ-100 + ASX 200 via Wikipedia's action API, Sharesight holdings; kept in kv), `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model).
 - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot).
 - `docs/plans/go-recode.md`: saved plan for a Go rewrite (not started); its key requirement is a modular design with self-contained features.
 - `deploy/systemd/`: the example unit `grokbot.service` and the optional `grokbot.service.d/hardening.conf` drop-in (docs/running.md).
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index ef011fd..dbf396a 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -108,9 +108,10 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf
   an exchange suffix (`NXT.AX`, `000660.KS`), an exchange prefix (`ASX:NXT`, `NASDAQ: NVDA`, mapped to Yahoo's suffix:
   `ASX:NXT` -> `NXT.AX`), `Name (TICKER)` (bold, name and ticker together, unless already bold), Sharesight's
   `NXT (ASX)`, `^GSPC` and `BTC-USD`. The system prompt asks the model for these forms. A bare word is linked only
-  when it is a known ticker: the S&P 500 and NASDAQ-100 constituents (read from their Wikipedia pages) and the US
-  Sharesight holdings, refreshed once a day in the background and kept in the database, so a reply never waits for
-  them. Any other capitalised word stays plain (`GAAP`, `EBITDA`, `AFR`, `M3`), so there is no stoplist to maintain. An
+  when it is a known ticker: the S&P 500, NASDAQ-100 and S&P/ASX 200 constituents (read through Wikipedia's API) and
+  the Sharesight holdings, refreshed once a day in the background and kept in the database, so a reply never waits for
+  them. An ASX code links with its suffix (`BHP` -> `BHP.AX`); a code in a US index and in the ASX 200 names two
+  companies, so it isn't linked bare unless a holding says which. Any other capitalised word stays plain (`GAAP`, `EBITDA`, `AFR`, `M3`), so there is no stoplist to maintain. An
   exchange suffix settles it on its own (`ALL.AX` is Aristocrat);
   single letters and index tickers that are also everyday words or acronyms (`IT`, `ALL`, `GM`, `UPS`, `HD`) need `$`,
   an exchange prefix or `Name (TICKER)`. A part of the refresh that fails is tried again in half an hour. A bare
diff --git a/lib/config.py b/lib/config.py
index 3479ce7..5a9c30a 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -56,11 +56,12 @@
 SEARCH_RESULTS = 20
 SEARCH_TOKENS = 400  # cap on each search write-up (OpenRouter search runner)
 MAX_SEARCHES = 5  # search calls run per round
-# Index constituents linked bare in replies (features/known_tickers.py): (name, Wikipedia page, fewest and most rows
-# that look right). Fetched in the background once a day, never while answering.
+# Index constituents linked bare in replies (features/known_tickers.py): (name, Wikipedia page, Yahoo suffix, fewest
+# and most rows that look right). Fetched in the background once a day, never while answering.
 INDEX_LISTS = (
-    ("S&P 500", "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies", 450, 550),
-    ("NASDAQ-100", "https://en.wikipedia.org/wiki/Nasdaq-100", 90, 110),
+    ("S&P 500", "List of S&P 500 companies", "", 450, 550),
+    ("NASDAQ-100", "Nasdaq-100", "", 90, 110),
+    ("ASX 200", "S&P/ASX 200", "AX", 190, 210),
 )
 KNOWN_TICKERS_REFRESH = 24 * 3600  # seconds between refreshes of the indexes and the US holdings
 KNOWN_TICKERS_RETRY = 30 * 60  # seconds before a failed part (a page down, Sharesight not up yet) is tried again
diff --git a/lib/features/known_tickers.py b/lib/features/known_tickers.py
index 043e78c..c68e990 100644
--- a/lib/features/known_tickers.py
+++ b/lib/features/known_tickers.py
@@ -1,4 +1,4 @@
-"""Tickers a reply may name bare and still get a link: the S&P 500, the NASDAQ-100 and the US Sharesight holdings.
+"""Tickers a reply may name bare and still get a link: the S&P 500, the NASDAQ-100, the ASX 200 and the holdings.
 
 Refreshed in the background once a day and kept in the kv table, so a restart has them at once; replies only read
 tickers.KNOWN and never wait for a fetch."""
@@ -20,8 +20,12 @@
 
 log = logging.getLogger("bot")
 KV_KEY = "known_tickers"
-SYMBOL_RE = re.compile(r"^[A-Z]{1,5}(?:-[A-Z])?$")
-HEADERS = {"User-Agent": "grokbot/1.0 (Telegram stock-chat bot; daily index list)"}
+US_SYMBOL_RE = re.compile(r"^[A-Z]{1,5}(?:-[A-Z])?$")
+ASX_SYMBOL_RE = re.compile(r"^[A-Z0-9]{3,5}$")  # BHP, A2M, AX1
+# Wikimedia refuses programs whose User-Agent names no contact; its action API is the way in for programs.
+HEADERS = {"User-Agent": "grokbot/1.0 (+https://github.com/robdevops/grokbot; daily index list)"}
+API = "https://en.wikipedia.org/w/api.php"
+HEADER_NAMES = ("symbol", "ticker", "ticker symbol", "code", "asx code")
 
 
 class _Tables(HTMLParser):
@@ -55,9 +59,9 @@ def handle_data(self, data):
             self._cell.append(data)
 
 
-def symbols_from_html(page: str) -> list[str]:
+def symbols_from_html(page: str, suffix: str = "") -> list[str]:
     """The ticker column of a Wikipedia index page: the table with id "constituents", else the first table with a
-    "Symbol" or "Ticker" header. Yahoo writes share classes with a hyphen (BRK.B -> BRK-B)."""
+    "Symbol", "Ticker" or "Code" header. US share classes as Yahoo writes them (BRK.B -> BRK-B); ASX codes as listed."""
     parser = _Tables()
     parser.feed(page)
     tables = sorted(parser.tables, key=lambda t: t["id"] != "constituents")
@@ -66,23 +70,30 @@ def symbols_from_html(page: str) -> list[str]:
         if not rows:
             continue
         header = [re.sub(r"\[.*?\]", "", h).strip().lower() for h in rows[0]]  # "Ticker[12]": a footnote
-        col = next((i for i, h in enumerate(header) if h in ("symbol", "ticker", "ticker symbol")), None)
+        col = next((i for i, h in enumerate(header) if h in HEADER_NAMES), None)
         if col is None:
             continue
-        found = [row[col].replace(".", "-") for row in rows[1:] if len(row) > col]
-        return [s for s in found if SYMBOL_RE.match(s)]
+        found = [row[col] if suffix else row[col].replace(".", "-") for row in rows[1:] if len(row) > col]
+        return [s for s in found if (ASX_SYMBOL_RE if suffix else US_SYMBOL_RE).match(s)]
     return []
 
 
-async def fetch_index(client: httpx.AsyncClient, name: str, url: str, low: int, high: int) -> list[str] | None:
-    """An index's tickers, or None if the page couldn't be read or doesn't look right."""
+async def fetch_index(client: httpx.AsyncClient, name: str, title: str, suffix: str, low: int,
+                      high: int) -> list[str] | None:
+    """An index's codes, or None if the page couldn't be read or doesn't look right."""
+    params = {"action": "parse", "page": title, "prop": "text", "format": "json", "formatversion": "2",
+              "redirects": "1"}
     try:
-        r = await client.get(url, headers=HEADERS, follow_redirects=True)
-        r.raise_for_status()
-    except httpx.HTTPError as e:
+        r = await client.get(API, params=params, headers=HEADERS, follow_redirects=True)
+        if r.status_code != 200:
+            log.warning("Known tickers: couldn't fetch the %s list: HTTP %d: %s", name, r.status_code,
+                        " ".join(r.text.split())[:200])
+            return None
+        page = r.json()["parse"]["text"]
+    except (httpx.HTTPError, ValueError, KeyError, TypeError) as e:
         log.warning("Known tickers: couldn't fetch the %s list: %s", name, e)
         return None
-    found = symbols_from_html(r.text)
+    found = symbols_from_html(page, suffix)
     if not low <= len(found) <= high:
         log.warning("Known tickers: the %s page gave %d tickers (expected %d-%d); keeping the last list",
                     name, len(found), low, high)
@@ -90,8 +101,9 @@ async def fetch_index(client: httpx.AsyncClient, name: str, url: str, low: int,
     return found
 
 
-async def us_holdings(ctx: Ctx) -> list[str] | None:
-    """The codes of the US-listed Sharesight holdings, or None if Sharesight isn't up or isn't set up."""
+async def held_symbols(ctx: Ctx) -> dict[str, str] | None:
+    """The Sharesight holdings on exchanges we know, code -> Yahoo symbol ({"NXT": "NXT.AX", "GM": "GM"}), or None if
+    Sharesight isn't up or isn't set up."""
     server = next((s for s in ctx.registry.up() if s.cfg.get("gate") == "portfolio"), None)
     if server is None or not ctx.st.portfolio_names:
         return None
@@ -100,15 +112,40 @@ async def us_holdings(ctx: Ctx) -> list[str] | None:
     except Exception as e:  # the MCP server's own errors, bad JSON
         log.warning("Known tickers: couldn't read the holdings: %s", e)
         return None
-    pairs = (code_and_market(key) for key in held)
-    return sorted(code for code, market in pairs if tickers.yahoo_symbol(code, market) == code)
-
-
-def apply(saved: dict) -> None:
-    """Make the saved lists the ones replies link. Yahoo writes share classes BRK-B; the model may write BRK.B."""
-    codes = {c for name, _, _, _ in config.INDEX_LISTS for c in saved.get("indexes", {}).get(name, [])}
-    codes |= {c.replace(".", "-") for c in saved.get("held", [])}
-    tickers.set_known({**{c: c for c in codes}, **{c.replace("-", "."): c for c in codes if "-" in c}})
+    symbols = {}
+    for code, market in (code_and_market(key) for key in held):
+        symbol = tickers.yahoo_symbol(code, market)
+        if symbol == code:  # a US listing: Yahoo writes share classes BRK-B
+            code = symbol = code.replace(".", "-")
+        if symbol:
+            symbols[code] = symbol
+    return symbols
+
+
+def known(saved: dict) -> tuple[dict[str, str], set[str]]:
+    """Code -> Yahoo symbol from the saved lists, and the codes left out because they name a company on two
+    exchanges (in a US index and in the ASX 200) unless a holding settles which. US share classes are known both
+    ways (BRK-B and BRK.B, as the model may write it)."""
+    by_code: dict[str, set[str]] = {}
+    for name, _, suffix, _, _ in config.INDEX_LISTS:
+        for code in saved.get("indexes", {}).get(name, []):
+            by_code.setdefault(code, set()).add(f"{code}.{suffix}" if suffix else code)
+    ambiguous = {code for code, symbols in by_code.items() if len(symbols) > 1}
+    codes = {code: next(iter(symbols)) for code, symbols in by_code.items() if code not in ambiguous}
+    held = saved.get("held", {})
+    if isinstance(held, list):  # saved by an earlier version: US codes only
+        held = {code.replace(".", "-"): code.replace(".", "-") for code in held}
+    codes.update(held)
+    ambiguous -= set(held)
+    codes.update({c.replace("-", "."): s for c, s in list(codes.items()) if "-" in c and s == c})
+    return codes, ambiguous
+
+
+def apply(saved: dict) -> set[str]:
+    """Make the saved lists the ones replies link; returns the ambiguous codes left out."""
+    codes, ambiguous = known(saved)
+    tickers.set_known(codes)
+    return ambiguous
 
 
 def load(ctx: Ctx) -> dict:
@@ -125,21 +162,22 @@ async def refresh(ctx: Ctx, saved: dict, client: httpx.AsyncClient) -> dict:
     """Fetch the indexes and the holdings; a part that fails keeps its last list and is tried again sooner
     (KNOWN_TICKERS_RETRY) than the daily refresh."""
     indexes, failed = dict(saved.get("indexes", {})), False
-    for name, url, low, high in config.INDEX_LISTS:
-        found = await fetch_index(client, name, url, low, high)
+    for name, title, suffix, low, high in config.INDEX_LISTS:
+        found = await fetch_index(client, name, title, suffix, low, high)
         if found is None:
             failed = True
         else:
             indexes[name] = found
-    held = await us_holdings(ctx)
+    held = await held_symbols(ctx)
     failed |= held is None and bool(ctx.st.portfolio_names) and any(
         s.cfg.get("gate") == "portfolio" for s in ctx.registry.servers.values())
     ts = int(time.time()) - (config.KNOWN_TICKERS_REFRESH - config.KNOWN_TICKERS_RETRY if failed else 0)
-    new = {"indexes": indexes, "held": saved.get("held", []) if held is None else held, "ts": ts}
+    new = {"indexes": indexes, "held": saved.get("held", {}) if held is None else held, "ts": ts}
     await asyncio.to_thread(ctx.store.kv_set, KV_KEY, json.dumps(new))
-    apply(new)
-    log.info("Known tickers: %s, %d held, %d in all", ", ".join(f"{len(v)} {k}" for k, v in indexes.items()) or
-             "no index lists", len(new["held"]), len(tickers.KNOWN))
+    ambiguous = apply(new)
+    log.info("Known tickers: %s, %d held, %d in all (%d on two exchanges left out)",
+             ", ".join(f"{len(v)} {k}" for k, v in indexes.items()) or "no index lists", len(new["held"]),
+             len(tickers.KNOWN), len(ambiguous))
     return new
 
 
diff --git a/lib/tickers_data.py b/lib/tickers_data.py
index 008435c..4832ae8 100644
--- a/lib/tickers_data.py
+++ b/lib/tickers_data.py
@@ -19,11 +19,15 @@
 }
 
 # Index tickers that are also everyday words or acronyms in capitals ("IT spending", "GM expanded 200bp", "UPS demand",
-# "HD maps"): never linked bare, only as $GM, NYSE:GM or General Motors (GM). A fixed set: the indexes don't grow.
+# "HD maps", "GPT models"): never linked bare, only as $GM, NYSE:GM or General Motors (GM). A fixed set: the indexes
+# barely change.
 WORD_TICKERS = {
     "IT", "ALL", "ON", "NOW", "KEY", "LOW", "ARE", "DAY", "HAS", "WELL", "BALL", "FAST", "CAT", "AMP", "SO", "ED",
     "GM", "UPS", "IP", "IR", "ICE", "PTC", "RF", "RL", "USB", "HD", "TECH", "EA", "CE", "MAR", "DD", "CF", "MS",
     "FANG", "MSCI", "COST", "BR", "GE", "HP", "PH", "TT", "CTA",
+    # ASX 200
+    "GPT", "LLC", "WOW", "ZIP", "CAR", "SUN", "WEB", "HUB", "ARB", "ANN", "ORG", "MIN", "BEN", "PRU", "NAN", "IEL",
+    "TNE", "LOV", "IPL", "SGR", "CHC", "EVN", "DOW", "VEA", "NEC", "PNI", "IFT", "BOE", "DEG",
 }
 
 # An exchange as the model, the press or Sharesight writes it ("ASX:NXT", "NXT (ASX)") -> Yahoo's suffix ("" = US,
diff --git a/tests/test_known_tickers.py b/tests/test_known_tickers.py
index a5756d0..9c62592 100644
--- a/tests/test_known_tickers.py
+++ b/tests/test_known_tickers.py
@@ -18,6 +18,7 @@ def page(rows, table_id="constituents", header="Symbol"):
 
 
 def test_the_ticker_column_is_read_and_share_classes_use_a_hyphen():
+    assert kt.symbols_from_html(page(["BHP", "A2M", "bad"], header="Code"), "AX") == ["BHP", "A2M"]  # ASX codes
     assert kt.symbols_from_html(page(["NVDA", "BRK.B", "bad one", "GOOGL"])) == ["NVDA", "BRK-B", "GOOGL"]
     assert kt.symbols_from_html(page(["AAPL"], table_id="", header="Ticker")) == ["AAPL"]  # no id: by its header
     assert kt.symbols_from_html("
Name
x
") == []
@@ -29,42 +30,65 @@ class Sharesight(FakeMcp):
     async def call(self, tool, args):
         if tool == "list_portfolios":
             return json.dumps({"portfolios": [{"name": "Bob", "id": 7}]})
-        rows = [["GM", "NYSE", "General Motors"], ["NXT", "ASX", "NextDC"], ["ZZZ", "NASDAQ", "Zed"]]
+        rows = [["GM", "NYSE", "General Motors"], ["NXT", "ASX", "NextDC"], ["QBE", "ASX", "QBE Insurance"],
+                ["ZZZ", "MOON", "Unknown exchange"]]
         return json.dumps({"report": {"holdings": {"columns": ["code", "market", "name"], "rows": rows}}})
 
 
-def client(pages: dict[str, str]) -> httpx.AsyncClient:
+def wikipedia(pages: dict[str, str], seen: list | None = None) -> httpx.AsyncClient:
+    """Wikipedia's action API: the page HTML inside {"parse": {"text": ...}}; a page not given answers 403."""
     def reply(request: httpx.Request) -> httpx.Response:
-        text = pages.get(str(request.url))
-        return httpx.Response(200, text=text) if text is not None else httpx.Response(503)
+        if seen is not None:
+            seen.append(request)
+        title = request.url.params.get("page")
+        if title not in pages:
+            return httpx.Response(403, text="Please set a user-agent and respect our robot policy")
+        return httpx.Response(200, json={"parse": {"title": title, "text": pages[title]}})
     return httpx.AsyncClient(transport=httpx.MockTransport(reply))
 
 
-async def test_refresh_merges_the_indexes_and_the_us_holdings_and_keeps_a_list_that_fails(env, store, monkeypatch):
-    (sp, sp_url, _, _), (ndx, ndx_url, _, _) = config.INDEX_LISTS
-    monkeypatch.setattr(config, "INDEX_LISTS", ((sp, sp_url, 2, 5), (ndx, ndx_url, 2, 5)))
+def small_lists(monkeypatch):
+    lists = tuple((name, title, suffix, 2, 5) for name, title, suffix, _, _ in config.INDEX_LISTS)
+    monkeypatch.setattr(config, "INDEX_LISTS", lists)
+    return [title for _, title, _, _, _ in lists]
+
+
+async def test_refresh_reads_the_api_with_a_contact_and_links_asx_codes_to_ax(env, store, monkeypatch, caplog):
+    sp, ndx, asx = small_lists(monkeypatch)
     st = config.load({**env, "PORTFOLIO_NAMES": "Bob"})
     ctx = Ctx(st, store, ScriptedBackend([]), registry(Sharesight("sharesight", gate="portfolio")), FakeBot())
+    pages = {sp: page(["NVDA", "BRK.B", "QBE", "ALL"]), ndx: page(["NVDA", "AMD"]),
+             asx: page(["BHP", "A2M", "ALL", "QBE"], header="Code")}
+    seen: list = []
     before = dict(tickers.KNOWN)
     try:
-        saved = await kt.refresh(ctx, {}, client({sp_url: page(["NVDA", "BRK.B"]), ndx_url: page(["NVDA", "AMD"])}))
-        assert saved["indexes"] == {sp: ["NVDA", "BRK-B"], ndx: ["NVDA", "AMD"]} and saved["held"] == ["GM", "ZZZ"]
-        assert set(tickers.KNOWN) == {"NVDA", "BRK-B", "BRK.B", "AMD", "GM", "ZZZ"}  # NXT is ASX: not a bare US code
-        assert tickers.KNOWN["BRK.B"] == "BRK-B" and saved["ts"] > time.time() - 5  # all parts worked: due tomorrow
-        # the next day the S&P page is down and the NASDAQ page looks wrong: both keep yesterday's list
-        again = await kt.refresh(ctx, saved, client({ndx_url: page(["ONLY"])}))
+        saved = await kt.refresh(ctx, {}, wikipedia(pages, seen))
+        assert seen[0].url.params["action"] == "parse" and "https://" in seen[0].headers["user-agent"]
+        assert saved["indexes"]["ASX 200"] == ["BHP", "A2M", "ALL", "QBE"]
+        assert saved["held"] == {"GM": "GM", "NXT": "NXT.AX", "QBE": "QBE.AX"}  # MOON is no exchange we know
+        k = tickers.KNOWN
+        assert k["BHP"] == "BHP.AX" and k["A2M"] == "A2M.AX" and k["NVDA"] == "NVDA" and k["BRK.B"] == "BRK-B"
+        assert "ALL" not in k and k["QBE"] == "QBE.AX"  # on both exchanges: left out, unless a holding settles it
+        assert saved["ts"] > time.time() - 5  # every part worked: due tomorrow
+        # the next day the S&P page refuses and the NASDAQ page looks wrong: both keep yesterday's list
+        again = await kt.refresh(ctx, saved, wikipedia({ndx: page(["ONLY"]), asx: pages[asx]}))
         assert again["indexes"] == saved["indexes"]
-        due = again["ts"] + config.KNOWN_TICKERS_REFRESH - time.time()
-        assert due <= config.KNOWN_TICKERS_RETRY + 5  # a part failed: tried again in half an hour, not a day
+        assert "HTTP 403: Please set a user-agent" in caplog.text  # why it was refused
+        assert again["ts"] + config.KNOWN_TICKERS_REFRESH - time.time() <= config.KNOWN_TICKERS_RETRY + 5
         tickers.set_known({})
-        assert kt.load(ctx) == again and "BRK-B" in tickers.KNOWN  # a restart has them from the kv table at once
+        assert kt.load(ctx) == again and tickers.KNOWN["BHP"] == "BHP.AX"  # a restart has them from the kv table
     finally:
         tickers.set_known(before)
 
 
+def test_lists_saved_by_the_earlier_version_still_load():
+    codes, ambiguous = kt.known({"indexes": {"S&P 500": ["NVDA"]}, "held": ["GM", "BRK.B"]})
+    assert codes == {"NVDA": "NVDA", "GM": "GM", "BRK-B": "BRK-B", "BRK.B": "BRK-B"} and ambiguous == set()
+
+
 async def test_without_sharesight_the_held_list_stays_as_it_was(env, store):
     ctx = Ctx(config.load(env), store, ScriptedBackend([]), registry(), FakeBot())
-    assert await kt.us_holdings(ctx) is None
+    assert await kt.held_symbols(ctx) is None
 
 
 async def test_the_loop_waits_for_the_mcp_when_overdue_and_otherwise_until_due(env, store, monkeypatch):
diff --git a/tests/test_tickers.py b/tests/test_tickers.py
index feb41d8..c59fc2c 100644
--- a/tests/test_tickers.py
+++ b/tests/test_tickers.py
@@ -117,6 +117,12 @@ def test_names_with_an_escaped_ampersand_are_bolded_whole():
     assert link_tickers("S&P Global (SPGI)", known) == f"S&P Global ({a('SPGI', 'SPGI')})"
 
 
+def test_asx_codes_from_the_known_set_link_with_their_suffix_but_word_codes_do_not():
+    known = {"BHP": "BHP.AX", "GPT": "GPT.AX"}
+    assert link_tickers("BHP fell; GPT models", known) == f"{a('BHP.AX', 'BHP')} fell; GPT models"
+    assert link_tickers("GPT Group (GPT)", known) == f"GPT Group ({a('GPT.AX', 'GPT')})"
+
+
 def test_protected_markup_is_left_alone():
     assert link_tickers('NVDA and $AMD') == f'NVDA and {a("AMD", "AMD")}'
     assert link_tickers("
$NVDA 12.3
 $AMD").startswith("
$NVDA 12.3
 
Date: Thu, 5 Nov 2026 07:20:45 +0000
Subject: [PATCH 21/30] Test the ASX 200 page layout (Code column, number codes
 like 360)

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 tests/test_known_tickers.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/test_known_tickers.py b/tests/test_known_tickers.py
index 9c62592..24f82f7 100644
--- a/tests/test_known_tickers.py
+++ b/tests/test_known_tickers.py
@@ -118,3 +118,15 @@ async def refresh(ctx, saved, client):
         tickers.set_known(before)
     assert waits[0] == config.KNOWN_TICKERS_START_DELAY and refreshed == [{}]  # nothing saved yet: refresh soon
     assert config.KNOWN_TICKERS_REFRESH - 5 < waits[1] <= config.KNOWN_TICKERS_REFRESH  # then a day later
+
+
+def test_the_asx_200_page_layout_reads_number_codes_too():
+    html = ('
As of 5 April 2026, the constituent stocks of the ASX 200
'
+            ''
+            ''
+            ''
+            '
CodeCompanySector
360Life360Information Technology
4DX4DMedicalHealthcare
A2Ma2 Milk CompanyConsumer Staples
')
+    codes = kt.symbols_from_html(html, "AX")
+    assert codes == ["360", "4DX", "A2M"]
+    out = tickers.link_tickers("Life360 (360) and 4DX up; 360 degrees", {c: f"{c}.AX" for c in codes})
+    assert 'quote/360.AX">360) and 
Date: Thu, 5 Nov 2026 07:27:40 +0000
Subject: [PATCH 22/30] Known tickers: read the NASDAQ-100 from "List of
 NASDAQ-100 companies"

The "Nasdaq-100" article has no constituents table any more (0 tickers in production).

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 lib/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/config.py b/lib/config.py
index 5a9c30a..e741cd2 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -60,7 +60,7 @@
 # and most rows that look right). Fetched in the background once a day, never while answering.
 INDEX_LISTS = (
     ("S&P 500", "List of S&P 500 companies", "", 450, 550),
-    ("NASDAQ-100", "Nasdaq-100", "", 90, 110),
+    ("NASDAQ-100", "List of NASDAQ-100 companies", "", 90, 110),
     ("ASX 200", "S&P/ASX 200", "AX", 190, 210),
 )
 KNOWN_TICKERS_REFRESH = 24 * 3600  # seconds between refreshes of the indexes and the US holdings

From 2aaf21f2eb8bbc826e56f820006121dd0bbdac78 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 07:35:43 +0000
Subject: [PATCH 23/30] FINANCE_LINK_PROVIDER: ticker links to Yahoo Finance
 (default) or Google Finance

Google takes CODE:EXCHANGE (NXT.AX -> NXT:ASX, 0700.HK -> 700:HKG); a US ticker names no exchange
(NASDAQ or NYSE), so it opens a Google Finance search. History compaction and the holding-news
mute codes read either provider's links back.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 CLAUDE.md                    |  2 +-
 docs/configuration.md        |  1 +
 docs/how-it-works.md         |  3 ++-
 lib/app.py                   |  3 ++-
 lib/config.py                |  6 +++++
 lib/features/holding_news.py |  9 ++++----
 lib/history.py               |  6 ++---
 lib/tickers.py               | 45 ++++++++++++++++++++++++++++++++++--
 tests/test_tickers.py        | 25 ++++++++++++++++++++
 9 files changed, 87 insertions(+), 13 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 5a31c00..698d2cc 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -19,7 +19,7 @@ MCP data tools and web search. User-facing docs are in `README.md` (quick start)
 - `store.py`: SQLite `Store`: history (`messages` shared by groups, `dm_messages` per bot), users, kv,
   holding-news state, `usage` ledger, `text_cache` (finished /narrative and /events messages, 24 h), `period_cache` (finished Sharesight periods, kept until flushed). Thread-safe; call through `asyncio.to_thread`.
 - `msgtext.py` sender_name/describe. `history.py` transcript lines. `textfmt.py` HTML helpers
-  (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo links on marked tickers (`$X`, `X.AX`, `ASX:X`, `Name (X)`, `X (ASX)`) and the known set `tickers.KNOWN`; bare words otherwise never link (+ `tickers_data.py`: exchange prefixes -> Yahoo suffixes, the gate's NOT_TICKERS). `holdings.py` current Sharesight holdings (holding news, known tickers).
+  (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo (or Google, FINANCE_LINK_PROVIDER) links on marked tickers (`$X`, `X.AX`, `ASX:X`, `Name (X)`, `X (ASX)`) and the known set `tickers.KNOWN`; bare words otherwise never link (+ `tickers_data.py`: exchange prefixes -> Yahoo suffixes, the gate's NOT_TICKERS). `holdings.py` current Sharesight holdings (holding news, known tickers).
 - `mcp/`: `server.py` (MCPServer lifecycle, TTL cache, `Registry`), `schema.py` (neutral `ToolDef`,
   schema diet), `results.py` (result slimming).
 - `llm/`: `base.py` (Request/Step/Answer/Backend), `chat.py` (shared chat-completions loop), `xai.py` +
diff --git a/docs/configuration.md b/docs/configuration.md
index 23df421..c2c7fd2 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -26,6 +26,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o
 | `ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/tokens`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. |
 | `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/tokens` and `/flushcache` stay admin-only. Who counts as an admin: `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). |
 | `DATA_DIR` | systemd's `STATE_DIRECTORY`, else the working directory | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. With systemd the default is its `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)). In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. |
+| `FINANCE_LINK_PROVIDER` | `yahoo` | Where ticker links go: `yahoo` (Yahoo Finance) or `google` (Google Finance: `NXT.AX` opens `NXT:ASX`; a US ticker, whose exchange the bot doesn't know, opens a Google Finance search). |
 | `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. |
 | `MAX_TOKENS` | `3000` | Reply cap in tokens, reasoning included. |
 | `MOVERS_BOTS` | empty (feature off) | Usernames of bots whose end-of-day big-movers lists get explained. |
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index dbf396a..5a28bf3 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -104,7 +104,8 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf
 ## Formatting and ticker links
 - Replies are Telegram HTML. Markdown the model slips in (`**bold**`, links, backticks) is
   converted; trailing "not advice"/"NFA"/"DYOR" disclaimers are stripped.
-- **Yahoo Finance links:** only a ticker written in a marked form is linked to its Yahoo Finance page: `$NVDA`,
+- **Ticker links:** only a ticker written in a marked form is linked to its Yahoo Finance page (or Google Finance,
+  with `FINANCE_LINK_PROVIDER=google`): `$NVDA`,
   an exchange suffix (`NXT.AX`, `000660.KS`), an exchange prefix (`ASX:NXT`, `NASDAQ: NVDA`, mapped to Yahoo's suffix:
   `ASX:NXT` -> `NXT.AX`), `Name (TICKER)` (bold, name and ticker together, unless already bold), Sharesight's
   `NXT (ASX)`, `^GSPC` and `BTC-USD`. The system prompt asks the model for these forms. A bare word is linked only
diff --git a/lib/app.py b/lib/app.py
index 37a0f7f..24ab4c0 100644
--- a/lib/app.py
+++ b/lib/app.py
@@ -32,7 +32,7 @@
     filters,
 )
 
-from . import config
+from . import config, tickers
 from .context import Ctx
 from .features import chat_commands, dm_buttons, holding_news, known_tickers, post
 from .features.charts import buttons as chart_buttons
@@ -163,6 +163,7 @@ async def register_menu(ctx: Ctx) -> None:
 
 def build_ctx(st: config.Settings) -> Ctx:
     ctx = Ctx(st, Store(st.db_path), build_backend(st), Registry([]))
+    tickers.set_provider(st.link_provider)
 
     async def on_down(server: MCPServer) -> None:
         await alert_down(ctx, server)
diff --git a/lib/config.py b/lib/config.py
index e741cd2..99293d2 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -26,6 +26,7 @@
     "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /tokens; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.",
     "ADMIN_ONLY": "on|off. On (default): only admins may DM the bot or get Sharesight data. Off: everyone may. /tokens and /flushcache stay admin-only.",
     "DATA_DIR": "Directory for the bot's data (the database); Default: systemd's STATE_DIRECTORY, else the working directory.",
+    "FINANCE_LINK_PROVIDER": "Where ticker links go: yahoo (default, Yahoo Finance) or google (Google Finance).",
     "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.",
     "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).",
     "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.",
@@ -88,6 +89,7 @@ class Settings:
     max_tokens: int
     history_max: int
     darkmode: str  # "off" | "on" | "auto": the charts' theme
+    link_provider: str  # "yahoo" | "google": where ticker links go
     db_path: str
     tz: ZoneInfo
     admin_chats: frozenset[int]  # user IDs (alerts + admin commands) and group chat IDs (alerts only)
@@ -186,6 +188,9 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
     search = env.get("SEARCH", "on").strip().lower() != "off"
     if provider == "openrouter" and not search_model:
         search = False  # nothing to run the searches with
+    link_provider = env.get("FINANCE_LINK_PROVIDER", "").strip().lower() or "yahoo"
+    if link_provider not in ("yahoo", "google"):
+        raise ConfigError(f"FINANCE_LINK_PROVIDER must be yahoo or google, not {link_provider!r}.")
     darkmode = env.get("DARKMODE", "").strip().lower() or "off"
     if darkmode not in ("off", "on", "auto"):
         raise ConfigError(f"DARKMODE must be off, on or auto, not {darkmode!r}.")
@@ -201,6 +206,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
         max_tokens=int(env.get("MAX_TOKENS") or 3000),
         history_max=int(env.get("HISTORY_MAX") or 10),
         darkmode=darkmode,
+        link_provider=link_provider,
         db_path=database_path(env),
         tz=ZoneInfo(env.get("TZ", "UTC")),
         admin_chats=_ids(env.get("ADMIN_CHAT_IDS", "")),
diff --git a/lib/features/holding_news.py b/lib/features/holding_news.py
index 8b7211c..aba5c88 100644
--- a/lib/features/holding_news.py
+++ b/lib/features/holding_news.py
@@ -24,7 +24,7 @@
 from ..telegram.draft import Draft
 from ..telegram.send import send_html
 from ..textfmt import ANY_TAG_RE, md_to_html, split_html, strip_disclaimer
-from ..tickers import link_tickers, yahoo_symbol
+from ..tickers import link_tickers, quote_labels, yahoo_symbol
 
 log = logging.getLogger("bot")
 UNSUBSCRIBE_BUTTON = InlineKeyboardMarkup([[InlineKeyboardButton("🔕 Unsubscribe…", callback_data="hn:menu")]])
@@ -52,12 +52,11 @@ def known_symbols(holdings: dict[str, str]) -> dict[str, str]:
 
 
 def news_codes(news: str, holdings: dict[str, str]) -> list[str]:
-    """Holding codes that appear as bold tickers or Yahoo links in a news message (as the holdings write them: an HK
-    holding "700" is linked as 0700.HK)."""
+    """Holding codes that appear as bold tickers or quote links in a news message (as the holdings write them: an HK
+    holding "700" is linked as 0700.HK but labelled 700)."""
     held = {_plain_code(k.split(" ")[0]): k.split(" ")[0].upper() for k in holdings}
     found = {m.upper() for m in re.findall(r"([^<]{1,12})", news)}
-    found |= {m.split(".")[0].upper()
-              for m in re.findall(r"finance\.yahoo\.com/quote/([\w.\-^]+)", news)}
+    found |= {label.upper() for label in quote_labels(news)}
     return sorted({held[c] for c in map(_plain_code, found) if c in held})
 
 
diff --git a/lib/history.py b/lib/history.py
index 55c7db3..cc5b8df 100644
--- a/lib/history.py
+++ b/lib/history.py
@@ -9,18 +9,18 @@
 
 from .store import HistoryRow
 from .textfmt import ANY_TAG_RE
+from .tickers import QUOTE_PREFIXES
 
 LAST_REPLY_MAX = 6000  # characters of the bot's latest reply kept in the prompt (~1,500 tokens)
 _LINK = re.compile(r'(.*?)', re.DOTALL)
-_YAHOO = "https://finance.yahoo.com/quote/"
 
 
 def compact(text: str) -> str:
-    """Stored Telegram HTML as cheap plain text: ticker links become the symbol, other links
+    """Stored Telegram HTML as cheap plain text: ticker links (Yahoo or Google) become the symbol, other links
     "label (url)", tags are dropped and whitespace is collapsed."""
     def link(m: re.Match) -> str:
         url, label = m.group(1), m.group(2)
-        return label if url.startswith(_YAHOO) else f"{label} ({url})"
+        return label if url.startswith(QUOTE_PREFIXES) else f"{label} ({url})"
 
     plain = html.unescape(ANY_TAG_RE.sub("", _LINK.sub(link, text)))
     return " ".join(plain.split())
diff --git a/lib/tickers.py b/lib/tickers.py
index 76fad47..35d3b35 100644
--- a/lib/tickers.py
+++ b/lib/tickers.py
@@ -38,9 +38,50 @@ def is_ticker(word: str) -> bool:
     return not (word in NOT_TICKERS or NOT_TICKER_RE.match(word))
 
 
+YAHOO_QUOTE = "https://finance.yahoo.com/quote/"
+GOOGLE_QUOTE = "https://www.google.com/finance/quote/"
+GOOGLE_SEARCH = "https://www.google.com/finance?q="
+QUOTE_PREFIXES = (YAHOO_QUOTE, GOOGLE_QUOTE, GOOGLE_SEARCH)  # history.compact and news_codes read these links back
+PROVIDER = ["yahoo"]  # FINANCE_LINK_PROVIDER, set at startup (app.build_ctx)
+# Yahoo's suffix -> Google Finance's exchange code ("NXT.AX" -> "NXT:ASX")
+GOOGLE_EXCHANGES = {
+    "AX": "ASX", "NZ": "NZE", "L": "LON", "TO": "TSE", "V": "CVE", "HK": "HKG", "T": "TYO", "KS": "KRX",
+    "KQ": "KOSDAQ", "DE": "ETR", "F": "FRA", "PA": "EPA", "AS": "AMS", "BR": "EBR", "SW": "SWX", "MI": "BIT",
+    "MC": "BME", "ST": "STO", "OL": "OSL", "CO": "CPH", "HE": "HEL", "SI": "SGX", "SS": "SHA", "SZ": "SHE",
+    "TW": "TPE", "BO": "BOM", "NS": "NSE", "SA": "BVMF", "MX": "BMV", "JO": "JSE", "TA": "TLV",
+}
+
+
+def set_provider(name: str) -> None:
+    PROVIDER[0] = name
+
+
 def yahoo_url(symbol: str) -> str:
     sym = f"{symbol}-USD" if symbol in CRYPTO else symbol
-    return "https://finance.yahoo.com/quote/" + quote(sym, safe="")
+    return YAHOO_QUOTE + quote(sym, safe="")
+
+
+def google_url(symbol: str) -> str:
+    """Google Finance's page for a Yahoo symbol: CODE:EXCHANGE where the suffix names it (NXT.AX -> NXT:ASX, 0700.HK ->
+    700:HKG), BTC-USD as is, else a search (a US ticker gives no exchange: NASDAQ or NYSE)."""
+    base, _, suffix = symbol.rpartition(".")
+    if base and suffix in GOOGLE_EXCHANGES:
+        code = base.lstrip("0") or "0" if suffix == "HK" else base
+        return GOOGLE_QUOTE + quote(f"{code}:{GOOGLE_EXCHANGES[suffix]}", safe=":")
+    if symbol in CRYPTO or symbol.endswith("-USD"):
+        return GOOGLE_QUOTE + quote(symbol if symbol.endswith("-USD") else f"{symbol}-USD", safe="")
+    return GOOGLE_SEARCH + quote(symbol.replace("-", "."), safe="")  # BRK-B is BRK.B at Google
+
+
+def quote_url(symbol: str) -> str:
+    """The quote page for a Yahoo symbol, at the FINANCE_LINK_PROVIDER."""
+    return google_url(symbol) if PROVIDER[0] == "google" else yahoo_url(symbol)
+
+
+def quote_labels(html: str) -> set[str]:
+    """The labels of the quote links in a reply (the codes as shown: NXT, 700), whichever provider made them."""
+    starts = "|".join(re.escape(p) for p in QUOTE_PREFIXES)
+    return set(re.findall(rf'([^<]+)', html))
 
 
 def ticker_label(symbol: str) -> str:
@@ -98,7 +139,7 @@ def _link_re(codes: frozenset[str]) -> re.Pattern:
 
 
 def _link(symbol: str, label: str) -> str:
-    return f'{label}'
+    return f'{label}'
 
 
 def _paren(paren: str, known: Mapping[str, str]) -> str | None:
diff --git a/tests/test_tickers.py b/tests/test_tickers.py
index c59fc2c..8375217 100644
--- a/tests/test_tickers.py
+++ b/tests/test_tickers.py
@@ -149,3 +149,28 @@ def test_link_labelled_with_its_own_url_is_numbered_but_named_links_and_code_are
     assert link_tickers('https://y.com/p… ok') == '[1] ok'
     assert link_tickers('Reuters') == 'Reuters'
     assert link_tickers("https://x.com") == "https://x.com"
+
+
+def test_google_finance_links_name_the_exchange_where_the_symbol_does(env):
+    from lib import config
+    from lib.history import compact
+    assert config.load({**env, "FINANCE_LINK_PROVIDER": "Google"}).link_provider == "google"
+    assert config.load(env).link_provider == "yahoo"
+    try:
+        config.load({**env, "FINANCE_LINK_PROVIDER": "bing"})
+        raise AssertionError("bing accepted")
+    except config.ConfigError:
+        pass
+    g = tickers.google_url
+    assert g("NXT.AX") == "https://www.google.com/finance/quote/NXT:ASX"
+    assert g("0700.HK") == "https://www.google.com/finance/quote/700:HKG" and g("7203.T").endswith("/7203:TYO")
+    assert g("BTC") == g("BTC-USD") == "https://www.google.com/finance/quote/BTC-USD"
+    assert g("NVDA") == "https://www.google.com/finance?q=NVDA" and g("BRK-B").endswith("?q=BRK.B")
+    tickers.set_provider("google")
+    try:
+        out = link_tickers("ASX:NXT and $NVDA")
+        assert out == ('NXT and '
+                       'NVDA')
+        assert compact(out) == "NXT and NVDA" and tickers.quote_labels(out) == {"NXT", "NVDA"}
+    finally:
+        tickers.set_provider("yahoo")

From d7d9874a956ba620c766d185f672f603eae34000 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 07:38:34 +0000
Subject: [PATCH 24/30] FINANCE_LINK_TARGET (was _PROVIDER); Google links name
 the US exchange

Google Finance quotes are CODE:EXCHANGE (JPM:NYSE, SPCX:NASDAQ, BRK.A:NYSE). A US ticker's exchange
now comes from the text (NYSE:JPM, JPM (NYSE)), the Sharesight holdings, the NASDAQ-100 (all
NASDAQ) or the S&P 500 list, whose ticker links point at nyse.com or nasdaq.com; kept in kv with
the lists. Only a ticker none of these place opens a Google Finance search.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 CLAUDE.md                     |  2 +-
 docs/configuration.md         |  2 +-
 docs/how-it-works.md          |  2 +-
 lib/app.py                    |  2 +-
 lib/config.py                 | 12 +++---
 lib/features/known_tickers.py | 77 +++++++++++++++++++++++++----------
 lib/tickers.py                | 47 +++++++++++++--------
 tests/test_known_tickers.py   | 15 +++++--
 tests/test_tickers.py         | 29 ++++++++++---
 9 files changed, 132 insertions(+), 56 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 698d2cc..0a3962e 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -19,7 +19,7 @@ MCP data tools and web search. User-facing docs are in `README.md` (quick start)
 - `store.py`: SQLite `Store`: history (`messages` shared by groups, `dm_messages` per bot), users, kv,
   holding-news state, `usage` ledger, `text_cache` (finished /narrative and /events messages, 24 h), `period_cache` (finished Sharesight periods, kept until flushed). Thread-safe; call through `asyncio.to_thread`.
 - `msgtext.py` sender_name/describe. `history.py` transcript lines. `textfmt.py` HTML helpers
-  (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo (or Google, FINANCE_LINK_PROVIDER) links on marked tickers (`$X`, `X.AX`, `ASX:X`, `Name (X)`, `X (ASX)`) and the known set `tickers.KNOWN`; bare words otherwise never link (+ `tickers_data.py`: exchange prefixes -> Yahoo suffixes, the gate's NOT_TICKERS). `holdings.py` current Sharesight holdings (holding news, known tickers).
+  (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo (or Google, FINANCE_LINK_TARGET) links on marked tickers (`$X`, `X.AX`, `ASX:X`, `Name (X)`, `X (ASX)`) and the known set `tickers.KNOWN`; bare words otherwise never link (+ `tickers_data.py`: exchange prefixes -> Yahoo suffixes, the gate's NOT_TICKERS). `holdings.py` current Sharesight holdings (holding news, known tickers).
 - `mcp/`: `server.py` (MCPServer lifecycle, TTL cache, `Registry`), `schema.py` (neutral `ToolDef`,
   schema diet), `results.py` (result slimming).
 - `llm/`: `base.py` (Request/Step/Answer/Backend), `chat.py` (shared chat-completions loop), `xai.py` +
diff --git a/docs/configuration.md b/docs/configuration.md
index c2c7fd2..d03c850 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -26,7 +26,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o
 | `ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/tokens`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. |
 | `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/tokens` and `/flushcache` stay admin-only. Who counts as an admin: `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). |
 | `DATA_DIR` | systemd's `STATE_DIRECTORY`, else the working directory | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. With systemd the default is its `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)). In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. |
-| `FINANCE_LINK_PROVIDER` | `yahoo` | Where ticker links go: `yahoo` (Yahoo Finance) or `google` (Google Finance: `NXT.AX` opens `NXT:ASX`; a US ticker, whose exchange the bot doesn't know, opens a Google Finance search). |
+| `FINANCE_LINK_TARGET` | `yahoo` | Where ticker links go: `yahoo` (Yahoo Finance) or `google` (Google Finance, `CODE:EXCHANGE`: `NXT.AX` opens `NXT:ASX`, `JPM` opens `JPM:NYSE`). A US ticker's exchange comes from the text (`NYSE:JPM`, `JPM (NYSE)`), the holdings, the NASDAQ-100 or the S&P 500 list; one the bot can't place opens a Google Finance search. |
 | `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. |
 | `MAX_TOKENS` | `3000` | Reply cap in tokens, reasoning included. |
 | `MOVERS_BOTS` | empty (feature off) | Usernames of bots whose end-of-day big-movers lists get explained. |
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index 5a28bf3..bc60d37 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -105,7 +105,7 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf
 - Replies are Telegram HTML. Markdown the model slips in (`**bold**`, links, backticks) is
   converted; trailing "not advice"/"NFA"/"DYOR" disclaimers are stripped.
 - **Ticker links:** only a ticker written in a marked form is linked to its Yahoo Finance page (or Google Finance,
-  with `FINANCE_LINK_PROVIDER=google`): `$NVDA`,
+  with `FINANCE_LINK_TARGET=google`): `$NVDA`,
   an exchange suffix (`NXT.AX`, `000660.KS`), an exchange prefix (`ASX:NXT`, `NASDAQ: NVDA`, mapped to Yahoo's suffix:
   `ASX:NXT` -> `NXT.AX`), `Name (TICKER)` (bold, name and ticker together, unless already bold), Sharesight's
   `NXT (ASX)`, `^GSPC` and `BTC-USD`. The system prompt asks the model for these forms. A bare word is linked only
diff --git a/lib/app.py b/lib/app.py
index 24ab4c0..164da28 100644
--- a/lib/app.py
+++ b/lib/app.py
@@ -163,7 +163,7 @@ async def register_menu(ctx: Ctx) -> None:
 
 def build_ctx(st: config.Settings) -> Ctx:
     ctx = Ctx(st, Store(st.db_path), build_backend(st), Registry([]))
-    tickers.set_provider(st.link_provider)
+    tickers.set_target(st.link_target)
 
     async def on_down(server: MCPServer) -> None:
         await alert_down(ctx, server)
diff --git a/lib/config.py b/lib/config.py
index 99293d2..18dc320 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -26,7 +26,7 @@
     "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /tokens; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.",
     "ADMIN_ONLY": "on|off. On (default): only admins may DM the bot or get Sharesight data. Off: everyone may. /tokens and /flushcache stay admin-only.",
     "DATA_DIR": "Directory for the bot's data (the database); Default: systemd's STATE_DIRECTORY, else the working directory.",
-    "FINANCE_LINK_PROVIDER": "Where ticker links go: yahoo (default, Yahoo Finance) or google (Google Finance).",
+    "FINANCE_LINK_TARGET": "Where ticker links go: yahoo (default, Yahoo Finance) or google (Google Finance).",
     "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.",
     "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).",
     "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.",
@@ -89,7 +89,7 @@ class Settings:
     max_tokens: int
     history_max: int
     darkmode: str  # "off" | "on" | "auto": the charts' theme
-    link_provider: str  # "yahoo" | "google": where ticker links go
+    link_target: str  # "yahoo" | "google": where ticker links go
     db_path: str
     tz: ZoneInfo
     admin_chats: frozenset[int]  # user IDs (alerts + admin commands) and group chat IDs (alerts only)
@@ -188,9 +188,9 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
     search = env.get("SEARCH", "on").strip().lower() != "off"
     if provider == "openrouter" and not search_model:
         search = False  # nothing to run the searches with
-    link_provider = env.get("FINANCE_LINK_PROVIDER", "").strip().lower() or "yahoo"
-    if link_provider not in ("yahoo", "google"):
-        raise ConfigError(f"FINANCE_LINK_PROVIDER must be yahoo or google, not {link_provider!r}.")
+    link_target = env.get("FINANCE_LINK_TARGET", "").strip().lower() or "yahoo"
+    if link_target not in ("yahoo", "google"):
+        raise ConfigError(f"FINANCE_LINK_TARGET must be yahoo or google, not {link_target!r}.")
     darkmode = env.get("DARKMODE", "").strip().lower() or "off"
     if darkmode not in ("off", "on", "auto"):
         raise ConfigError(f"DARKMODE must be off, on or auto, not {darkmode!r}.")
@@ -206,7 +206,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
         max_tokens=int(env.get("MAX_TOKENS") or 3000),
         history_max=int(env.get("HISTORY_MAX") or 10),
         darkmode=darkmode,
-        link_provider=link_provider,
+        link_target=link_target,
         db_path=database_path(env),
         tz=ZoneInfo(env.get("TZ", "UTC")),
         admin_chats=_ids(env.get("ADMIN_CHAT_IDS", "")),
diff --git a/lib/features/known_tickers.py b/lib/features/known_tickers.py
index c68e990..a2eead8 100644
--- a/lib/features/known_tickers.py
+++ b/lib/features/known_tickers.py
@@ -28,14 +28,20 @@
 HEADER_NAMES = ("symbol", "ticker", "ticker symbol", "code", "asx code")
 
 
+# Where a US list's ticker links point -> the exchange as Google Finance names it (the S&P 500 list links each
+# ticker to its exchange's own quote page).
+EXCHANGE_SITES = {"nyse.com": "NYSE", "nasdaq.com": "NASDAQ", "cboe.com": "BATS"}
+
+
 class _Tables(HTMLParser):
-    """Every  on a page as {"id": ..., "rows": [[cell text, ...], ...]}, nested ones included."""
+    """Every 
 on a page as {"id": ..., "rows": [[{"text": [...], "href": first link}, ...], ...]}, nested
+    ones included."""
 
     def __init__(self) -> None:
         super().__init__()
         self.tables: list[dict] = []
         self._open: list[dict] = []
-        self._cell: list[str] | None = None
+        self._cell: dict | None = None
 
     def handle_starttag(self, tag, attrs):
         if tag == "table":
@@ -45,8 +51,10 @@ def handle_starttag(self, tag, attrs):
         elif tag == "tr" and self._open:
             self._open[-1]["rows"].append([])
         elif tag in ("td", "th") and self._open and self._open[-1]["rows"]:
-            self._cell = []
+            self._cell = {"text": [], "href": ""}
             self._open[-1]["rows"][-1].append(self._cell)
+        elif tag == "a" and self._cell is not None and not self._cell["href"]:
+            self._cell["href"] = dict(attrs).get("href") or ""
 
     def handle_endtag(self, tag):
         if tag == "table" and self._open:
@@ -56,31 +64,45 @@ def handle_endtag(self, tag):
 
     def handle_data(self, data):
         if self._cell is not None:
-            self._cell.append(data)
+            self._cell["text"].append(data)
 
 
 def symbols_from_html(page: str, suffix: str = "") -> list[str]:
     """The ticker column of a Wikipedia index page: the table with id "constituents", else the first table with a
     "Symbol", "Ticker" or "Code" header. US share classes as Yahoo writes them (BRK.B -> BRK-B); ASX codes as listed."""
+    return list(listings(page, suffix))
+
+
+def listings(page: str, suffix: str = "") -> dict[str, str]:
+    """symbols_from_html's codes, each with the exchange its link points to ("NYSE", "NASDAQ"; "" if none)."""
     parser = _Tables()
     parser.feed(page)
     tables = sorted(parser.tables, key=lambda t: t["id"] != "constituents")
     for table in tables:
-        rows = [["".join(cell).strip() for cell in row] for row in table["rows"]]
+        rows = table["rows"]
         if not rows:
             continue
-        header = [re.sub(r"\[.*?\]", "", h).strip().lower() for h in rows[0]]  # "Ticker[12]": a footnote
+        header = [re.sub(r"\[.*?\]", "", "".join(h["text"])).strip().lower() for h in rows[0]]  # "Ticker[12]"
         col = next((i for i, h in enumerate(header) if h in HEADER_NAMES), None)
         if col is None:
             continue
-        found = [row[col] if suffix else row[col].replace(".", "-") for row in rows[1:] if len(row) > col]
-        return [s for s in found if (ASX_SYMBOL_RE if suffix else US_SYMBOL_RE).match(s)]
-    return []
+        found = {}
+        for row in rows[1:]:
+            if len(row) <= col:
+                continue
+            code = "".join(row[col]["text"]).strip()
+            code = code if suffix else code.replace(".", "-")
+            if (ASX_SYMBOL_RE if suffix else US_SYMBOL_RE).match(code):
+                href = row[col]["href"]
+                found[code] = next((x for site, x in EXCHANGE_SITES.items() if site in href), "")
+        return found
+    return {}
 
 
 async def fetch_index(client: httpx.AsyncClient, name: str, title: str, suffix: str, low: int,
-                      high: int) -> list[str] | None:
-    """An index's codes, or None if the page couldn't be read or doesn't look right."""
+                      high: int) -> dict[str, str] | None:
+    """An index's codes, each with its exchange where the page says ("" if not), or None if the page couldn't be read
+    or doesn't look right."""
     params = {"action": "parse", "page": title, "prop": "text", "format": "json", "formatversion": "2",
               "redirects": "1"}
     try:
@@ -93,17 +115,19 @@ async def fetch_index(client: httpx.AsyncClient, name: str, title: str, suffix:
     except (httpx.HTTPError, ValueError, KeyError, TypeError) as e:
         log.warning("Known tickers: couldn't fetch the %s list: %s", name, e)
         return None
-    found = symbols_from_html(page, suffix)
+    found = listings(page, suffix)
     if not low <= len(found) <= high:
         log.warning("Known tickers: the %s page gave %d tickers (expected %d-%d); keeping the last list",
                     name, len(found), low, high)
         return None
+    if name == "NASDAQ-100":
+        found = dict.fromkeys(found, "NASDAQ")  # its members are NASDAQ listings by definition
     return found
 
 
-async def held_symbols(ctx: Ctx) -> dict[str, str] | None:
-    """The Sharesight holdings on exchanges we know, code -> Yahoo symbol ({"NXT": "NXT.AX", "GM": "GM"}), or None if
-    Sharesight isn't up or isn't set up."""
+async def held_symbols(ctx: Ctx) -> tuple[dict[str, str], dict[str, str]] | None:
+    """The Sharesight holdings on exchanges we know: code -> Yahoo symbol ({"NXT": "NXT.AX", "GM": "GM"}) and, for US
+    ones, code -> exchange ({"GM": "NYSE"}); None if Sharesight isn't up or isn't set up."""
     server = next((s for s in ctx.registry.up() if s.cfg.get("gate") == "portfolio"), None)
     if server is None or not ctx.st.portfolio_names:
         return None
@@ -112,14 +136,15 @@ async def held_symbols(ctx: Ctx) -> dict[str, str] | None:
     except Exception as e:  # the MCP server's own errors, bad JSON
         log.warning("Known tickers: couldn't read the holdings: %s", e)
         return None
-    symbols = {}
+    symbols, exchanges = {}, {}
     for code, market in (code_and_market(key) for key in held):
         symbol = tickers.yahoo_symbol(code, market)
         if symbol == code:  # a US listing: Yahoo writes share classes BRK-B
             code = symbol = code.replace(".", "-")
+            exchanges[code] = market.upper()
         if symbol:
             symbols[code] = symbol
-    return symbols
+    return symbols, exchanges
 
 
 def known(saved: dict) -> tuple[dict[str, str], set[str]]:
@@ -145,6 +170,8 @@ def apply(saved: dict) -> set[str]:
     """Make the saved lists the ones replies link; returns the ambiguous codes left out."""
     codes, ambiguous = known(saved)
     tickers.set_known(codes)
+    exchanges = {c: x for c, x in saved.get("exchanges", {}).items() if x}
+    tickers.set_us_exchanges({**exchanges, **{c.replace("-", "."): x for c, x in exchanges.items() if "-" in c}})
     return ambiguous
 
 
@@ -162,22 +189,28 @@ async def refresh(ctx: Ctx, saved: dict, client: httpx.AsyncClient) -> dict:
     """Fetch the indexes and the holdings; a part that fails keeps its last list and is tried again sooner
     (KNOWN_TICKERS_RETRY) than the daily refresh."""
     indexes, failed = dict(saved.get("indexes", {})), False
+    exchanges = dict(saved.get("exchanges", {}))
     for name, title, suffix, low, high in config.INDEX_LISTS:
         found = await fetch_index(client, name, title, suffix, low, high)
         if found is None:
             failed = True
         else:
-            indexes[name] = found
-    held = await held_symbols(ctx)
+            indexes[name] = list(found)
+            exchanges.update({c: x for c, x in found.items() if x and not suffix})
+    got = await held_symbols(ctx)
+    held = None if got is None else got[0]
+    if got is not None:
+        exchanges.update(got[1])
     failed |= held is None and bool(ctx.st.portfolio_names) and any(
         s.cfg.get("gate") == "portfolio" for s in ctx.registry.servers.values())
     ts = int(time.time()) - (config.KNOWN_TICKERS_REFRESH - config.KNOWN_TICKERS_RETRY if failed else 0)
-    new = {"indexes": indexes, "held": saved.get("held", {}) if held is None else held, "ts": ts}
+    new = {"indexes": indexes, "held": saved.get("held", {}) if held is None else held, "exchanges": exchanges,
+           "ts": ts}
     await asyncio.to_thread(ctx.store.kv_set, KV_KEY, json.dumps(new))
     ambiguous = apply(new)
-    log.info("Known tickers: %s, %d held, %d in all (%d on two exchanges left out)",
+    log.info("Known tickers: %s, %d held, %d in all (%d on two exchanges left out), %d US with their exchange",
              ", ".join(f"{len(v)} {k}" for k, v in indexes.items()) or "no index lists", len(new["held"]),
-             len(tickers.KNOWN), len(ambiguous))
+             len(tickers.KNOWN), len(ambiguous), len(exchanges))
     return new
 
 
diff --git a/lib/tickers.py b/lib/tickers.py
index 35d3b35..32dd8a1 100644
--- a/lib/tickers.py
+++ b/lib/tickers.py
@@ -42,7 +42,7 @@ def is_ticker(word: str) -> bool:
 GOOGLE_QUOTE = "https://www.google.com/finance/quote/"
 GOOGLE_SEARCH = "https://www.google.com/finance?q="
 QUOTE_PREFIXES = (YAHOO_QUOTE, GOOGLE_QUOTE, GOOGLE_SEARCH)  # history.compact and news_codes read these links back
-PROVIDER = ["yahoo"]  # FINANCE_LINK_PROVIDER, set at startup (app.build_ctx)
+TARGET = ["yahoo"]  # FINANCE_LINK_TARGET, set at startup (app.build_ctx)
 # Yahoo's suffix -> Google Finance's exchange code ("NXT.AX" -> "NXT:ASX")
 GOOGLE_EXCHANGES = {
     "AX": "ASX", "NZ": "NZE", "L": "LON", "TO": "TSE", "V": "CVE", "HK": "HKG", "T": "TYO", "KS": "KRX",
@@ -52,8 +52,19 @@ def is_ticker(word: str) -> bool:
 }
 
 
-def set_provider(name: str) -> None:
-    PROVIDER[0] = name
+# A US listing's exchange as Google Finance names it ("JPM": "NYSE"), from features/known_tickers.py; Yahoo needs none.
+US_EXCHANGES: dict[str, str] = {}
+GOOGLE_US = {"NASDAQ": "NASDAQ", "NYSE": "NYSE", "NYSEARCA": "NYSEARCA", "NYSEAMERICAN": "NYSEAMERICAN",
+             "AMEX": "NYSEAMERICAN", "CBOE": "BATS", "BATS": "BATS", "OTC": "OTCMKTS", "OTCMKTS": "OTCMKTS"}
+
+
+def set_target(name: str) -> None:
+    TARGET[0] = name
+
+
+def set_us_exchanges(exchanges: Mapping[str, str]) -> None:
+    US_EXCHANGES.clear()
+    US_EXCHANGES.update(exchanges)
 
 
 def yahoo_url(symbol: str) -> str:
@@ -61,25 +72,29 @@ def yahoo_url(symbol: str) -> str:
     return YAHOO_QUOTE + quote(sym, safe="")
 
 
-def google_url(symbol: str) -> str:
-    """Google Finance's page for a Yahoo symbol: CODE:EXCHANGE where the suffix names it (NXT.AX -> NXT:ASX, 0700.HK ->
-    700:HKG), BTC-USD as is, else a search (a US ticker gives no exchange: NASDAQ or NYSE)."""
+def google_url(symbol: str, market: str = "") -> str:
+    """Google Finance's page for a Yahoo symbol: CODE:EXCHANGE (NXT.AX -> NXT:ASX, 0700.HK -> 700:HKG, BRK-B ->
+    BRK.B:NYSE; a US exchange comes from `market`, as the text wrote it, or US_EXCHANGES), BTC-USD as is, else a
+    search, for a US ticker whose exchange we don't know."""
     base, _, suffix = symbol.rpartition(".")
     if base and suffix in GOOGLE_EXCHANGES:
         code = base.lstrip("0") or "0" if suffix == "HK" else base
         return GOOGLE_QUOTE + quote(f"{code}:{GOOGLE_EXCHANGES[suffix]}", safe=":")
     if symbol in CRYPTO or symbol.endswith("-USD"):
         return GOOGLE_QUOTE + quote(symbol if symbol.endswith("-USD") else f"{symbol}-USD", safe="")
-    return GOOGLE_SEARCH + quote(symbol.replace("-", "."), safe="")  # BRK-B is BRK.B at Google
+    exchange = GOOGLE_US.get(market.upper()) or US_EXCHANGES.get(symbol)
+    if exchange and not symbol.startswith("^"):
+        return GOOGLE_QUOTE + quote(f"{symbol.replace('-', '.')}:{exchange}", safe=":.")  # BRK-B is BRK.B at Google
+    return GOOGLE_SEARCH + quote(symbol.replace("-", "."), safe="")
 
 
-def quote_url(symbol: str) -> str:
-    """The quote page for a Yahoo symbol, at the FINANCE_LINK_PROVIDER."""
-    return google_url(symbol) if PROVIDER[0] == "google" else yahoo_url(symbol)
+def quote_url(symbol: str, market: str = "") -> str:
+    """The quote page for a Yahoo symbol, at the FINANCE_LINK_TARGET (`market`: the exchange the text named)."""
+    return google_url(symbol, market) if TARGET[0] == "google" else yahoo_url(symbol)
 
 
 def quote_labels(html: str) -> set[str]:
-    """The labels of the quote links in a reply (the codes as shown: NXT, 700), whichever provider made them."""
+    """The labels of the quote links in a reply (the codes as shown: NXT, 700), whichever site they point to."""
     starts = "|".join(re.escape(p) for p in QUOTE_PREFIXES)
     return set(re.findall(rf'([^<]+)', html))
 
@@ -138,8 +153,8 @@ def _link_re(codes: frozenset[str]) -> re.Pattern:
     return re.compile(_MARKERS.replace("{known}", known))
 
 
-def _link(symbol: str, label: str) -> str:
-    return f'{label}'
+def _link(symbol: str, label: str, market: str = "") -> str:
+    return f'{label}'
 
 
 def _paren(paren: str, known: Mapping[str, str]) -> str | None:
@@ -173,7 +188,7 @@ def plain() -> str:  # not a ticker in brackets ("NVDA (Q3 FY26)"): the words be
             symbol = yahoo_symbol(code, paren) if re.fullmatch(_CODE, code) else None
             if symbol is None or (not code.isdigit() and not is_ticker(code) and code not in known):
                 return plain()
-            return " ".join(words[:-1] + [_link(symbol, code)]) + f" ({paren})"
+            return " ".join(words[:-1] + [_link(symbol, code, paren)]) + f" ({paren})"
         symbol = _paren(paren, known)
         if symbol is None:
             return plain()
@@ -182,12 +197,12 @@ def plain() -> str:  # not a ticker in brackets ("NVDA (Q3 FY26)"): the words be
         while skip < len(words) - 1 and words[skip] in SENTENCE_OPENERS:
             skip += 1
         lead = " ".join(words[:skip]) + " " if skip else ""
-        body = f"{' '.join(words[skip:])} ({_link(symbol, label)})"
+        body = f"{' '.join(words[skip:])} ({_link(symbol, label, paren.partition(':')[0] if ':' in paren else '')})"
         return lead + (body if bold else f"{body}")
     if m.group("exch"):
         code = m.group("ecode")
         symbol = yahoo_symbol(code, m.group("exch"))
-        return _link(symbol, code) if symbol else m.group(0)
+        return _link(symbol, code, m.group("exch")) if symbol else m.group(0)
     if m.group("cash"):
         code = m.group("cash")
         base, _, suffix = code.rpartition(".")
diff --git a/tests/test_known_tickers.py b/tests/test_known_tickers.py
index 24f82f7..f819b1c 100644
--- a/tests/test_known_tickers.py
+++ b/tests/test_known_tickers.py
@@ -12,7 +12,8 @@
 
 
 def page(rows, table_id="constituents", header="Symbol"):
-    body = "".join(f"" for s in rows)
+    site = {"JPM": "https://www.nyse.com/quote/XNYS:JPM", "NVDA": "https://www.nasdaq.com/market-activity/stocks/nvda"}
+    body = "".join(f"" for s in rows)
     return (f"
{s}Co {s}
{s}Co {s}
Founded
"
             f"{body}
{header}Security
")
 
@@ -57,18 +58,22 @@ async def test_refresh_reads_the_api_with_a_contact_and_links_asx_codes_to_ax(en
     sp, ndx, asx = small_lists(monkeypatch)
     st = config.load({**env, "PORTFOLIO_NAMES": "Bob"})
     ctx = Ctx(st, store, ScriptedBackend([]), registry(Sharesight("sharesight", gate="portfolio")), FakeBot())
-    pages = {sp: page(["NVDA", "BRK.B", "QBE", "ALL"]), ndx: page(["NVDA", "AMD"]),
+    pages = {sp: page(["NVDA", "BRK.B", "QBE", "ALL", "JPM"]), ndx: page(["NVDA", "AMD"]),
              asx: page(["BHP", "A2M", "ALL", "QBE"], header="Code")}
     seen: list = []
     before = dict(tickers.KNOWN)
     try:
         saved = await kt.refresh(ctx, {}, wikipedia(pages, seen))
         assert seen[0].url.params["action"] == "parse" and "https://" in seen[0].headers["user-agent"]
-        assert saved["indexes"]["ASX 200"] == ["BHP", "A2M", "ALL", "QBE"]
+        assert saved["indexes"]["ASX 200"] == ["BHP", "A2M", "ALL", "QBE"] and saved["exchanges"]["JPM"] == "NYSE"
         assert saved["held"] == {"GM": "GM", "NXT": "NXT.AX", "QBE": "QBE.AX"}  # MOON is no exchange we know
         k = tickers.KNOWN
         assert k["BHP"] == "BHP.AX" and k["A2M"] == "A2M.AX" and k["NVDA"] == "NVDA" and k["BRK.B"] == "BRK-B"
         assert "ALL" not in k and k["QBE"] == "QBE.AX"  # on both exchanges: left out, unless a holding settles it
+        # the US exchanges, for Google Finance links: the S&P list's ticker links, the NASDAQ-100, the holdings
+        assert saved["exchanges"]["NVDA"] == "NASDAQ" and saved["exchanges"]["AMD"] == "NASDAQ"
+        assert saved["exchanges"]["GM"] == "NYSE" and tickers.US_EXCHANGES["JPM"] == "NYSE"
+        assert "BRK-B" not in saved["exchanges"]  # its test link points nowhere: no exchange, a Google search
         assert saved["ts"] > time.time() - 5  # every part worked: due tomorrow
         # the next day the S&P page refuses and the NASDAQ page looks wrong: both keep yesterday's list
         again = await kt.refresh(ctx, saved, wikipedia({ndx: page(["ONLY"]), asx: pages[asx]}))
@@ -91,6 +96,10 @@ async def test_without_sharesight_the_held_list_stays_as_it_was(env, store):
     assert await kt.held_symbols(ctx) is None
 
 
+def test_the_exchange_comes_from_where_a_ticker_links():
+    assert kt.listings(page(["JPM", "NVDA", "MMM"])) == {"JPM": "NYSE", "NVDA": "NASDAQ", "MMM": ""}
+
+
 async def test_the_loop_waits_for_the_mcp_when_overdue_and_otherwise_until_due(env, store, monkeypatch):
     import asyncio
     import time
diff --git a/tests/test_tickers.py b/tests/test_tickers.py
index 8375217..6c7e7d9 100644
--- a/tests/test_tickers.py
+++ b/tests/test_tickers.py
@@ -154,10 +154,10 @@ def test_link_labelled_with_its_own_url_is_numbered_but_named_links_and_code_are
 def test_google_finance_links_name_the_exchange_where_the_symbol_does(env):
     from lib import config
     from lib.history import compact
-    assert config.load({**env, "FINANCE_LINK_PROVIDER": "Google"}).link_provider == "google"
-    assert config.load(env).link_provider == "yahoo"
+    assert config.load({**env, "FINANCE_LINK_TARGET": "Google"}).link_target == "google"
+    assert config.load(env).link_target == "yahoo"
     try:
-        config.load({**env, "FINANCE_LINK_PROVIDER": "bing"})
+        config.load({**env, "FINANCE_LINK_TARGET": "bing"})
         raise AssertionError("bing accepted")
     except config.ConfigError:
         pass
@@ -166,11 +166,30 @@ def test_google_finance_links_name_the_exchange_where_the_symbol_does(env):
     assert g("0700.HK") == "https://www.google.com/finance/quote/700:HKG" and g("7203.T").endswith("/7203:TYO")
     assert g("BTC") == g("BTC-USD") == "https://www.google.com/finance/quote/BTC-USD"
     assert g("NVDA") == "https://www.google.com/finance?q=NVDA" and g("BRK-B").endswith("?q=BRK.B")
-    tickers.set_provider("google")
+    tickers.set_target("google")
     try:
         out = link_tickers("ASX:NXT and $NVDA")
         assert out == ('NXT and '
                        'NVDA')
         assert compact(out) == "NXT and NVDA" and tickers.quote_labels(out) == {"NXT", "NVDA"}
     finally:
-        tickers.set_provider("yahoo")
+        tickers.set_target("yahoo")
+
+
+def test_google_links_take_the_us_exchange_from_the_text_or_the_known_lists():
+    g = tickers.google_url
+    before = dict(tickers.US_EXCHANGES)
+    try:
+        tickers.set_us_exchanges({"JPM": "NYSE", "SPCX": "NASDAQ", "BRK-B": "NYSE"})
+        assert g("JPM") == "https://www.google.com/finance/quote/JPM:NYSE"
+        assert g("SPCX") == "https://www.google.com/finance/quote/SPCX:NASDAQ"
+        assert g("BRK-B") == "https://www.google.com/finance/quote/BRK.B:NYSE"
+        assert g("CBA.AX") == "https://www.google.com/finance/quote/CBA:ASX"
+        assert g("ACME", "NYSE") == "https://www.google.com/finance/quote/ACME:NYSE"  # NYSE:ACME, ACME (NYSE)
+        assert g("ACME") == "https://www.google.com/finance?q=ACME"  # no exchange known: a search
+        tickers.set_target("google")
+        assert 'quote/NVDA:NASDAQ"' in link_tickers("NASDAQ: NVDA and NVDA (NASDAQ) and Nvidia (NASDAQ:NVDA)")
+        assert 'quote/JPM:NYSE">JPM' in link_tickers("$JPM")
+    finally:
+        tickers.set_target("yahoo")
+        tickers.set_us_exchanges(before)

From ac16638fd0fd58b6c2b80eba11cb44d2d26708ec Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 07:45:39 +0000
Subject: [PATCH 25/30] FINANCE_LINK_TARGET=tradingview; quote URLs move to
 lib/quote_links.py

TradingView pages are EXCHANGE-CODE (ASX-NXT, NYSE-BRK.A, NASDAQ-SPCX), with the US exchange from
the same sources as Google's; a code it can't place opens TradingView's page for the bare code.
The three sites' URL builders leave tickers.py (near its size limit) for their own module.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 CLAUDE.md                     |   2 +-
 docs/configuration.md         |   2 +-
 docs/how-it-works.md          |   4 +-
 lib/app.py                    |   4 +-
 lib/config.py                 |   8 +--
 lib/features/holding_news.py  |   3 +-
 lib/features/known_tickers.py |   4 +-
 lib/history.py                |   2 +-
 lib/quote_links.py            | 101 ++++++++++++++++++++++++++++++++++
 lib/tickers.py                |  65 +---------------------
 tests/test_known_tickers.py   |   4 +-
 tests/test_tickers.py         |  47 ++++++++++++----
 12 files changed, 155 insertions(+), 91 deletions(-)
 create mode 100644 lib/quote_links.py

diff --git a/CLAUDE.md b/CLAUDE.md
index 0a3962e..96085ff 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -19,7 +19,7 @@ MCP data tools and web search. User-facing docs are in `README.md` (quick start)
 - `store.py`: SQLite `Store`: history (`messages` shared by groups, `dm_messages` per bot), users, kv,
   holding-news state, `usage` ledger, `text_cache` (finished /narrative and /events messages, 24 h), `period_cache` (finished Sharesight periods, kept until flushed). Thread-safe; call through `asyncio.to_thread`.
 - `msgtext.py` sender_name/describe. `history.py` transcript lines. `textfmt.py` HTML helpers
-  (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` Yahoo (or Google, FINANCE_LINK_TARGET) links on marked tickers (`$X`, `X.AX`, `ASX:X`, `Name (X)`, `X (ASX)`) and the known set `tickers.KNOWN`; bare words otherwise never link (+ `tickers_data.py`: exchange prefixes -> Yahoo suffixes, the gate's NOT_TICKERS). `holdings.py` current Sharesight holdings (holding news, known tickers).
+  (md_to_html, split_html, plain_text, TG_TAG_RE). `tickers.py` quote links (`quote_links.py`: Yahoo, Google or TradingView, FINANCE_LINK_TARGET) on marked tickers (`$X`, `X.AX`, `ASX:X`, `Name (X)`, `X (ASX)`) and the known set `tickers.KNOWN`; bare words otherwise never link (+ `tickers_data.py`: exchange prefixes -> Yahoo suffixes, the gate's NOT_TICKERS). `holdings.py` current Sharesight holdings (holding news, known tickers).
 - `mcp/`: `server.py` (MCPServer lifecycle, TTL cache, `Registry`), `schema.py` (neutral `ToolDef`,
   schema diet), `results.py` (result slimming).
 - `llm/`: `base.py` (Request/Step/Answer/Backend), `chat.py` (shared chat-completions loop), `xai.py` +
diff --git a/docs/configuration.md b/docs/configuration.md
index d03c850..af80cb3 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -26,7 +26,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o
 | `ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/tokens`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. |
 | `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/tokens` and `/flushcache` stay admin-only. Who counts as an admin: `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). |
 | `DATA_DIR` | systemd's `STATE_DIRECTORY`, else the working directory | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. With systemd the default is its `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)). In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. |
-| `FINANCE_LINK_TARGET` | `yahoo` | Where ticker links go: `yahoo` (Yahoo Finance) or `google` (Google Finance, `CODE:EXCHANGE`: `NXT.AX` opens `NXT:ASX`, `JPM` opens `JPM:NYSE`). A US ticker's exchange comes from the text (`NYSE:JPM`, `JPM (NYSE)`), the holdings, the NASDAQ-100 or the S&P 500 list; one the bot can't place opens a Google Finance search. |
+| `FINANCE_LINK_TARGET` | `yahoo` | Where ticker links go: `yahoo` (Yahoo Finance), `google` (Google Finance, `CODE:EXCHANGE`: `NXT.AX` opens `NXT:ASX`, `JPM` opens `JPM:NYSE`) or `tradingview` (`EXCHANGE-CODE`: `ASX-NXT`, `NYSE-BRK.A`). A US ticker's exchange comes from the text (`NYSE:JPM`, `JPM (NYSE)`), the holdings, the NASDAQ-100 or the S&P 500 list; one the bot can't place opens a Google Finance search, or TradingView's page for the bare code. |
 | `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. |
 | `MAX_TOKENS` | `3000` | Reply cap in tokens, reasoning included. |
 | `MOVERS_BOTS` | empty (feature off) | Usernames of bots whose end-of-day big-movers lists get explained. |
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index bc60d37..863e4f5 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -104,8 +104,8 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf
 ## Formatting and ticker links
 - Replies are Telegram HTML. Markdown the model slips in (`**bold**`, links, backticks) is
   converted; trailing "not advice"/"NFA"/"DYOR" disclaimers are stripped.
-- **Ticker links:** only a ticker written in a marked form is linked to its Yahoo Finance page (or Google Finance,
-  with `FINANCE_LINK_TARGET=google`): `$NVDA`,
+- **Ticker links:** only a ticker written in a marked form is linked to its Yahoo Finance page (or Google Finance or TradingView,
+  with `FINANCE_LINK_TARGET`): `$NVDA`,
   an exchange suffix (`NXT.AX`, `000660.KS`), an exchange prefix (`ASX:NXT`, `NASDAQ: NVDA`, mapped to Yahoo's suffix:
   `ASX:NXT` -> `NXT.AX`), `Name (TICKER)` (bold, name and ticker together, unless already bold), Sharesight's
   `NXT (ASX)`, `^GSPC` and `BTC-USD`. The system prompt asks the model for these forms. A bare word is linked only
diff --git a/lib/app.py b/lib/app.py
index 164da28..d205681 100644
--- a/lib/app.py
+++ b/lib/app.py
@@ -32,7 +32,7 @@
     filters,
 )
 
-from . import config, tickers
+from . import config, quote_links
 from .context import Ctx
 from .features import chat_commands, dm_buttons, holding_news, known_tickers, post
 from .features.charts import buttons as chart_buttons
@@ -163,7 +163,7 @@ async def register_menu(ctx: Ctx) -> None:
 
 def build_ctx(st: config.Settings) -> Ctx:
     ctx = Ctx(st, Store(st.db_path), build_backend(st), Registry([]))
-    tickers.set_target(st.link_target)
+    quote_links.set_target(st.link_target)
 
     async def on_down(server: MCPServer) -> None:
         await alert_down(ctx, server)
diff --git a/lib/config.py b/lib/config.py
index 18dc320..0bb8ebc 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -26,7 +26,7 @@
     "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /tokens; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.",
     "ADMIN_ONLY": "on|off. On (default): only admins may DM the bot or get Sharesight data. Off: everyone may. /tokens and /flushcache stay admin-only.",
     "DATA_DIR": "Directory for the bot's data (the database); Default: systemd's STATE_DIRECTORY, else the working directory.",
-    "FINANCE_LINK_TARGET": "Where ticker links go: yahoo (default, Yahoo Finance) or google (Google Finance).",
+    "FINANCE_LINK_TARGET": "Where ticker links go: yahoo (default, Yahoo Finance), google (Google Finance) or tradingview.",
     "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.",
     "MAX_TOKENS": "Reply cap in tokens, reasoning included (default 3000).",
     "MOVERS_BOTS": "Usernames of bots whose end-of-day big-movers lists get explained. Empty (default) = feature off.",
@@ -89,7 +89,7 @@ class Settings:
     max_tokens: int
     history_max: int
     darkmode: str  # "off" | "on" | "auto": the charts' theme
-    link_target: str  # "yahoo" | "google": where ticker links go
+    link_target: str  # "yahoo" | "google" | "tradingview": where ticker links go
     db_path: str
     tz: ZoneInfo
     admin_chats: frozenset[int]  # user IDs (alerts + admin commands) and group chat IDs (alerts only)
@@ -189,8 +189,8 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
     if provider == "openrouter" and not search_model:
         search = False  # nothing to run the searches with
     link_target = env.get("FINANCE_LINK_TARGET", "").strip().lower() or "yahoo"
-    if link_target not in ("yahoo", "google"):
-        raise ConfigError(f"FINANCE_LINK_TARGET must be yahoo or google, not {link_target!r}.")
+    if link_target not in ("yahoo", "google", "tradingview"):
+        raise ConfigError(f"FINANCE_LINK_TARGET must be yahoo, google or tradingview, not {link_target!r}.")
     darkmode = env.get("DARKMODE", "").strip().lower() or "off"
     if darkmode not in ("off", "on", "auto"):
         raise ConfigError(f"DARKMODE must be off, on or auto, not {darkmode!r}.")
diff --git a/lib/features/holding_news.py b/lib/features/holding_news.py
index aba5c88..24f5d6d 100644
--- a/lib/features/holding_news.py
+++ b/lib/features/holding_news.py
@@ -21,10 +21,11 @@
 from ..llm.base import BUSY_REPLY, is_busy
 from ..llm.gate import Route
 from ..prompts import HOLDING_NEWS_PROMPT
+from ..quote_links import quote_labels
 from ..telegram.draft import Draft
 from ..telegram.send import send_html
 from ..textfmt import ANY_TAG_RE, md_to_html, split_html, strip_disclaimer
-from ..tickers import link_tickers, quote_labels, yahoo_symbol
+from ..tickers import link_tickers, yahoo_symbol
 
 log = logging.getLogger("bot")
 UNSUBSCRIBE_BUTTON = InlineKeyboardMarkup([[InlineKeyboardButton("🔕 Unsubscribe…", callback_data="hn:menu")]])
diff --git a/lib/features/known_tickers.py b/lib/features/known_tickers.py
index a2eead8..f638904 100644
--- a/lib/features/known_tickers.py
+++ b/lib/features/known_tickers.py
@@ -14,7 +14,7 @@
 
 import httpx
 
-from .. import config, tickers
+from .. import config, quote_links, tickers
 from ..context import Ctx
 from ..holdings import code_and_market, current_holdings
 
@@ -171,7 +171,7 @@ def apply(saved: dict) -> set[str]:
     codes, ambiguous = known(saved)
     tickers.set_known(codes)
     exchanges = {c: x for c, x in saved.get("exchanges", {}).items() if x}
-    tickers.set_us_exchanges({**exchanges, **{c.replace("-", "."): x for c, x in exchanges.items() if "-" in c}})
+    quote_links.set_us_exchanges({**exchanges, **{c.replace("-", "."): x for c, x in exchanges.items() if "-" in c}})
     return ambiguous
 
 
diff --git a/lib/history.py b/lib/history.py
index cc5b8df..c39b416 100644
--- a/lib/history.py
+++ b/lib/history.py
@@ -7,9 +7,9 @@
 from datetime import datetime
 from zoneinfo import ZoneInfo
 
+from .quote_links import QUOTE_PREFIXES
 from .store import HistoryRow
 from .textfmt import ANY_TAG_RE
-from .tickers import QUOTE_PREFIXES
 
 LAST_REPLY_MAX = 6000  # characters of the bot's latest reply kept in the prompt (~1,500 tokens)
 _LINK = re.compile(r'(.*?)', re.DOTALL)
diff --git a/lib/quote_links.py b/lib/quote_links.py
new file mode 100644
index 0000000..d6b390c
--- /dev/null
+++ b/lib/quote_links.py
@@ -0,0 +1,101 @@
+"""Where ticker links go (FINANCE_LINK_TARGET): Yahoo Finance, Google Finance or TradingView, built from a Yahoo symbol."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from urllib.parse import quote
+
+from .tickers_data import CRYPTO
+
+YAHOO_QUOTE = "https://finance.yahoo.com/quote/"
+GOOGLE_QUOTE = "https://www.google.com/finance/quote/"
+GOOGLE_SEARCH = "https://www.google.com/finance?q="
+TRADINGVIEW = "https://www.tradingview.com/symbols/"
+QUOTE_PREFIXES = (YAHOO_QUOTE, GOOGLE_QUOTE, GOOGLE_SEARCH, TRADINGVIEW)  # history.compact, news_codes read these back
+TARGETS = ("yahoo", "google", "tradingview")
+TARGET = ["yahoo"]  # set at startup (app.build_ctx)
+
+# Yahoo's suffix -> the exchange as each site names it ("NXT.AX" -> Google "NXT:ASX", TradingView "ASX-NXT")
+GOOGLE_EXCHANGES = {
+    "AX": "ASX", "NZ": "NZE", "L": "LON", "TO": "TSE", "V": "CVE", "HK": "HKG", "T": "TYO", "KS": "KRX",
+    "KQ": "KOSDAQ", "DE": "ETR", "F": "FRA", "PA": "EPA", "AS": "AMS", "BR": "EBR", "SW": "SWX", "MI": "BIT",
+    "MC": "BME", "ST": "STO", "OL": "OSL", "CO": "CPH", "HE": "HEL", "SI": "SGX", "SS": "SHA", "SZ": "SHE",
+    "TW": "TPE", "BO": "BOM", "NS": "NSE", "SA": "BVMF", "MX": "BMV", "JO": "JSE", "TA": "TLV",
+}
+TRADINGVIEW_EXCHANGES = {
+    "AX": "ASX", "NZ": "NZX", "L": "LSE", "TO": "TSX", "V": "TSXV", "HK": "HKEX", "T": "TSE", "KS": "KRX",
+    "KQ": "KRX", "DE": "XETR", "F": "FWB", "PA": "EURONEXT", "AS": "EURONEXT", "BR": "EURONEXT", "SW": "SIX",
+    "MI": "MIL", "MC": "BME", "ST": "OMXSTO", "OL": "OSL", "CO": "OMXCOP", "HE": "OMXHEX", "SI": "SGX", "SS": "SSE",
+    "SZ": "SZSE", "TW": "TWSE", "BO": "BSE", "NS": "NSE", "SA": "BMFBOVESPA", "MX": "BMV", "JO": "JSE", "TA": "TASE",
+}
+# A US listing's exchange as Google Finance names it ("JPM": "NYSE"), from features/known_tickers.py; Yahoo needs none.
+US_EXCHANGES: dict[str, str] = {}
+GOOGLE_US = {"NASDAQ": "NASDAQ", "NYSE": "NYSE", "NYSEARCA": "NYSEARCA", "NYSEAMERICAN": "NYSEAMERICAN",
+             "AMEX": "NYSEAMERICAN", "CBOE": "BATS", "BATS": "BATS", "OTC": "OTCMKTS", "OTCMKTS": "OTCMKTS"}
+TRADINGVIEW_US = {"NASDAQ": "NASDAQ", "NYSE": "NYSE", "NYSEARCA": "AMEX", "NYSEAMERICAN": "AMEX", "BATS": "CBOE",
+                  "OTCMKTS": "OTC"}  # from Google's names above
+TRADINGVIEW_INDICES = {"^GSPC": "SP-SPX", "^IXIC": "NASDAQ-IXIC", "^NDX": "NASDAQ-NDX", "^DJI": "DJ-DJI",
+                       "^AXJO": "ASX-XJO", "^VIX": "CBOE-VIX"}
+
+
+def set_target(name: str) -> None:
+    TARGET[0] = name
+
+
+def set_us_exchanges(exchanges: Mapping[str, str]) -> None:
+    US_EXCHANGES.clear()
+    US_EXCHANGES.update(exchanges)
+
+
+def _listing(symbol: str, market: str, table: Mapping[str, str]) -> tuple[str, str | None]:
+    """A Yahoo symbol as (code, exchange name in `table`), the exchange None when unknown: NXT.AX -> (NXT, ASX),
+    0700.HK -> (700, HKG), BRK-B -> (BRK.B, NYSE) from `market` (as the text named it) or US_EXCHANGES."""
+    base, _, suffix = symbol.rpartition(".")
+    if base and suffix in table:
+        return (base.lstrip("0") or "0") if suffix == "HK" else base, table[suffix]
+    return symbol.replace("-", "."), GOOGLE_US.get(market.upper()) or US_EXCHANGES.get(symbol)
+
+
+def yahoo_url(symbol: str) -> str:
+    sym = f"{symbol}-USD" if symbol in CRYPTO else symbol
+    return YAHOO_QUOTE + quote(sym, safe="")
+
+
+def google_url(symbol: str, market: str = "") -> str:
+    """Google Finance: CODE:EXCHANGE (NXT:ASX, BRK.B:NYSE), BTC-USD as is, else a search (an exchange we can't place)."""
+    if symbol in CRYPTO or symbol.endswith("-USD"):
+        return GOOGLE_QUOTE + quote(symbol if symbol.endswith("-USD") else f"{symbol}-USD", safe="")
+    code, exchange = _listing(symbol, market, GOOGLE_EXCHANGES)
+    if exchange and not symbol.startswith("^"):
+        return GOOGLE_QUOTE + quote(f"{code}:{exchange}", safe=":.")
+    return GOOGLE_SEARCH + quote(code, safe="")
+
+
+def tradingview_url(symbol: str, market: str = "") -> str:
+    """TradingView: EXCHANGE-CODE (ASX-NXT, NYSE-BRK.A); a code alone where we can't place it (TradingView picks its
+    main listing); BTCUSD for crypto."""
+    if symbol in TRADINGVIEW_INDICES:
+        return f"{TRADINGVIEW}{TRADINGVIEW_INDICES[symbol]}/"
+    if symbol in CRYPTO or symbol.endswith("-USD"):
+        return f"{TRADINGVIEW}{quote(symbol.removesuffix('-USD') + 'USD', safe='')}/"
+    code, exchange = _listing(symbol, market, TRADINGVIEW_EXCHANGES)
+    if exchange and exchange not in TRADINGVIEW_EXCHANGES.values():
+        exchange = TRADINGVIEW_US.get(exchange)  # a US exchange, as Google names it
+    name = f"{exchange}-{code}" if exchange else code.lstrip("^")
+    return f"{TRADINGVIEW}{quote(name, safe='.-')}/"
+
+
+def quote_url(symbol: str, market: str = "") -> str:
+    """The quote page for a Yahoo symbol at the FINANCE_LINK_TARGET (`market`: the exchange the text named)."""
+    if TARGET[0] == "google":
+        return google_url(symbol, market)
+    if TARGET[0] == "tradingview":
+        return tradingview_url(symbol, market)
+    return yahoo_url(symbol)
+
+
+def quote_labels(html: str) -> set[str]:
+    """The labels of the quote links in a reply (the codes as shown: NXT, 700), whichever site they point to."""
+    starts = "|".join(re.escape(p) for p in QUOTE_PREFIXES)
+    return set(re.findall(rf'([^<]+)', html))
diff --git a/lib/tickers.py b/lib/tickers.py
index 32dd8a1..0416df6 100644
--- a/lib/tickers.py
+++ b/lib/tickers.py
@@ -5,9 +5,9 @@
 import re
 from collections.abc import Mapping
 from functools import lru_cache
-from urllib.parse import quote
 
-from .tickers_data import CRYPTO, EXCHANGE_PREFIXES, EXCHANGE_SUFFIXES, NOT_TICKERS, WORD_TICKERS
+from .quote_links import quote_url
+from .tickers_data import EXCHANGE_PREFIXES, EXCHANGE_SUFFIXES, NOT_TICKERS, WORD_TICKERS
 
 # TICKER_RE, ONE_LETTER_RE and is_ticker read the user's text (llm/gate.py, charts/breakdown.py); replies link
 # only the marked forms below and known tickers.
@@ -38,67 +38,6 @@ def is_ticker(word: str) -> bool:
     return not (word in NOT_TICKERS or NOT_TICKER_RE.match(word))
 
 
-YAHOO_QUOTE = "https://finance.yahoo.com/quote/"
-GOOGLE_QUOTE = "https://www.google.com/finance/quote/"
-GOOGLE_SEARCH = "https://www.google.com/finance?q="
-QUOTE_PREFIXES = (YAHOO_QUOTE, GOOGLE_QUOTE, GOOGLE_SEARCH)  # history.compact and news_codes read these links back
-TARGET = ["yahoo"]  # FINANCE_LINK_TARGET, set at startup (app.build_ctx)
-# Yahoo's suffix -> Google Finance's exchange code ("NXT.AX" -> "NXT:ASX")
-GOOGLE_EXCHANGES = {
-    "AX": "ASX", "NZ": "NZE", "L": "LON", "TO": "TSE", "V": "CVE", "HK": "HKG", "T": "TYO", "KS": "KRX",
-    "KQ": "KOSDAQ", "DE": "ETR", "F": "FRA", "PA": "EPA", "AS": "AMS", "BR": "EBR", "SW": "SWX", "MI": "BIT",
-    "MC": "BME", "ST": "STO", "OL": "OSL", "CO": "CPH", "HE": "HEL", "SI": "SGX", "SS": "SHA", "SZ": "SHE",
-    "TW": "TPE", "BO": "BOM", "NS": "NSE", "SA": "BVMF", "MX": "BMV", "JO": "JSE", "TA": "TLV",
-}
-
-
-# A US listing's exchange as Google Finance names it ("JPM": "NYSE"), from features/known_tickers.py; Yahoo needs none.
-US_EXCHANGES: dict[str, str] = {}
-GOOGLE_US = {"NASDAQ": "NASDAQ", "NYSE": "NYSE", "NYSEARCA": "NYSEARCA", "NYSEAMERICAN": "NYSEAMERICAN",
-             "AMEX": "NYSEAMERICAN", "CBOE": "BATS", "BATS": "BATS", "OTC": "OTCMKTS", "OTCMKTS": "OTCMKTS"}
-
-
-def set_target(name: str) -> None:
-    TARGET[0] = name
-
-
-def set_us_exchanges(exchanges: Mapping[str, str]) -> None:
-    US_EXCHANGES.clear()
-    US_EXCHANGES.update(exchanges)
-
-
-def yahoo_url(symbol: str) -> str:
-    sym = f"{symbol}-USD" if symbol in CRYPTO else symbol
-    return YAHOO_QUOTE + quote(sym, safe="")
-
-
-def google_url(symbol: str, market: str = "") -> str:
-    """Google Finance's page for a Yahoo symbol: CODE:EXCHANGE (NXT.AX -> NXT:ASX, 0700.HK -> 700:HKG, BRK-B ->
-    BRK.B:NYSE; a US exchange comes from `market`, as the text wrote it, or US_EXCHANGES), BTC-USD as is, else a
-    search, for a US ticker whose exchange we don't know."""
-    base, _, suffix = symbol.rpartition(".")
-    if base and suffix in GOOGLE_EXCHANGES:
-        code = base.lstrip("0") or "0" if suffix == "HK" else base
-        return GOOGLE_QUOTE + quote(f"{code}:{GOOGLE_EXCHANGES[suffix]}", safe=":")
-    if symbol in CRYPTO or symbol.endswith("-USD"):
-        return GOOGLE_QUOTE + quote(symbol if symbol.endswith("-USD") else f"{symbol}-USD", safe="")
-    exchange = GOOGLE_US.get(market.upper()) or US_EXCHANGES.get(symbol)
-    if exchange and not symbol.startswith("^"):
-        return GOOGLE_QUOTE + quote(f"{symbol.replace('-', '.')}:{exchange}", safe=":.")  # BRK-B is BRK.B at Google
-    return GOOGLE_SEARCH + quote(symbol.replace("-", "."), safe="")
-
-
-def quote_url(symbol: str, market: str = "") -> str:
-    """The quote page for a Yahoo symbol, at the FINANCE_LINK_TARGET (`market`: the exchange the text named)."""
-    return google_url(symbol, market) if TARGET[0] == "google" else yahoo_url(symbol)
-
-
-def quote_labels(html: str) -> set[str]:
-    """The labels of the quote links in a reply (the codes as shown: NXT, 700), whichever site they point to."""
-    starts = "|".join(re.escape(p) for p in QUOTE_PREFIXES)
-    return set(re.findall(rf'([^<]+)', html))
-
-
 def ticker_label(symbol: str) -> str:
     """What the reader sees: SQX.AX becomes SQX, BRK.B stays BRK.B."""
     base, _, suffix = symbol.partition(".")
diff --git a/tests/test_known_tickers.py b/tests/test_known_tickers.py
index f819b1c..831c3fd 100644
--- a/tests/test_known_tickers.py
+++ b/tests/test_known_tickers.py
@@ -3,7 +3,7 @@
 
 import httpx
 
-from lib import config, tickers
+from lib import config, quote_links, tickers
 from lib.context import Ctx
 from lib.features import known_tickers as kt
 
@@ -72,7 +72,7 @@ async def test_refresh_reads_the_api_with_a_contact_and_links_asx_codes_to_ax(en
         assert "ALL" not in k and k["QBE"] == "QBE.AX"  # on both exchanges: left out, unless a holding settles it
         # the US exchanges, for Google Finance links: the S&P list's ticker links, the NASDAQ-100, the holdings
         assert saved["exchanges"]["NVDA"] == "NASDAQ" and saved["exchanges"]["AMD"] == "NASDAQ"
-        assert saved["exchanges"]["GM"] == "NYSE" and tickers.US_EXCHANGES["JPM"] == "NYSE"
+        assert saved["exchanges"]["GM"] == "NYSE" and quote_links.US_EXCHANGES["JPM"] == "NYSE"
         assert "BRK-B" not in saved["exchanges"]  # its test link points nowhere: no exchange, a Google search
         assert saved["ts"] > time.time() - 5  # every part worked: due tomorrow
         # the next day the S&P page refuses and the NASDAQ page looks wrong: both keep yesterday's list
diff --git a/tests/test_tickers.py b/tests/test_tickers.py
index 6c7e7d9..c6c8041 100644
--- a/tests/test_tickers.py
+++ b/tests/test_tickers.py
@@ -1,5 +1,6 @@
-from lib import tickers
-from lib.tickers import link_tickers, ticker_label, yahoo_symbol, yahoo_url
+from lib import quote_links, tickers
+from lib.quote_links import yahoo_url
+from lib.tickers import link_tickers, ticker_label, yahoo_symbol
 
 Y = "https://finance.yahoo.com/quote/"
 
@@ -161,35 +162,57 @@ def test_google_finance_links_name_the_exchange_where_the_symbol_does(env):
         raise AssertionError("bing accepted")
     except config.ConfigError:
         pass
-    g = tickers.google_url
+    g = quote_links.google_url
     assert g("NXT.AX") == "https://www.google.com/finance/quote/NXT:ASX"
     assert g("0700.HK") == "https://www.google.com/finance/quote/700:HKG" and g("7203.T").endswith("/7203:TYO")
     assert g("BTC") == g("BTC-USD") == "https://www.google.com/finance/quote/BTC-USD"
     assert g("NVDA") == "https://www.google.com/finance?q=NVDA" and g("BRK-B").endswith("?q=BRK.B")
-    tickers.set_target("google")
+    quote_links.set_target("google")
     try:
         out = link_tickers("ASX:NXT and $NVDA")
         assert out == ('NXT and '
                        'NVDA')
-        assert compact(out) == "NXT and NVDA" and tickers.quote_labels(out) == {"NXT", "NVDA"}
+        assert compact(out) == "NXT and NVDA" and quote_links.quote_labels(out) == {"NXT", "NVDA"}
     finally:
-        tickers.set_target("yahoo")
+        quote_links.set_target("yahoo")
 
 
 def test_google_links_take_the_us_exchange_from_the_text_or_the_known_lists():
-    g = tickers.google_url
-    before = dict(tickers.US_EXCHANGES)
+    g = quote_links.google_url
+    before = dict(quote_links.US_EXCHANGES)
     try:
-        tickers.set_us_exchanges({"JPM": "NYSE", "SPCX": "NASDAQ", "BRK-B": "NYSE"})
+        quote_links.set_us_exchanges({"JPM": "NYSE", "SPCX": "NASDAQ", "BRK-B": "NYSE"})
         assert g("JPM") == "https://www.google.com/finance/quote/JPM:NYSE"
         assert g("SPCX") == "https://www.google.com/finance/quote/SPCX:NASDAQ"
         assert g("BRK-B") == "https://www.google.com/finance/quote/BRK.B:NYSE"
         assert g("CBA.AX") == "https://www.google.com/finance/quote/CBA:ASX"
         assert g("ACME", "NYSE") == "https://www.google.com/finance/quote/ACME:NYSE"  # NYSE:ACME, ACME (NYSE)
         assert g("ACME") == "https://www.google.com/finance?q=ACME"  # no exchange known: a search
-        tickers.set_target("google")
+        quote_links.set_target("google")
         assert 'quote/NVDA:NASDAQ"' in link_tickers("NASDAQ: NVDA and NVDA (NASDAQ) and Nvidia (NASDAQ:NVDA)")
         assert 'quote/JPM:NYSE">JPM' in link_tickers("$JPM")
     finally:
-        tickers.set_target("yahoo")
-        tickers.set_us_exchanges(before)
+        quote_links.set_target("yahoo")
+        quote_links.set_us_exchanges(before)
+
+
+def test_tradingview_links_are_exchange_dash_code():
+    tv = quote_links.tradingview_url
+    before = dict(quote_links.US_EXCHANGES)
+    try:
+        quote_links.set_us_exchanges({"BRK-A": "NYSE", "SPCX": "NASDAQ", "SPY": "NYSEARCA"})
+        assert tv("NXT.AX") == "https://www.tradingview.com/symbols/ASX-NXT/"
+        assert tv("BRK-A") == "https://www.tradingview.com/symbols/NYSE-BRK.A/"
+        assert tv("SPCX") == "https://www.tradingview.com/symbols/NASDAQ-SPCX/"
+        assert tv("SPY") == "https://www.tradingview.com/symbols/AMEX-SPY/"  # TradingView files NYSE Arca under AMEX
+        assert tv("0700.HK").endswith("/HKEX-700/") and tv("7203.T").endswith("/TSE-7203/")
+        assert tv("ACME", "NYSE").endswith("/NYSE-ACME/") and tv("ACME").endswith("/symbols/ACME/")  # its main listing
+        assert tv("BTC-USD").endswith("/BTCUSD/") and tv("^GSPC").endswith("/SP-SPX/")
+        quote_links.set_target("tradingview")
+        out = link_tickers("ASX:NXT and $SPCX")
+        assert out == ('NXT and '
+                       'SPCX')
+        assert quote_links.quote_labels(out) == {"NXT", "SPCX"}
+    finally:
+        quote_links.set_target("yahoo")
+        quote_links.set_us_exchanges(before)

From 5ea6e16c0ed070107b79806b73b7a837d8d240a9 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 09:05:55 +0000
Subject: [PATCH 26/30] /usage: requests and cost per kind under each period

Shows what the daily holding-news check (and movers, DM buttons, chat commands, filings) costs
next to chat; every request was already recorded with its kind.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 docs/features.md         |  2 +-
 lib/store.py             |  6 ++++++
 lib/telegram/commands.py | 14 +++++++++++---
 tests/test_features.py   |  5 +++++
 tests/test_store.py      |  4 ++++
 5 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/docs/features.md b/docs/features.md
index 996767a..cc27545 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -94,7 +94,7 @@ for its definition.
 |---|---|---|
 | `/start` | DM | Greeting (and the buttons if enabled) |
 | `/holdingnews` | DM, if holding news is on | Run the holding-news check now |
-| `/tokens` (or `/usage`, `/credits`) | an admin (see below), in a group or a DM | Provider balance (OpenRouter; xAI has none to show), then requests, tokens (and % cached) and cost per model for the last 24 h and 7 days |
+| `/tokens` (or `/usage`, `/credits`) | an admin (see below), in a group or a DM | Provider balance (OpenRouter; xAI has none to show), then requests, tokens (and % cached) and cost per model for the last 24 h and 7 days, and the requests and cost per kind (chat, the daily holding-news check, movers, DM buttons, chat commands, filings) |
 | `/ [text]` from `chat_commands` | anyone (admins only for a `gate: portfolio` server) | Commands configured per MCP server in `mcp_servers.json`: the model answers with only that server tool, which it has to call. See [How it works](how-it-works.md#mcp-data-tools). In the command menu |
 | `/clear` | anyone, in a private chat | The bot forgets its stored history of that chat (clearing the chat on your own screen doesn't tell the bot anything, so use this too). Neither the command nor its reply is stored. Not in groups, whose history is shared by everyone and every bot in them. In the command menu of private chats |
 | `/segments TICKER` | anyone, in a group or a DM | The latest quarter's revenue breakdown pies from TickerScout `get_segments`, drawn without the model (the same as a plain "TICKER revenue breakdown" message; see Charts). In the command menu |
diff --git a/lib/store.py b/lib/store.py
index 525883c..ee4a544 100644
--- a/lib/store.py
+++ b/lib/store.py
@@ -260,6 +260,12 @@ def usage_summary(self, since_ts: int) -> list[tuple]:
             "FROM usage WHERE bot_id = ? AND ts >= ? GROUP BY model ORDER BY SUM(cost) DESC",
             (self.bot_id or 0, since_ts))
 
+    def usage_by_kind(self, since_ts: int) -> list[tuple]:
+        """(kind, requests, cost) per kind of request (chat, holding-news, ...) since a time."""
+        return self._run(
+            "SELECT kind, COUNT(*), SUM(cost) FROM usage WHERE bot_id = ? AND ts >= ? GROUP BY kind "
+            "ORDER BY SUM(cost) DESC", (self.bot_id or 0, since_ts))
+
     # -- Sharesight period cache -------------------------------------------------------
     def period_get(self, portfolio_id: int, first: str, last: str) -> dict[tuple[str, str], PeriodRow]:
         """Cached windows of a portfolio lying within first..last (ISO dates), by (start, end)."""
diff --git a/lib/telegram/commands.py b/lib/telegram/commands.py
index 7595f3d..a1b576f 100644
--- a/lib/telegram/commands.py
+++ b/lib/telegram/commands.py
@@ -42,7 +42,12 @@ async def _reply(ctx: Ctx, update: Update, text: str) -> None:
     await asyncio.to_thread(ctx.store.save, sent)
 
 
-def usage_text(rows: list[tuple], label: str) -> str:
+KIND_NAMES = {"chat": "chat", "holding-news": "holding news (daily)", "movers": "movers", "preset": "DM buttons",
+              "command": "chat commands", "filing": "filings"}  # the `kind` ask() records each request under
+
+
+def usage_text(rows: list[tuple], label: str, kinds: list[tuple] = ()) -> str:
+    """One line per model, then (with more than one kind) the requests and cost per kind."""
     if not rows:
         return f"No usage recorded in the {label}."
     lines = [f"Usage, {label}:"]
@@ -50,12 +55,14 @@ def usage_text(rows: list[tuple], label: str) -> str:
         pct = 100 * (cached or 0) / tin if tin else 0
         lines.append(f"{model}: {n} requests, in {tin or 0:,} ({pct:.0f}% cached), "
                      f"out {tout or 0:,}, ${cost or 0:.4f}")
+    if len(kinds) > 1:
+        lines.append("By kind: " + ", ".join(f"{KIND_NAMES.get(k, k)} {n} (${c or 0:.4f})" for k, n, c in kinds))
     return "\n".join(lines)
 
 
 async def credits(ctx: Ctx, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     """/tokens - admins only: the provider balance (OpenRouter has one), then tokens and cost per
-    model for the last 24 hours and 7 days."""
+    model for the last 24 hours and 7 days, and the requests and cost per kind (chat, the daily holding news...)."""
     if not await _gate(ctx, update):
         return
     try:
@@ -67,7 +74,8 @@ async def credits(ctx: Ctx, update: Update, context: ContextTypes.DEFAULT_TYPE)
     parts = [balance]
     for label, secs in (("last 24 hours", 86_400), ("last 7 days", 7 * 86_400)):
         rows = await asyncio.to_thread(ctx.store.usage_summary, now - secs)
-        parts.append(usage_text(rows, label))
+        kinds = await asyncio.to_thread(ctx.store.usage_by_kind, now - secs)
+        parts.append(usage_text(rows, label, kinds))
     await _reply(ctx, update, "\n\n".join(parts))
 
 
diff --git a/tests/test_features.py b/tests/test_features.py
index a38bc4a..f08eafa 100644
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -223,6 +223,11 @@ async def test_credits_shows_balance_and_usage_to_admins_only(env, store):
     text = owner.replies[0]["text"]
     assert "no credit balance" in text and "m: 1 requests, in 1,000 (50% cached)" in text
     assert commands.usage_text([], "last 24 hours") == "No usage recorded in the last 24 hours."
+    assert "By kind" not in text  # one kind only: nothing to split
+    store.add_usage(5, "m", "holding-news", 1, 100, 0, 10, 0.25)
+    again = user_msg(ctx.bot, "/credits", user_id=5, mid=3)
+    await commands.credits(ctx, update_for(again), None)
+    assert "By kind: chat 1 ($0.5000), holding news (daily) 1 ($0.2500)" in again.replies[0]["text"]
 
 
 async def test_credits_still_shows_usage_when_the_balance_fails(env, store):
diff --git a/tests/test_store.py b/tests/test_store.py
index e5dd645..b0e7189 100644
--- a/tests/test_store.py
+++ b/tests/test_store.py
@@ -79,6 +79,10 @@ def test_kv_users_news_and_usage(store):
     assert store.recent_news("bob", 0) == ["• A: thing happened"]
     store.add_usage(-1, "m", "chat", 2, 1000, 400, 50, 0.01)
     assert store.usage_summary(0)[0][:5] == ("m", 1, 1000, 400, 50)
+    store.add_usage(-1, "m", "holding-news", 1, 10, 0, 5, 0.25)
+    store.add_usage(-1, "m", "chat", 1, 10, 0, 5, 0.02)
+    assert store.usage_by_kind(0) == [("holding-news", 1, 0.25), ("chat", 2, 0.03)]  # dearest first
+    assert store.usage_by_kind(2**40) == []  # only the window asked for
 
 
 def test_writes_are_committed_immediately(tmp_path):

From 59a5b5d73a9b6aada56f628039a81da2beaa0f5a Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 09:08:50 +0000
Subject: [PATCH 27/30] Menu lists /usage (was /tokens); docs name /usage first

/tokens and /credits still work. Telegram shows the menu commands in autocomplete, so admins now
see /usage there.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 CLAUDE.md                | 2 +-
 README.md                | 2 +-
 docs/caching.md          | 2 +-
 docs/configuration.md    | 4 ++--
 docs/features.md         | 2 +-
 docs/how-it-works.md     | 2 +-
 lib/app.py               | 8 ++++----
 lib/config.py            | 4 ++--
 lib/telegram/commands.py | 4 ++--
 lib/telegram/handlers.py | 2 +-
 tests/test_app.py        | 4 ++--
 11 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 96085ff..9fc30a5 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -26,7 +26,7 @@ MCP data tools and web search. User-facing docs are in `README.md` (quick start)
   `openrouter.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs).
 - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun.
 - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming
-  drafts), `commands.py` (/tokens), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `known_tickers.py` (daily background refresh of `tickers.KNOWN`: S&P 500 + NASDAQ-100 + ASX 200 via Wikipedia's action API, Sharesight holdings; kept in kv), `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model).
+  drafts), `commands.py` (/usage), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `known_tickers.py` (daily background refresh of `tickers.KNOWN`: S&P 500 + NASDAQ-100 + ASX 200 via Wikipedia's action API, Sharesight holdings; kept in kv), `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model).
 - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot).
 - `docs/plans/go-recode.md`: saved plan for a Go rewrite (not started); its key requirement is a modular design with self-contained features.
 - `deploy/systemd/`: the example unit `grokbot.service` and the optional `grokbot.service.d/hardening.conf` drop-in (docs/running.md).
diff --git a/README.md b/README.md
index aa5bdd4..5f05c54 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ without asking the model. Every key is listed in [How it works](docs/how-it-work
 ## More
 - [Configuration](docs/configuration.md): providers, every environment variable, choosing a model.
 - [How it works](docs/how-it-works.md): how it answers, search, MCP data tools (`mcp_servers.json`), formatting.
-- [Features and commands](docs/features.md): charts, holding-news DM, movers replies, DM buttons, `/tokens`.
+- [Features and commands](docs/features.md): charts, holding-news DM, movers replies, DM buttons, `/usage`.
 - [Caching and token saving](docs/caching.md): the Sharesight history cache and the token-saving heuristics.
 - [Running the bot](docs/running.md): systemd, several instances, logs.
 - [Troubleshooting](docs/troubleshooting.md): startup errors, silent bots, MCP servers down.
diff --git a/docs/caching.md b/docs/caching.md
index 9b70288..31ff4ba 100644
--- a/docs/caching.md
+++ b/docs/caching.md
@@ -52,5 +52,5 @@ Each answer's log line shows the cached percentage.
   symbol, tags and Yahoo URLs dropped), lines cut to 240 characters and the bot's own to 160.
 - **Smaller prompts:** the system prompt is about half its previous size, the repeated instruction
   tail lives in the (cached) system prompt, images go at low detail.
-- **Measure it:** every request is recorded in the `usage` table (`/tokens`), and
+- **Measure it:** every request is recorded in the `usage` table (`/usage`), and
   `make prompt-report` prints where a typical request's tokens go.
diff --git a/docs/configuration.md b/docs/configuration.md
index af80cb3..566376f 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -23,8 +23,8 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o
 ### Advanced and optional
 | Variable | Default | Meaning |
 |---|---|---|
-| `ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/tokens`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. |
-| `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/tokens` and `/flushcache` stay admin-only. Who counts as an admin: `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). |
+| `ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/usage`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. |
+| `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/usage` and `/flushcache` stay admin-only. Who counts as an admin: `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). |
 | `DATA_DIR` | systemd's `STATE_DIRECTORY`, else the working directory | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. With systemd the default is its `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)). In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. |
 | `FINANCE_LINK_TARGET` | `yahoo` | Where ticker links go: `yahoo` (Yahoo Finance), `google` (Google Finance, `CODE:EXCHANGE`: `NXT.AX` opens `NXT:ASX`, `JPM` opens `JPM:NYSE`) or `tradingview` (`EXCHANGE-CODE`: `ASX-NXT`, `NYSE-BRK.A`). A US ticker's exchange comes from the text (`NYSE:JPM`, `JPM (NYSE)`), the holdings, the NASDAQ-100 or the S&P 500 list; one the bot can't place opens a Google Finance search, or TradingView's page for the bare code. |
 | `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. |
diff --git a/docs/features.md b/docs/features.md
index cc27545..598b22a 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -94,7 +94,7 @@ for its definition.
 |---|---|---|
 | `/start` | DM | Greeting (and the buttons if enabled) |
 | `/holdingnews` | DM, if holding news is on | Run the holding-news check now |
-| `/tokens` (or `/usage`, `/credits`) | an admin (see below), in a group or a DM | Provider balance (OpenRouter; xAI has none to show), then requests, tokens (and % cached) and cost per model for the last 24 h and 7 days, and the requests and cost per kind (chat, the daily holding-news check, movers, DM buttons, chat commands, filings) |
+| `/usage` (or `/tokens`, `/credits`) | an admin (see below), in a group or a DM | Provider balance (OpenRouter; xAI has none to show), then requests, tokens (and % cached) and cost per model for the last 24 h and 7 days, and the requests and cost per kind (chat, the daily holding-news check, movers, DM buttons, chat commands, filings) |
 | `/ [text]` from `chat_commands` | anyone (admins only for a `gate: portfolio` server) | Commands configured per MCP server in `mcp_servers.json`: the model answers with only that server tool, which it has to call. See [How it works](how-it-works.md#mcp-data-tools). In the command menu |
 | `/clear` | anyone, in a private chat | The bot forgets its stored history of that chat (clearing the chat on your own screen doesn't tell the bot anything, so use this too). Neither the command nor its reply is stored. Not in groups, whose history is shared by everyone and every bot in them. In the command menu of private chats |
 | `/segments TICKER` | anyone, in a group or a DM | The latest quarter's revenue breakdown pies from TickerScout `get_segments`, drawn without the model (the same as a plain "TICKER revenue breakdown" message; see Charts). In the command menu |
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index 863e4f5..57003e2 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -85,7 +85,7 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf
   "Usage: /forecast " without asking the model. A command whose server is down, or whose tool isn't
   enabled (blocked, or not read-only and missing from `allowed_tools`), says so. A command on a `gate: portfolio`
   server is for admins only. Names follow Telegram's rules (lowercase letters, digits and `_`, at most 32). A name the
-  bot already uses (`/tokens`, `/clear`, …), a name used twice, or an entry without a `tool` is skipped, and the startup
+  bot already uses (`/usage`, `/clear`, …), a name used twice, or an entry without a `tool` is skipped, and the startup
   log warns about each, and about a tool that isn't enabled.
 - **Tool names are exact:** `blocked_tools` and `allowed_tools` take the server's real tool names, as
   printed in the startup log (`get_analyst_estimates`, not `analyst_estimates`). An entry that matches
diff --git a/lib/app.py b/lib/app.py
index d205681..76c564d 100644
--- a/lib/app.py
+++ b/lib/app.py
@@ -143,20 +143,20 @@ async def log_tool_size(ctx: Ctx) -> None:
 
 async def register_menu(ctx: Ctx) -> None:
     """The command menu. A scoped menu replaces the default one, so each lists everything for its scope: /segments,
-    /narrative, /events and the chat commands of mcp_servers.json for everyone; /clear too in private chats; /tokens too for the people who can use it
+    /narrative, /events and the chat commands of mcp_servers.json for everyone; /clear too in private chats; /usage too for the people who can use it
     (administrators of every group, and the users in ADMIN_CHAT_IDS in their own chat with the bot)."""
     everyone = [BotCommand("segments", "Revenue breakdown pies: /segments TICKER"),
                 BotCommand("narrative", "10-K/10-Q narrative: /narrative TICKER [section]"),
                 BotCommand("events", "8-K events: /events TICKER [section]")]
     everyone += [BotCommand(c.name, c.description) for c in ctx.chat_commands]  # from mcp_servers.json
     private = everyone + [BotCommand("clear", "Forget our earlier messages")]
-    tokens = BotCommand("tokens", "Provider balance, token use and cost")
+    usage = BotCommand("usage", "Provider balance, token use and cost")
     try:
         await ctx.bot.set_my_commands(everyone)
         await ctx.bot.set_my_commands(private, scope=BotCommandScopeAllPrivateChats())
-        await ctx.bot.set_my_commands(everyone + [tokens], scope=BotCommandScopeAllChatAdministrators())
+        await ctx.bot.set_my_commands(everyone + [usage], scope=BotCommandScopeAllChatAdministrators())
         for chat_id in ctx.st.admin_chats:
-            await ctx.bot.set_my_commands(private + [tokens], scope=BotCommandScopeChat(chat_id))
+            await ctx.bot.set_my_commands(private + [usage], scope=BotCommandScopeChat(chat_id))
     except TelegramError as e:
         log.warning("Couldn't register the command menu: %s", e)
 
diff --git a/lib/config.py b/lib/config.py
index 0bb8ebc..c7266fd 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -23,8 +23,8 @@
     "SEARCH": "on|off. Web search (and X search on xAI). Default on.",
     "DARKMODE": "Chart theme: off (default, light), on (dark) or auto (dark from 9 pm to 7 am in TZ).",
     # Advanced and optional
-    "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /tokens; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.",
-    "ADMIN_ONLY": "on|off. On (default): only admins may DM the bot or get Sharesight data. Off: everyone may. /tokens and /flushcache stay admin-only.",
+    "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /usage; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.",
+    "ADMIN_ONLY": "on|off. On (default): only admins may DM the bot or get Sharesight data. Off: everyone may. /usage and /flushcache stay admin-only.",
     "DATA_DIR": "Directory for the bot's data (the database); Default: systemd's STATE_DIRECTORY, else the working directory.",
     "FINANCE_LINK_TARGET": "Where ticker links go: yahoo (default, Yahoo Finance), google (Google Finance) or tradingview.",
     "HISTORY_MAX": "At most this many messages of chat history in the prompt (default 10); the window's start moves every half of it, so the prompt cache keeps working.",
diff --git a/lib/telegram/commands.py b/lib/telegram/commands.py
index a1b576f..4483d53 100644
--- a/lib/telegram/commands.py
+++ b/lib/telegram/commands.py
@@ -1,4 +1,4 @@
-"""Commands: /tokens (also /usage, /credits; admins) and /narrative and /events (anyone), and the admin cache flush."""
+"""Commands: /usage (also /tokens, /credits; admins) and /narrative and /events (anyone), and the admin cache flush."""
 
 from __future__ import annotations
 
@@ -61,7 +61,7 @@ def usage_text(rows: list[tuple], label: str, kinds: list[tuple] = ()) -> str:
 
 
 async def credits(ctx: Ctx, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    """/tokens - admins only: the provider balance (OpenRouter has one), then tokens and cost per
+    """/usage (or /tokens, /credits) - admins only: the provider balance (OpenRouter has one), then tokens and cost per
     model for the last 24 hours and 7 days, and the requests and cost per kind (chat, the daily holding news...)."""
     if not await _gate(ctx, update):
         return
diff --git a/lib/telegram/handlers.py b/lib/telegram/handlers.py
index c3c340d..9d73e27 100644
--- a/lib/telegram/handlers.py
+++ b/lib/telegram/handlers.py
@@ -158,7 +158,7 @@ def _trigger(self, msg: Message) -> Trigger | None:
         return Trigger(text, private, is_movers, reply_target, replied_to_bot, nocontext)
 
     async def _command(self, msg: Message, trig: Trigger) -> bool:
-        """Private-chat commands handled here (/tokens has its own handler)."""
+        """Private-chat commands handled here (/usage has its own handler)."""
         if not trig.private or not trig.text.startswith("/"):
             return False
         command = trig.text.split()[0].split("@")[0].lower()
diff --git a/tests/test_app.py b/tests/test_app.py
index f716893..cd4ce8e 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -199,8 +199,8 @@ async def set_my_commands(commands, scope=None):
     everyone = ["segments", "narrative", "events"]
     assert calls == [(everyone, "NoneType", None),
                      (everyone + ["clear"], "BotCommandScopeAllPrivateChats", None),
-                     (everyone + ["tokens"], "BotCommandScopeAllChatAdministrators", None),
-                     (everyone + ["clear", "tokens"], "BotCommandScopeChat", 5)]
+                     (everyone + ["usage"], "BotCommandScopeAllChatAdministrators", None),
+                     (everyone + ["clear", "usage"], "BotCommandScopeChat", 5)]
 
 
 def test_the_mcp_librarys_unlabelled_protocol_and_ping_lines_are_dropped(monkeypatch):

From 8b46fb3504bebecba6ffb22e6ea91a54298cdb3a Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 09:13:37 +0000
Subject: [PATCH 28/30] Holding news: one daily search for everyone's holdings,
 split per person

Each recipient used to get a separate searched request, so a stock two people hold was searched
twice and every request paid its own overhead (holding news was the biggest cost in /usage).
The daily check now asks once for the union of everyone's unmuted holdings, with everyone's recent
items as the don't-repeat list, then gives each person the items about their own holdings; an item
naming no holding goes to everyone. /holdingnews still checks just the asker.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 docs/features.md             |  4 ++-
 lib/features/holding_news.py | 66 +++++++++++++++++++++++++++++-------
 tests/test_features.py       | 30 ++++++++++++++++
 3 files changed, 86 insertions(+), 14 deletions(-)

diff --git a/docs/features.md b/docs/features.md
index 598b22a..b3e22d9 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -7,7 +7,9 @@ Each is off by default. The holding-news DM and the movers reply are switched on
   `TZ`, `portfolio:username` pairs such as
   `MyPortfolio:alice,MySMSF:alice`; empty, the default, means off):
   reads each person's current Sharesight holdings, searches for *major* news from the past 24 hours
-  (a forced search), and DMs only what qualifies (most days: nothing). Stories already reported in
+  (a forced search), and DMs only what qualifies (most days: nothing). The daily check is one search for everyone's
+  holdings together, so a stock two people hold costs one search; each person then gets the items about their own
+  holdings (an item that names none goes to everyone). Stories already reported in
   the last 3 days are not repeated. Each message has an **Unsubscribe** button: per ticker, or all
   holding news, with an **Undo** button afterwards. People must have messaged the bot (or a group it
   is in) once so it knows their user ID. `/holdingnews` in a DM runs the check now and always replies.
diff --git a/lib/features/holding_news.py b/lib/features/holding_news.py
index 24f5d6d..fb0ddcf 100644
--- a/lib/features/holding_news.py
+++ b/lib/features/holding_news.py
@@ -79,17 +79,20 @@ def undo_button(code: str) -> InlineKeyboardMarkup:
         [[InlineKeyboardButton(f"↩️ Undo (unsubscribed from {what})", callback_data=f"hn:undo:{code}")]])
 
 
-async def holding_news_text(ctx: Ctx, username: str, holdings: dict[str, str]) -> str | None:
-    """Ask the model for major news about these holdings; None if there's nothing."""
+async def holding_news_text(ctx: Ctx, usernames: list[str], holdings: dict[str, str]) -> str | None:
+    """Ask the model for major news about these holdings (of these people); None if there's nothing."""
     since = int((datetime.now() - timedelta(days=3)).timestamp())
-    recent = await asyncio.to_thread(ctx.store.recent_news, username, since)
+    recent: dict[str, None] = {}  # every person's recent items, each once
+    for name in usernames:
+        for news in await asyncio.to_thread(ctx.store.recent_news, name, since):
+            recent.update(dict.fromkeys(line for line in news.splitlines() if line.strip()))
     already = ("\nAlready reported in the last few days; don't repeat these unless there's a "
                "genuinely new development:\n" + "\n".join(recent) + "\n") if recent else ""
     prompt = HOLDING_NEWS_PROMPT.format(
         now=ctx.now(), what=ctx.backend.search_what, already=already,
         holdings="\n".join(f"- {k}: {v}" for k, v in sorted(holdings.items())))
     answer = await ask(ctx, [{"type": "text", "text": prompt}], Route(), must_search=True,
-                       cache_key=f"holding-news-{username}", kind="holding-news")
+                       cache_key="holding-news", kind="holding-news")
     raw = strip_disclaimer(answer.text)
     if is_nothing(raw):
         return None
@@ -121,24 +124,61 @@ async def run_holding_news(ctx: Ctx, only_username: str | None = None, *, userna
         if ((only_username is None or username == only_username)
                 and (usernames is None or username in usernames)):
             people.setdefault(username, []).append(portfolio)
-    results: dict[str, tuple[str, list[str]] | None] = {}
-    for username, portfolio_names in people.items():
-        results.update(await _check_person(ctx, server, username, portfolio_names, only_username, busy))
-    return results
+    if only_username:  # /holdingnews: that person's holdings, answered by the caller
+        return {} if not people else await _check_person(ctx, server, only_username, people[only_username], True, busy)
+    return await _check_together(ctx, server, people, busy)
 
 
-async def _check_person(ctx: Ctx, server, username: str, portfolio_names: list[str],
-                        only_username: str | None, busy: list[str] | None = None) -> dict:
+async def _held(ctx: Ctx, server, username: str, portfolio_names: list[str]) -> dict[str, str] | None:
+    """A person's holdings without the ones they muted; None if they muted it all."""
     muted = await asyncio.to_thread(ctx.store.muted_codes, username)
     if "*" in muted:
         log.info("Holding news: @%s has unsubscribed from all of it", username)
+        return None
+    holdings = await current_holdings(server, portfolio_names)
+    return {k: v for k, v in holdings.items() if k.split(" ")[0].upper() not in muted}
+
+
+async def _check_together(ctx: Ctx, server, people: dict[str, list[str]], busy: list[str] | None) -> dict:
+    """One search for everyone's holdings at once (a stock two people hold is searched once), then each person gets
+    the items about their own holdings. An item naming no holding goes to everyone rather than being lost."""
+    held: dict[str, dict[str, str]] = {}
+    try:
+        for username, portfolio_names in people.items():
+            if (h := await _held(ctx, server, username, portfolio_names)):
+                held[username] = h
+        if not held:
+            return {}
+        together = {k: v for h in held.values() for k, v in h.items()}
+        log.info("Holding news: one check of %d holdings for %d people", len(together), len(held))
+        news = await holding_news_text(ctx, list(held), together)
+    except Exception as e:
+        if is_busy(e):  # automated, so no chat message; one log line is enough
+            log.warning("Holding news check failed for %s, the provider is busy: %s",
+                        ", ".join(f"@{u}" for u in people), " ".join(str(e).split()))
+            if busy is not None:
+                busy.extend(people)
+        else:
+            log.exception("Holding news check failed")
         return {}
+    items = news.splitlines() if news else []
+    results: dict[str, tuple[str, list[str]] | None] = {}
+    for username, holdings in held.items():
+        mine = "\n".join(ln for ln in items if news_codes(ln, holdings) or not news_codes(ln, together))
+        results[username] = (mine, news_codes(mine, holdings)) if mine else None
+        log.info("Holding news for @%s: %s", username, "found" if mine else "nothing major")
+        if mine:
+            await _deliver(ctx, username, mine, results[username][1])
+    return results
+
+
+async def _check_person(ctx: Ctx, server, username: str, portfolio_names: list[str],
+                        only_username: str | None, busy: list[str] | None = None) -> dict:
     try:
-        holdings = await current_holdings(server, portfolio_names)
-        holdings = {k: v for k, v in holdings.items() if k.split(" ")[0].upper() not in muted}
+        holdings = await _held(ctx, server, username, portfolio_names)
         if not holdings:
             return {}
-        news = await holding_news_text(ctx, username, holdings)
+        news = await holding_news_text(ctx, [username], holdings)
     except Exception as e:
         if is_busy(e):  # automated, so no chat message; one log line is enough
             log.warning("Holding news check failed for @%s, the provider is busy: %s",
diff --git a/tests/test_features.py b/tests/test_features.py
index f08eafa..b3047a3 100644
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -86,6 +86,36 @@ def test_known_symbols_and_listed_codes_give_yahoo_symbols():
     assert movers.listed_codes("≥ 5% at close:\nBHP +6%") == {}
 
 
+class TwoPortfolios(FakeMcp):
+    async def call(self, tool, args):
+        if tool == "list_portfolios":
+            return json.dumps({"portfolios": [{"name": "Bob", "id": 1}, {"name": "Alice", "id": 2}]})
+        rows = {1: [["NVDA", "NASDAQ", "Nvidia"], ["AMD", "NASDAQ", "AMD"]],
+                2: [["NVDA", "NASDAQ", "Nvidia"], ["BHP", "ASX", "BHP Group"]]}[args["portfolio_id"]]
+        return json.dumps({"report": {"holdings": {"columns": ["code", "market", "name"], "rows": rows}}})
+
+
+async def test_one_search_covers_everyone_and_each_person_gets_their_own_items(env, store):
+    st = config.load({**env, "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Bob:bob_llama,Alice:alice_llama"})
+    news = ("• Nvidia (NVDA): beat estimates\n• AMD fell on a guidance cut\n• BHP (ASX): trading halt\n"
+            "• Chip export rules tightened")
+    ctx = Ctx(st, store, ScriptedBackend([step(news)]), registry(TwoPortfolios("sharesight", tools=("x",))), FakeBot())
+    store.remember_user(make_msg(1, "x", username="bob_llama", chat_id=5))
+    store.remember_user(make_msg(2, "x", username="alice_llama", chat_id=6))
+    store.add_news("alice_llama", "• BHP (ASX): an older item")
+    results = await holding_news.run_holding_news(ctx)
+    assert len(ctx.backend.seen) == 1  # NVDA, which both hold, is searched once, in one request
+    prompt = ctx.backend.seen[0]["prompt"]
+    assert prompt.count("NVDA (NASDAQ)") == 1 and "AMD (NASDAQ)" in prompt and "BHP (ASX)" in prompt
+    assert "an older item" in prompt  # anyone's recent items aren't repeated
+    bob, bob_codes = results["bob_llama"]
+    alice, alice_codes = results["alice_llama"]
+    assert "NVDA" in bob and "AMD" in bob and "BHP" not in bob and bob_codes == ["AMD", "NVDA"]
+    assert "NVDA" in alice and "AMD fell" not in alice and "BHP" in alice and alice_codes == ["BHP", "NVDA"]
+    assert "export rules" in bob and "export rules" in alice  # names no holding: everyone gets it
+    assert {m["chat_id"] for m in ctx.bot.sent} == {5, 6}
+
+
 async def test_digest_says_nothing_when_nothing_and_respects_mutes(env, store):
     ctx = holding_ctx(env, store, [step("NOTHING")])
     assert (await holding_news.run_holding_news(ctx)) == {"bob_llama": None} and ctx.bot.sent == []

From 7a0962d1631e976f1bb5c7a6e75b99dc11f48e83 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 09:34:26 +0000
Subject: [PATCH 29/30] Holding news: skip days, a window back to the last
 check, no repeated articles

SHARESIGHT_HOLDING_NEWS_SKIP_DAYS (e.g. sun,mon) skips the daily check on those weekdays. Each
check now searches back to the last one that got an answer (kv; at least 24 h, at most 7 days),
so skipped days, a busy provider or an outage leave no gap; the prompt and the message header
say how far (past 72h). Repeats: the don't-repeat list always reaches past the last check, and
an item citing an article already sent is dropped whatever the model says. The feature becomes a
package (holding_news/ with text.py) to stay under the module size limit.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 CLAUDE.md                                     |   2 +-
 docs/configuration.md                         |   1 +
 docs/features.md                              |   5 +-
 lib/config.py                                 |  18 +++
 .../__init__.py}                              | 115 ++++++++----------
 lib/features/holding_news/text.py             |  76 ++++++++++++
 lib/prompts.py                                |   4 +-
 tests/test_features.py                        |  68 +++++++++++
 8 files changed, 220 insertions(+), 69 deletions(-)
 rename lib/features/{holding_news.py => holding_news/__init__.py} (80%)
 create mode 100644 lib/features/holding_news/text.py

diff --git a/CLAUDE.md b/CLAUDE.md
index 9fc30a5..2d6f592 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -26,7 +26,7 @@ MCP data tools and web search. User-facing docs are in `README.md` (quick start)
   `openrouter.py` (the only provider-specific code), `runner.py` (THE tool loop), `policy.py` (THE retry ladder), `gate.py` (which tools a message needs).
 - `prompts.py` all prompts. `ask.py` one entry point: route -> Request -> policy.ask -> usage -> NEEDS_TOOLS rerun.
 - `telegram/`: `handlers.py` (trigger -> request -> reply), `send.py`, `draft.py` (typing + streaming
-  drafts), `commands.py` (/usage), `access.py` (who is an admin). `features/`: `holding_news.py`, `movers.py`, `known_tickers.py` (daily background refresh of `tickers.KNOWN`: S&P 500 + NASDAQ-100 + ASX 200 via Wikipedia's action API, Sharesight holdings; kept in kv), `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model).
+  drafts), `commands.py` (/usage), `access.py` (who is an admin). `features/`: `holding_news/` (`__init__.py` the daily check and its schedule, `text.py` bullets, codes, repeats, the window since the last check), `movers.py`, `known_tickers.py` (daily background refresh of `tickers.KNOWN`: S&P 500 + NASDAQ-100 + ASX 200 via Wikipedia's action API, Sharesight holdings; kept in kv), `dm_buttons.py`, `chat_commands.py` (`chat_commands` in mcp_servers.json: per-server slash commands that force one tool; handler `Handlers.chat_command`, loaded onto `Ctx.chat_commands` in `app.build_ctx`), `filings.py` (/narrative and /events: TickerScout `get_narrative` / `get_events` one section at a time, one message each, edited by the buttons; /narrative always goes through the model (it calls the tool itself), /events only when a section is too long for a message; handlers in `commands.py`), `post.py`, `periods/` (local `period_returns` tool, cache-first; `windows.py`, `rows.py`, `backfill.py` = PERIOD_BACKFILL loop), `charts/` (local `make_chart` tool, offered when a message asks for a chart; series from the model or Yahoo price history; `render.py` = the facade (Chart, Series, themes, `scene()`, `png()`), laid out by `cartesian.py` / `pie.py` into a `scene.py` Scene (plain draw ops in pixels, which the tests read), ticks in `ticks.py`, painted by `raster.py` with Pillow at 2x (no numpy or matplotlib); the handler sends one reply, the photo with the answer as caption; `buttons.py` = 1D/1M/1Y/5Y/reload under a ticker chart, redrawn without the model from the `charts` table; `pages.py` = several charts in one reply as one image with a button each; `breakdown.py` = fast path: a plain "TICKER revenue breakdown" is drawn from TickerScout `get_segments` without the model).
 - `app.py` wiring/startup; `context.py` `Ctx` (settings, store, backend, registry, bot).
 - `docs/plans/go-recode.md`: saved plan for a Go rewrite (not started); its key requirement is a modular design with self-contained features.
 - `deploy/systemd/`: the example unit `grokbot.service` and the optional `grokbot.service.d/hardening.conf` drop-in (docs/running.md).
diff --git a/docs/configuration.md b/docs/configuration.md
index 566376f..ca11542 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -36,5 +36,6 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o
 | `SEARCH_MODEL` | `MODEL` with `:online` added | OpenRouter only: model that runs the searches (`:online` is what gives an OpenRouter model web search). Set but empty = no search model, so search is off. |
 | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | empty (daily DM off) | Comma list of portfolio:telegram_username pairs to notify. |
 | `SHARESIGHT_HOLDING_NEWS_TIME` | `08:00` | HH:MM (in `TZ`) for the daily holding-news check. |
+| `SHARESIGHT_HOLDING_NEWS_SKIP_DAYS` | empty | Weekdays (in `TZ`) with no holding-news check, e.g. `sun,mon` (names or their first three letters). The next check covers the gap: it always searches back to the last check (at most 7 days). Empty = every day. |
 | `TELEGRAM_DM_BUTTONS` | `off` | `on` or `off`. Preset-prompt buttons in private chats. |
 | `TELEGRAM_IMAGE_PRELOAD_CHATID` | empty (off) | Chat ID of a private chat the bot can post in (a private group it is in, or a DM with you). The other periods of a chart are uploaded there in the background and reused when a period button is tapped, so the tap swaps the picture without fetching or drawing. Empty = off, and a tap fetches and draws. See [Charts](features.md#charts). |
diff --git a/docs/features.md b/docs/features.md
index b3e22d9..63b8d88 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -9,7 +9,10 @@ Each is off by default. The holding-news DM and the movers reply are switched on
   reads each person's current Sharesight holdings, searches for *major* news from the past 24 hours
   (a forced search), and DMs only what qualifies (most days: nothing). The daily check is one search for everyone's
   holdings together, so a stock two people hold costs one search; each person then gets the items about their own
-  holdings (an item that names none goes to everyone). Stories already reported in
+  holdings (an item that names none goes to everyone). Each check searches back to the last one (at least 24 hours,
+  at most 7 days), so days skipped with `SHARESIGHT_HOLDING_NEWS_SKIP_DAYS` (say `sun,mon`, after the US weekend), an
+  outage or a restart leave no gap; the message says how far back it looked (`past 72h`). An item that cites the same
+  article as one already sent is dropped, even if the model repeats it. Stories already reported in
   the last 3 days are not repeated. Each message has an **Unsubscribe** button: per ticker, or all
   holding news, with an **Undo** button afterwards. People must have messaged the bot (or a group it
   is in) once so it knows their user ID. `/holdingnews` in a DM runs the check now and always replies.
diff --git a/lib/config.py b/lib/config.py
index c7266fd..ca0bb37 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -36,6 +36,7 @@
     "SEARCH_MODEL": "OpenRouter only: model that runs the searches (default: MODEL with :online added).",
     "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.",
     "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (TZ) for the daily holding-news check (default 08:00).",
+    "SHARESIGHT_HOLDING_NEWS_SKIP_DAYS": "Weekdays (in TZ) with no holding-news check, e.g. sun,mon; the next check covers them. Empty (default) = every day.",
     "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).",
     "TELEGRAM_IMAGE_PRELOAD_CHATID": "Chat ID of a private chat the bot can post in. Chart periods are uploaded there ahead of time and reused when a period button is tapped. Empty (default) = off.",
 }
@@ -69,6 +70,7 @@
 KNOWN_TICKERS_START_DELAY = 60  # seconds after startup before a due refresh, so the MCP servers are up
 SOURCES_MAX = 3  # links to a search result the model is handed back as text (OpenRouter run_search)
 MAX_TG_MESSAGE = 4000  # Telegram's limit is 4096
+HOLDING_NEWS_MAX_WINDOW = 7 * 24  # hours: the longest a check looks back, after skipped days or an outage
 HOLDING_NEWS_RETRY_EVERY = 30 * 60  # seconds between retries of a holding-news check the provider was too busy for
 HOLDING_NEWS_RETRY_FOR = 12 * 3600  # seconds after which it gives up
 
@@ -99,6 +101,7 @@ class Settings:
     post_to_groups: bool
     preload_chat: int | None  # where chart periods are pre-uploaded
     holding_news_time: str
+    holding_news_skip: frozenset[int]  # weekdays with no check, Monday = 0
     holding_news_recipients: dict[str, str]
     portfolio_names: frozenset[str]
     period_backfill: frozenset[str]
@@ -154,6 +157,20 @@ def _pairs(text: str) -> dict[str, str]:
     }
 
 
+WEEKDAYS = ("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday")
+
+
+def _weekdays(env: Mapping[str, str], name: str) -> frozenset[int]:
+    """"sun, Monday" -> {6, 0}: day names or their first three letters."""
+    days = set()
+    for word in env.get(name, "").replace(",", " ").lower().split():
+        day = next((i for i, d in enumerate(WEEKDAYS) if word in (d, d[:3])), None)
+        if day is None:
+            raise ConfigError(f"{name}: {word!r} isn't a weekday (use mon, tue, ... or the full name).")
+        days.add(day)
+    return frozenset(days)
+
+
 def _names(text: str) -> frozenset[str]:
     return frozenset(n.strip().lower() for n in re.split(r"[,\n]", text) if n.strip())
 
@@ -218,6 +235,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
         post_to_groups=_flag(env, "POST_TO_GROUPS_FROM_DM"),
         preload_chat=_chat_id(env, "TELEGRAM_IMAGE_PRELOAD_CHATID"),
         holding_news_time=env.get("SHARESIGHT_HOLDING_NEWS_TIME", "08:00"),
+        holding_news_skip=_weekdays(env, "SHARESIGHT_HOLDING_NEWS_SKIP_DAYS"),
         holding_news_recipients=recipients,
         portfolio_names=_names(env.get("PORTFOLIO_NAMES", "")) or frozenset(recipients),
         period_backfill=_backfill(env.get("PERIOD_BACKFILL", "all")),
diff --git a/lib/features/holding_news.py b/lib/features/holding_news/__init__.py
similarity index 80%
rename from lib/features/holding_news.py
rename to lib/features/holding_news/__init__.py
index fb0ddcf..b8dc7a6 100644
--- a/lib/features/holding_news.py
+++ b/lib/features/holding_news/__init__.py
@@ -4,9 +4,8 @@
 from __future__ import annotations
 
 import asyncio
-import html
 import logging
-import re
+import time
 from datetime import datetime, timedelta
 from itertools import batched
 
@@ -14,58 +13,32 @@
 from telegram.error import Forbidden
 from telegram.ext import ContextTypes
 
-from .. import config
-from ..ask import ask
-from ..context import Ctx
-from ..holdings import code_and_market, current_holdings
-from ..llm.base import BUSY_REPLY, is_busy
-from ..llm.gate import Route
-from ..prompts import HOLDING_NEWS_PROMPT
-from ..quote_links import quote_labels
-from ..telegram.draft import Draft
-from ..telegram.send import send_html
-from ..textfmt import ANY_TAG_RE, md_to_html, split_html, strip_disclaimer
-from ..tickers import link_tickers, yahoo_symbol
+from ... import config
+from ...ask import ask
+from ...context import Ctx
+from ...holdings import current_holdings
+from ...llm.base import BUSY_REPLY, is_busy
+from ...llm.gate import Route
+from ...prompts import HOLDING_NEWS_PROMPT
+from ...telegram.draft import Draft
+from ...telegram.send import send_html
+from ...textfmt import md_to_html, split_html, strip_disclaimer
+from ...tickers import link_tickers
+from .text import (
+    WINDOW_DEFAULT,
+    as_bullets,
+    drop_repeats,
+    is_nothing,
+    known_symbols,
+    news_codes,
+    window_hours,
+)
 
 log = logging.getLogger("bot")
+LAST_RUN_KEY = "holding_news_last_run"  # kv: when the daily check last got an answer (its next window starts there)
 UNSUBSCRIBE_BUTTON = InlineKeyboardMarkup([[InlineKeyboardButton("🔕 Unsubscribe…", callback_data="hn:menu")]])
 
 
-def is_nothing(raw: str) -> bool:
-    """True for the model's "nothing to report" answer, however it's dressed up ("NOTHING",
-    "• NOTHING", "**Nothing.**", "None") or an empty reply."""
-    plain = html.unescape(ANY_TAG_RE.sub("", raw or ""))
-    words = re.sub(r"[^A-Za-z ]", " ", plain).split()
-    # A real item is longer, even one about a company called "Nothing ...".
-    return not words or (words[0].upper() in ("NOTHING", "NONE") and len(words) <= 5)
-
-
-def as_bullets(text: str) -> str:
-    """One "• " bullet per non-empty line, whatever bullet style (if any) the model used."""
-    lines = (re.sub(r"^\s*(?:[•\-*–·]|\d+[.)])\s+", "", ln).strip() for ln in text.splitlines())
-    return "\n".join(f"• {ln}" for ln in lines if ln)
-
-
-def known_symbols(holdings: dict[str, str]) -> dict[str, str]:
-    """The holdings' codes as Yahoo symbols ({"NXT": "NXT.AX"}), so the digest links them however the model writes them."""
-    pairs = (code_and_market(key) for key in holdings)
-    return {code: yahoo_symbol(code, market) or code for code, market in pairs}
-
-
-def news_codes(news: str, holdings: dict[str, str]) -> list[str]:
-    """Holding codes that appear as bold tickers or quote links in a news message (as the holdings write them: an HK
-    holding "700" is linked as 0700.HK but labelled 700)."""
-    held = {_plain_code(k.split(" ")[0]): k.split(" ")[0].upper() for k in holdings}
-    found = {m.upper() for m in re.findall(r"([^<]{1,12})", news)}
-    found |= {label.upper() for label in quote_labels(news)}
-    return sorted({held[c] for c in map(_plain_code, found) if c in held})
-
-
-def _plain_code(code: str) -> str:
-    code = code.upper()
-    return code.lstrip("0") or "0" if code.isdigit() else code
-
-
 def unsubscribe_menu(codes: list[str]) -> InlineKeyboardMarkup:
     rows = [[InlineKeyboardButton(c, callback_data=f"hn:mute:{c}") for c in row] for row in batched(codes, 3, strict=False)]
     rows.append([InlineKeyboardButton("All holding news", callback_data="hn:mute:*")])
@@ -79,29 +52,33 @@ def undo_button(code: str) -> InlineKeyboardMarkup:
         [[InlineKeyboardButton(f"↩️ Undo (unsubscribed from {what})", callback_data=f"hn:undo:{code}")]])
 
 
-async def holding_news_text(ctx: Ctx, usernames: list[str], holdings: dict[str, str]) -> str | None:
-    """Ask the model for major news about these holdings (of these people); None if there's nothing."""
-    since = int((datetime.now() - timedelta(days=3)).timestamp())
-    recent: dict[str, None] = {}  # every person's recent items, each once
-    for name in usernames:
-        for news in await asyncio.to_thread(ctx.store.recent_news, name, since):
-            recent.update(dict.fromkeys(line for line in news.splitlines() if line.strip()))
+async def holding_news_text(ctx: Ctx, usernames: list[str], holdings: dict[str, str],
+                            hours: int = WINDOW_DEFAULT) -> str | None:
+    """Ask the model for major news about these holdings (of these people) in the past `hours`; None if there's
+    nothing. Items already sent recently are listed in the prompt as not to repeat, and an item that cites the same
+    articles as one already sent is dropped whatever the model says."""
+    since = int(time.time()) - max(3 * 24, hours + 24) * 3600  # always reaching past the last check
+    digests = [news for name in usernames for news in await asyncio.to_thread(ctx.store.recent_news, name, since)]
+    recent = dict.fromkeys(line for news in digests for line in news.splitlines() if line.strip())
     already = ("\nAlready reported in the last few days; don't repeat these unless there's a "
                "genuinely new development:\n" + "\n".join(recent) + "\n") if recent else ""
     prompt = HOLDING_NEWS_PROMPT.format(
-        now=ctx.now(), what=ctx.backend.search_what, already=already,
+        now=ctx.now(), what=ctx.backend.search_what, already=already, hours=hours,
         holdings="\n".join(f"- {k}: {v}" for k, v in sorted(holdings.items())))
     answer = await ask(ctx, [{"type": "text", "text": prompt}], Route(), must_search=True,
                        cache_key="holding-news", kind="holding-news")
     raw = strip_disclaimer(answer.text)
     if is_nothing(raw):
         return None
-    return link_tickers(as_bullets(md_to_html(raw)), known=known_symbols(holdings))
+    news, dropped = drop_repeats(link_tickers(as_bullets(md_to_html(raw)), known=known_symbols(holdings)), digests)
+    for item in dropped:
+        log.debug("Holding news: dropped a repeat of an article already sent: %s", item[:120])
+    return news or None
 
 
-async def send_holding_news(ctx: Ctx, chat_id: int, news: str, codes: list[str]) -> None:
+async def send_holding_news(ctx: Ctx, chat_id: int, news: str, codes: list[str], hours: int = WINDOW_DEFAULT) -> None:
     """Send a holding-news message with its Unsubscribe button on the last part."""
-    chunks = split_html(f"📰 Holding news (past 24h)\n\n{news}")
+    chunks = split_html(f"📰 Holding news (past {hours}h)\n\n{news}")
     for i, chunk in enumerate(chunks):
         last = i == len(chunks) - 1
         sent = await send_html(ctx.bot, chat_id, chunk, UNSUBSCRIBE_BUTTON if last else None)
@@ -150,8 +127,11 @@ async def _check_together(ctx: Ctx, server, people: dict[str, list[str]], busy:
         if not held:
             return {}
         together = {k: v for h in held.values() for k, v in h.items()}
-        log.info("Holding news: one check of %d holdings for %d people", len(together), len(held))
-        news = await holding_news_text(ctx, list(held), together)
+        last = await asyncio.to_thread(ctx.store.kv_get, LAST_RUN_KEY)
+        hours = window_hours(time.time(), float(last) if last else None)
+        log.info("Holding news: one check of %d holdings for %d people, past %d hours", len(together), len(held), hours)
+        news = await holding_news_text(ctx, list(held), together, hours)
+        await asyncio.to_thread(ctx.store.kv_set, LAST_RUN_KEY, str(int(time.time())))  # it got an answer
     except Exception as e:
         if is_busy(e):  # automated, so no chat message; one log line is enough
             log.warning("Holding news check failed for %s, the provider is busy: %s",
@@ -168,7 +148,7 @@ async def _check_together(ctx: Ctx, server, people: dict[str, list[str]], busy:
         results[username] = (mine, news_codes(mine, holdings)) if mine else None
         log.info("Holding news for @%s: %s", username, "found" if mine else "nothing major")
         if mine:
-            await _deliver(ctx, username, mine, results[username][1])
+            await _deliver(ctx, username, mine, results[username][1], hours)
     return results
 
 
@@ -195,14 +175,14 @@ async def _check_person(ctx: Ctx, server, username: str, portfolio_names: list[s
     return {username: result}
 
 
-async def _deliver(ctx: Ctx, username: str, news: str, codes: list[str]) -> None:
+async def _deliver(ctx: Ctx, username: str, news: str, codes: list[str], hours: int = WINDOW_DEFAULT) -> None:
     chat_id = await asyncio.to_thread(ctx.store.user_id_for, username)
     if not chat_id:
         log.warning("Holding news: don't know @%s's user ID yet; they need to message the bot "
                     "or a group it's in", username)
         return
     try:
-        await send_holding_news(ctx, chat_id, news, codes)
+        await send_holding_news(ctx, chat_id, news, codes, hours)
         await asyncio.to_thread(ctx.store.add_news, username, news)
     except Forbidden:
         log.warning("Holding news: @%s hasn't started a private chat with the bot", username)
@@ -221,6 +201,11 @@ async def daily_loop(ctx: Ctx) -> None:
         wait = seconds_until(ctx.st.holding_news_time, datetime.now(ctx.st.tz))
         log.info("Next holding news check in %.1f hours", wait / 3600)
         await asyncio.sleep(wait)
+        today = datetime.now(ctx.st.tz)
+        if today.weekday() in ctx.st.holding_news_skip:  # the next check covers it: its window starts at the last
+            log.info("Holding news: skipped (%s)", today.strftime("%A"))
+            await asyncio.sleep(1)
+            continue
         busy: list[str] = []
         try:
             await run_holding_news(ctx, busy=busy)
diff --git a/lib/features/holding_news/text.py b/lib/features/holding_news/text.py
new file mode 100644
index 0000000..f76bc0c
--- /dev/null
+++ b/lib/features/holding_news/text.py
@@ -0,0 +1,76 @@
+"""Holding news text: the model's answer as bullets, the codes each names, repeats dropped, the search window."""
+
+from __future__ import annotations
+
+import html
+import math
+import re
+
+from ... import config
+from ...holdings import code_and_market
+from ...quote_links import QUOTE_PREFIXES, quote_labels
+from ...textfmt import ANY_TAG_RE
+from ...tickers import yahoo_symbol
+
+WINDOW_DEFAULT = 24  # hours a check covers: one day, or since the last run when that was longer ago
+HREF_RE = re.compile(r' bool:
+    """True for the model's "nothing to report" answer, however it's dressed up ("NOTHING",
+    "• NOTHING", "**Nothing.**", "None") or an empty reply."""
+    plain = html.unescape(ANY_TAG_RE.sub("", raw or ""))
+    words = re.sub(r"[^A-Za-z ]", " ", plain).split()
+    # A real item is longer, even one about a company called "Nothing ...".
+    return not words or (words[0].upper() in ("NOTHING", "NONE") and len(words) <= 5)
+
+
+def as_bullets(text: str) -> str:
+    """One "• " bullet per non-empty line, whatever bullet style (if any) the model used."""
+    lines = (re.sub(r"^\s*(?:[•\-*–·]|\d+[.)])\s+", "", ln).strip() for ln in text.splitlines())
+    return "\n".join(f"• {ln}" for ln in lines if ln)
+
+
+def known_symbols(holdings: dict[str, str]) -> dict[str, str]:
+    """The holdings' codes as Yahoo symbols ({"NXT": "NXT.AX"}), so the digest links them however the model writes them."""
+    pairs = (code_and_market(key) for key in holdings)
+    return {code: yahoo_symbol(code, market) or code for code, market in pairs}
+
+
+def news_codes(news: str, holdings: dict[str, str]) -> list[str]:
+    """Holding codes that appear as bold tickers or quote links in a news message (as the holdings write them: an HK
+    holding "700" is linked as 0700.HK but labelled 700)."""
+    held = {_plain_code(k.split(" ")[0]): k.split(" ")[0].upper() for k in holdings}
+    found = {m.upper() for m in re.findall(r"([^<]{1,12})", news)}
+    found |= {label.upper() for label in quote_labels(news)}
+    return sorted({held[c] for c in map(_plain_code, found) if c in held})
+
+
+def _plain_code(code: str) -> str:
+    code = code.upper()
+    return code.lstrip("0") or "0" if code.isdigit() else code
+
+
+def window_hours(now: float, last_run: float | None) -> int:
+    """Hours the daily check covers: since the last one (after skipped days, an outage or a restart), whole hours,
+    at least a day and at most HOLDING_NEWS_MAX_WINDOW; a day when there was no last run."""
+    if not last_run:
+        return WINDOW_DEFAULT
+    hours = math.ceil((now - last_run) / 3600)
+    return max(WINDOW_DEFAULT, min(hours, config.HOLDING_NEWS_MAX_WINDOW))
+
+
+def source_links(news: str) -> set[str]:
+    """The source links of a digest (not the quote links on tickers)."""
+    return {html.unescape(u) for u in HREF_RE.findall(news) if not u.startswith(QUOTE_PREFIXES)}
+
+
+def drop_repeats(news: str, recent: list[str]) -> tuple[str, list[str]]:
+    """The digest without the items whose sources were all in a recent digest (the same articles again), and the
+    items dropped. An item with no source link, or a new one, stays."""
+    seen = set().union(*(source_links(r) for r in recent)) if recent else set()
+    kept, dropped = [], []
+    for line in news.splitlines():
+        links = source_links(line)
+        (dropped if links and links <= seen else kept).append(line)
+    return "\n".join(kept), dropped
diff --git a/lib/prompts.py b/lib/prompts.py
index 2197c5b..ccab6b6 100644
--- a/lib/prompts.py
+++ b/lib/prompts.py
@@ -125,8 +125,8 @@ def chat_prompt(*, transcript: str, sender: str, private: bool, reply_quote: str
 Holdings:
 {holdings}
 
-Search {what} for MAJOR news about these companies published in the past 24 hours.
-The bar is high. Only material, price-moving events count: results or earnings, guidance changes, takeover or merger news, capital raisings, major contract wins or losses, regulatory or clinical-trial decisions, trading halts or suspensions, CEO or CFO departures, dividend cuts or suspensions, delistings, serious legal action. Ignore routine filings (director share dealings, buy-back updates, AGM notices, presentations with nothing new), general market or sector moves, analyst price-target changes, opinion pieces, and anything older than 24 hours or that you can't date. When in doubt, leave it out. Most days the answer is nothing.
+Search {what} for MAJOR news about these companies published in the past {hours} hours.
+The bar is high. Only material, price-moving events count: results or earnings, guidance changes, takeover or merger news, capital raisings, major contract wins or losses, regulatory or clinical-trial decisions, trading halts or suspensions, CEO or CFO departures, dividend cuts or suspensions, delistings, serious legal action. Ignore routine filings (director share dealings, buy-back updates, AGM notices, presentations with nothing new), general market or sector moves, analyst price-target changes, opinion pieces, and anything older than {hours} hours or that you can't date. When in doubt, leave it out. Most days the answer is nothing.
 
 If nothing qualifies, reply with exactly the word NOTHING, with no bullet or anything else. Otherwise reply with only the qualifying items as bullet points, one item per line, no intro and no blank lines between them. Start each line with "• ":
 • Name (TICKER): what happened, with one source link.
diff --git a/tests/test_features.py b/tests/test_features.py
index b3047a3..f547b5f 100644
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -140,6 +140,74 @@ async def test_busy_provider_is_retried_for_just_that_person_then_delivered(env,
     assert not any(r.exc_info for r in caplog.records)
 
 
+def test_the_window_reaches_back_to_the_last_check():
+    from lib.features.holding_news.text import window_hours
+    now = 1_000_000_000
+    assert window_hours(now, None) == 24 and window_hours(now, now - 2 * 3600) == 24  # a day at least
+    assert window_hours(now, now - 24 * 3600) == 24 and window_hours(now, now - 72 * 3600 + 60) == 72
+    assert window_hours(now, now - 30 * 86400) == config.HOLDING_NEWS_MAX_WINDOW == 168
+
+
+def test_an_item_citing_an_article_already_sent_is_dropped():
+    from lib.features.holding_news.text import drop_repeats
+    sent = ['• NVDA: beat [1]']
+    new = ('• NVDA: beat again [1]\n'
+           '• NVDA: guidance raised [2]\n'
+           '• AMD fell')
+    kept, dropped = drop_repeats(new, sent)
+    assert "x.com/a" not in kept and "x.com/b" in kept and "AMD fell" in kept and len(dropped) == 1
+    assert drop_repeats(new, []) == (new, [])  # the quote link on NVDA doesn't count as an article
+
+
+def test_skip_days_parse_names_and_abbreviations(env):
+    assert config.load({**env, "SHARESIGHT_HOLDING_NEWS_SKIP_DAYS": "sun, Monday"}).holding_news_skip == {6, 0}
+    assert config.load(env).holding_news_skip == frozenset()
+    with pytest.raises(config.ConfigError, match="funday"):
+        config.load({**env, "SHARESIGHT_HOLDING_NEWS_SKIP_DAYS": "funday"})
+
+
+async def test_after_skipped_days_the_check_covers_them_and_a_busy_one_keeps_the_window(env, store):
+    import time
+    ctx = holding_ctx(env, store, [Overloaded("at capacity"), step("• NVDA (NASDAQ): halted https://x.com/h")])
+    store.kv_set(holding_news.LAST_RUN_KEY, str(int(time.time()) - 72 * 3600 + 60))  # Saturday's, now it's Tuesday
+    await holding_news.run_holding_news(ctx, busy=[])
+    assert int(store.kv_get(holding_news.LAST_RUN_KEY)) < time.time() - 71 * 3600  # busy: the window stays open
+    await holding_news.run_holding_news(ctx)
+    assert "past 72 hours" in ctx.backend.seen[-1]["prompt"] and "(past 72h)" in ctx.bot.sent[-1]["text"]
+    assert int(store.kv_get(holding_news.LAST_RUN_KEY)) >= time.time() - 5  # answered: the next window starts now
+
+
+async def test_the_daily_loop_skips_its_days(env, store, monkeypatch, caplog):
+    import asyncio
+    import logging
+    st = config.load({**env, "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Bob:bob_llama",
+                      "SHARESIGHT_HOLDING_NEWS_SKIP_DAYS": "sun"})
+    ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot())
+    sunday = datetime(2026, 11, 8, 8, 0, tzinfo=st.tz)
+    ran, sleeps = [], []
+
+    class Clock(datetime):
+        @classmethod
+        def now(cls, tz=None):
+            return sunday
+
+    async def sleep(seconds):
+        sleeps.append(seconds)
+        if len(sleeps) > 2:
+            raise asyncio.CancelledError
+
+    async def run(ctx, **kw):
+        ran.append(kw)
+        return {}
+
+    monkeypatch.setattr(holding_news, "datetime", Clock)
+    monkeypatch.setattr(holding_news.asyncio, "sleep", sleep)
+    monkeypatch.setattr(holding_news, "run_holding_news", run)
+    with caplog.at_level(logging.INFO, logger="bot"), pytest.raises(asyncio.CancelledError):
+        await holding_news.daily_loop(ctx)
+    assert ran == [] and "Holding news: skipped (Sunday)" in caplog.text
+
+
 async def test_busy_provider_is_given_up_on_after_the_retry_window(env, store, monkeypatch, caplog):
     monkeypatch.setattr(config, "HOLDING_NEWS_RETRY_EVERY", 4 * 3600)
     monkeypatch.setattr(config, "HOLDING_NEWS_RETRY_FOR", 12 * 3600)

From 86954d0515bc17526ad942483643de7181a60893 Mon Sep 17 00:00:00 2001
From: Claude 
Date: Thu, 5 Nov 2026 09:40:08 +0000
Subject: [PATCH 30/30] Rename ADMIN_CHAT_IDS to TELEGRAM_ADMIN_CHAT_IDS

The old name is no longer read: rename it in the service's environment.

Co-Authored-By: Claude Opus 5.5 
Claude-Session: https://claude.ai/code/session_014k4kywyN5dAo39izUPvxbw
---
 docs/configuration.md       | 4 ++--
 docs/features.md            | 4 ++--
 docs/how-it-works.md        | 2 +-
 docs/plans/go-recode.md     | 2 +-
 lib/app.py                  | 4 ++--
 lib/config.py               | 4 ++--
 lib/telegram/access.py      | 4 ++--
 tests/test_access.py        | 2 +-
 tests/test_app.py           | 6 +++---
 tests/test_chat_commands.py | 4 ++--
 tests/test_config.py        | 2 +-
 tests/test_features.py      | 6 +++---
 tests/test_handlers.py      | 6 +++---
 13 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index ca11542..48abf35 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -23,8 +23,7 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o
 ### Advanced and optional
 | Variable | Default | Meaning |
 |---|---|---|
-| `ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/usage`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. |
-| `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/usage` and `/flushcache` stay admin-only. Who counts as an admin: `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). |
+| `ADMIN_ONLY` | `on` | `on`: only admins can DM the bot or get Sharesight (portfolio) data; anyone else asking about a portfolio is told it is for admins only. `off`: everyone can do both. Either way anyone can @mention the bot or reply to it in a group, and `/usage` and `/flushcache` stay admin-only. Who counts as an admin: `TELEGRAM_ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (see [Features](features.md#commands)). |
 | `DATA_DIR` | systemd's `STATE_DIRECTORY`, else the working directory | Directory for the bot's data (the SQLite database `chat_log.db`). Bots started with the same `DATA_DIR` share group history. With systemd the default is its `STATE_DIRECTORY` (see [Running under systemd](running.md#running-under-systemd)). In Docker, set `DATA_DIR=/data` and bind-mount a host directory there so history survives upgrades. |
 | `FINANCE_LINK_TARGET` | `yahoo` | Where ticker links go: `yahoo` (Yahoo Finance), `google` (Google Finance, `CODE:EXCHANGE`: `NXT.AX` opens `NXT:ASX`, `JPM` opens `JPM:NYSE`) or `tradingview` (`EXCHANGE-CODE`: `ASX-NXT`, `NYSE-BRK.A`). A US ticker's exchange comes from the text (`NYSE:JPM`, `JPM (NYSE)`), the holdings, the NASDAQ-100 or the S&P 500 list; one the bot can't place opens a Google Finance search, or TradingView's page for the bare code. |
 | `HISTORY_MAX` | `10` | The most messages of chat history in the prompt. Once a chat is longer, the window holds half to all of that and its start only moves every half of it, so the prompt cache keeps working. |
@@ -37,5 +36,6 @@ Set `TELEGRAM_BOT_TOKEN` and exactly one of the three LLM keys. With more than o
 | `SHARESIGHT_HOLDING_NEWS_RECIPIENTS` | empty (daily DM off) | Comma list of portfolio:telegram_username pairs to notify. |
 | `SHARESIGHT_HOLDING_NEWS_TIME` | `08:00` | HH:MM (in `TZ`) for the daily holding-news check. |
 | `SHARESIGHT_HOLDING_NEWS_SKIP_DAYS` | empty | Weekdays (in `TZ`) with no holding-news check, e.g. `sun,mon` (names or their first three letters). The next check covers the gap: it always searches back to the last check (at most 7 days). Empty = every day. |
+| `TELEGRAM_ADMIN_CHAT_IDS` | empty | Telegram IDs of the bot's admins (comma/space separated). Your user ID gets a message when an MCP server goes down and may use `/usage`; a group's chat ID gets the down-server messages only. Admins of the groups the bot is in count as admins too. |
 | `TELEGRAM_DM_BUTTONS` | `off` | `on` or `off`. Preset-prompt buttons in private chats. |
 | `TELEGRAM_IMAGE_PRELOAD_CHATID` | empty (off) | Chat ID of a private chat the bot can post in (a private group it is in, or a DM with you). The other periods of a chart are uploaded there in the background and reused when a period button is tapped, so the tap swaps the picture without fetching or drawing. Empty = off, and a tap fetches and draws. See [Charts](features.md#charts). |
diff --git a/docs/features.md b/docs/features.md
index 63b8d88..5eff733 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -107,10 +107,10 @@ for its definition.
 | `/events TICKER [section]` | anyone, in a group or a DM | TickerScout's 8-K event digest from `get_events`. Default `earnings`; sections: earnings, management changes, capital returns, debt and financing, m&a, governance, restructuring, legal. A section that fits one message is shown as TickerScout wrote it (kept 24 hours too), with no model, except that its dated items are put in ascending order (oldest first; a text whose items aren't all dated is left as it is); a longer one is shortened by the model as for `/narrative`. Buttons as for `/narrative`. In the command menu |
 | `/flushcache [portfolio_id] [year or month or day]` | an admin | Forget cached Sharesight period returns: all, or of one portfolio, or those overlapping a year (`2024`), month (`2024-05`) or day (`2024-05-17`), e.g. `/flushcache 684141 2024`; see [Caching Sharesight history](caching.md#caching-sharesight-history) |
 
-**Admins** are the user IDs in `ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (checked with
+**Admins** are the user IDs in `TELEGRAM_ADMIN_CHAT_IDS` plus anyone who administers a group the bot is in (checked with
 Telegram, cached for five minutes). The bot only knows a group once someone has written in it, or it was added, since
 it started tracking groups. Admins who post anonymously (as the group) can't be recognised. With `ADMIN_ONLY` on (the
 default), only admins can DM the bot, and Sharesight data is for admins only: anyone else asking about a portfolio
 gets "Portfolio data is for admins only." Holding-news DMs, buttons and down-server messages the bot sends itself are
-unaffected. With nobody in `ADMIN_CHAT_IDS` and no known group, nobody can DM the bot or reach Sharesight: set
+unaffected. With nobody in `TELEGRAM_ADMIN_CHAT_IDS` and no known group, nobody can DM the bot or reach Sharesight: set
 `ADMIN_ONLY=off` to allow everyone.
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index 57003e2..bea6ae0 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -96,7 +96,7 @@ saving tokens every round; a required parameter is never hidden); `gate` (`portf
 - **Everyone who can reach the bot can use every enabled tool**, including Sharesight portfolio
   data (there is no per-user tool restriction; Telegram/BotFather
   controls who may use the bot). Restrict the bot in BotFather, or leave Sharesight out.
-- **Down servers:** if a server dies, chats in `ADMIN_CHAT_IDS` get a message with the error, and the
+- **Down servers:** if a server dies, chats in `TELEGRAM_ADMIN_CHAT_IDS` get a message with the error, and the
   model is told the source is down (and quotes the error) instead of claiming it has no access. A
   dead server is not restarted until the bot restarts.
 - **Slimming:** results are compacted before the model sees them (see [token saving](caching.md#caching-and-token-saving)).
diff --git a/docs/plans/go-recode.md b/docs/plans/go-recode.md
index 5a2a77c..9ef5470 100644
--- a/docs/plans/go-recode.md
+++ b/docs/plans/go-recode.md
@@ -322,7 +322,7 @@ A. **Carve the seams in Python first (recommended; settle the open question).**
    - A second BotFather bot in the same groups.
    - `grokbot-shadow.service` is the same unit plus `User=grokbot` (same dynamic UID, so it shares
      `/var/lib/grokbot/chat_log.db`), `EnvironmentFile=/etc/grokbot-shadow.env` and `ExecStart=/opt/grokbot/grokbot shadow`.
-   - The shadow env turns off holding news, movers, buttons, posting and backfill, and sets `ADMIN_CHAT_IDS` to the maintainer.
+   - The shadow env turns off holding news, movers, buttons, posting and backfill, and sets `TELEGRAM_ADMIN_CHAT_IDS` to the maintainer.
    - What to compare:
      - ask both bots the same things;
      - diff the summary lines;
diff --git a/lib/app.py b/lib/app.py
index 76c564d..ee5bb42 100644
--- a/lib/app.py
+++ b/lib/app.py
@@ -119,7 +119,7 @@ def _drop_protocol_line(record: logging.LogRecord) -> bool:
 
 
 async def alert_down(ctx: Ctx, server: MCPServer) -> None:
-    """Tell ADMIN_CHAT_IDS that an MCP server died (and log it, so the model sees it in history)."""
+    """Tell TELEGRAM_ADMIN_CHAT_IDS that an MCP server died (and log it, so the model sees it in history)."""
     if not ctx.bot:
         return
     text = (f"⚠️ The {html.escape(server.label)} data source is down. "
@@ -144,7 +144,7 @@ async def log_tool_size(ctx: Ctx) -> None:
 async def register_menu(ctx: Ctx) -> None:
     """The command menu. A scoped menu replaces the default one, so each lists everything for its scope: /segments,
     /narrative, /events and the chat commands of mcp_servers.json for everyone; /clear too in private chats; /usage too for the people who can use it
-    (administrators of every group, and the users in ADMIN_CHAT_IDS in their own chat with the bot)."""
+    (administrators of every group, and the users in TELEGRAM_ADMIN_CHAT_IDS in their own chat with the bot)."""
     everyone = [BotCommand("segments", "Revenue breakdown pies: /segments TICKER"),
                 BotCommand("narrative", "10-K/10-Q narrative: /narrative TICKER [section]"),
                 BotCommand("events", "8-K events: /events TICKER [section]")]
diff --git a/lib/config.py b/lib/config.py
index ca0bb37..4229037 100644
--- a/lib/config.py
+++ b/lib/config.py
@@ -23,7 +23,6 @@
     "SEARCH": "on|off. Web search (and X search on xAI). Default on.",
     "DARKMODE": "Chart theme: off (default, light), on (dark) or auto (dark from 9 pm to 7 am in TZ).",
     # Advanced and optional
-    "ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /usage; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.",
     "ADMIN_ONLY": "on|off. On (default): only admins may DM the bot or get Sharesight data. Off: everyone may. /usage and /flushcache stay admin-only.",
     "DATA_DIR": "Directory for the bot's data (the database); Default: systemd's STATE_DIRECTORY, else the working directory.",
     "FINANCE_LINK_TARGET": "Where ticker links go: yahoo (default, Yahoo Finance), google (Google Finance) or tradingview.",
@@ -37,6 +36,7 @@
     "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Comma list of portfolio:telegram_username pairs to notify. Empty (default) = daily holding-news DM off.",
     "SHARESIGHT_HOLDING_NEWS_TIME": "HH:MM (TZ) for the daily holding-news check (default 08:00).",
     "SHARESIGHT_HOLDING_NEWS_SKIP_DAYS": "Weekdays (in TZ) with no holding-news check, e.g. sun,mon; the next check covers them. Empty (default) = every day.",
+    "TELEGRAM_ADMIN_CHAT_IDS": "Telegram IDs of the bot's admins (comma/space separated). Your user ID gets MCP-down alerts and may use /usage; a group's chat ID gets the alerts only. Admins of the groups the bot is in count as admins too.",
     "TELEGRAM_DM_BUTTONS": "on|off. Preset-prompt buttons in private chats (default off).",
     "TELEGRAM_IMAGE_PRELOAD_CHATID": "Chat ID of a private chat the bot can post in. Chart periods are uploaded there ahead of time and reused when a period button is tapped. Empty (default) = off.",
 }
@@ -226,7 +226,7 @@ def load(env: Mapping[str, str] | None = None) -> Settings:
         link_target=link_target,
         db_path=database_path(env),
         tz=ZoneInfo(env.get("TZ", "UTC")),
-        admin_chats=_ids(env.get("ADMIN_CHAT_IDS", "")),
+        admin_chats=_ids(env.get("TELEGRAM_ADMIN_CHAT_IDS", "")),
         admin_only=_flag(env, "ADMIN_ONLY", default=True),
         movers_bots=frozenset(
             u.lower().lstrip("@") for u in env.get("MOVERS_BOTS", "").replace(",", " ").split()
diff --git a/lib/telegram/access.py b/lib/telegram/access.py
index 30ab598..93826c4 100644
--- a/lib/telegram/access.py
+++ b/lib/telegram/access.py
@@ -1,4 +1,4 @@
-"""Who counts as an admin: a user in ADMIN_CHAT_IDS, or an admin of a group the bot has seen."""
+"""Who counts as an admin: a user in TELEGRAM_ADMIN_CHAT_IDS, or an admin of a group the bot has seen."""
 
 from __future__ import annotations
 
@@ -28,7 +28,7 @@ async def _group_admins(ctx: Ctx, chat_id: int) -> frozenset[int]:
 
 
 async def is_admin(ctx: Ctx, user_id: int) -> bool:
-    """True for a user in ADMIN_CHAT_IDS or an administrator of any group the bot knows. Anonymous
+    """True for a user in TELEGRAM_ADMIN_CHAT_IDS or an administrator of any group the bot knows. Anonymous
     admins post as the group itself, so they can't be recognised."""
     if user_id in ctx.st.admin_chats:
         return True
diff --git a/tests/test_access.py b/tests/test_access.py
index 99b11b2..f35a22f 100644
--- a/tests/test_access.py
+++ b/tests/test_access.py
@@ -15,7 +15,7 @@ def make_ctx(env, store, **extra):
 
 
 async def test_listed_user_is_admin_without_any_lookup(env, store):
-    ctx = make_ctx(env, store, ADMIN_CHAT_IDS="5, -100")
+    ctx = make_ctx(env, store, TELEGRAM_ADMIN_CHAT_IDS="5, -100")
     assert await access.is_admin(ctx, 5) and not await access.is_admin(ctx, 6)
 
 
diff --git a/tests/test_app.py b/tests/test_app.py
index cd4ce8e..7f1084d 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -35,7 +35,7 @@ def build(**extra):
     assert handler_kinds(plain) == ["CommandHandler"] * 6 + ["MessageHandler",
                                     "CallbackQueryHandler", "CallbackQueryHandler", "CallbackQueryHandler"]  # chart periods, charts, sections
     assert app.allowed_updates(st) == ["message", "edited_message", "callback_query"]  # chart buttons
-    full, st = build(ADMIN_CHAT_IDS="5", SHARESIGHT_HOLDING_NEWS_RECIPIENTS="Pf:alice")
+    full, st = build(TELEGRAM_ADMIN_CHAT_IDS="5", SHARESIGHT_HOLDING_NEWS_RECIPIENTS="Pf:alice")
     kinds = handler_kinds(full)
     assert kinds == ["CommandHandler"] * 6 + ["MessageHandler"] + ["CallbackQueryHandler"] * 4
     assert "callback_query" in app.allowed_updates(st)
@@ -128,7 +128,7 @@ def test_git_hash_follows_a_linked_worktree(tmp_path):
 
 
 async def test_alert_down_notifies_admin_chats_only(env, store):
-    st = config.load({**env, "ADMIN_CHAT_IDS": "-10 -11"})
+    st = config.load({**env, "TELEGRAM_ADMIN_CHAT_IDS": "-10 -11"})
     ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot())
     server = MCPServer("yahoo", {}, timeout=1, max_output=10)
     server.error = "boom "
@@ -193,7 +193,7 @@ async def test_the_menu_lists_commands_by_who_can_use_them(env, store):
     async def set_my_commands(commands, scope=None):
         calls.append(([c.command for c in commands], type(scope).__name__, getattr(scope, "chat_id", None)))
 
-    ctx = Ctx(config.load({**env, "ADMIN_CHAT_IDS": "5"}), store, ScriptedBackend([]), registry(), FakeBot())
+    ctx = Ctx(config.load({**env, "TELEGRAM_ADMIN_CHAT_IDS": "5"}), store, ScriptedBackend([]), registry(), FakeBot())
     ctx.bot = SimpleNamespace(set_my_commands=set_my_commands)
     await app.register_menu(ctx)
     everyone = ["segments", "narrative", "events"]
diff --git a/tests/test_chat_commands.py b/tests/test_chat_commands.py
index 6f14da2..f24a4b6 100644
--- a/tests/test_chat_commands.py
+++ b/tests/test_chat_commands.py
@@ -95,14 +95,14 @@ async def test_a_command_that_cannot_run_says_why_without_the_model(env, store,
 async def test_a_portfolio_command_is_for_admins_only(env, store):
     holdings = ChatCommand("holdings", "sharesight", "list_portfolios")
     server = FakeMcp("sharesight", tools=("list_portfolios",), gate="portfolio")
-    ctx, backend, h = chat(env, store, [], server, holdings, ADMIN_ONLY="on", ADMIN_CHAT_IDS="99")
+    ctx, backend, h = chat(env, store, [], server, holdings, ADMIN_ONLY="on", TELEGRAM_ADMIN_CHAT_IDS="99")
     m = user_msg(ctx.bot, "/holdings", chat_id=-100, user_id=5)
     await run_command(h, holdings, m)
     assert backend.seen == [] and m.replies[0]["text"] == "Portfolio data is for admins only."
 
 
 async def test_a_non_admin_dm_is_ignored_in_admin_only_mode(env, store):
-    ctx, backend, h = chat(env, store, [], weather(), ALERTS, ADMIN_ONLY="on", ADMIN_CHAT_IDS="99")
+    ctx, backend, h = chat(env, store, [], weather(), ALERTS, ADMIN_ONLY="on", TELEGRAM_ADMIN_CHAT_IDS="99")
     m = user_msg(ctx.bot, "/alerts", chat_id=5, user_id=5)
     await run_command(h, ALERTS, m)
     assert backend.seen == [] and m.replies == []
diff --git a/tests/test_config.py b/tests/test_config.py
index 95bcc1e..562593b 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -79,7 +79,7 @@ def test_search_off_without_search_model(env):
 
 
 def test_admin_chats_and_recipients(env):
-    s = config.load({**env, "ADMIN_CHAT_IDS": "-1, -2 3", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Bob:@Me"})
+    s = config.load({**env, "TELEGRAM_ADMIN_CHAT_IDS": "-1, -2 3", "SHARESIGHT_HOLDING_NEWS_RECIPIENTS": "Bob:@Me"})
     assert s.admin_chats == {-1, -2, 3} and s.holding_news_recipients == {"bob": "me"}
 
 
diff --git a/tests/test_features.py b/tests/test_features.py
index f547b5f..d27fe6e 100644
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -310,7 +310,7 @@ def test_movers_helpers():
 
 # ---- commands --------------------------------------------------------------------------
 async def test_credits_shows_balance_and_usage_to_admins_only(env, store):
-    st = config.load({**env, "ADMIN_CHAT_IDS": "5, -100"})
+    st = config.load({**env, "TELEGRAM_ADMIN_CHAT_IDS": "5, -100"})
     ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot())
     store.add_usage(-1, "m", "chat", 1, 1000, 500, 20, 0.5)
     stranger = user_msg(ctx.bot, "/credits", user_id=9, mid=1)
@@ -333,7 +333,7 @@ class Broken(ScriptedBackend):
         async def credits(self):
             raise RuntimeError("boom")
 
-    ctx = Ctx(config.load({**env, "ADMIN_CHAT_IDS": "5"}), store, Broken([]), registry(), FakeBot())
+    ctx = Ctx(config.load({**env, "TELEGRAM_ADMIN_CHAT_IDS": "5"}), store, Broken([]), registry(), FakeBot())
     store.add_usage(-1, "m", "chat", 1, 10, 0, 2, 0.1)
     msg = user_msg(ctx.bot, "/credits", user_id=5)
     await commands.credits(ctx, update_for(msg), None)
@@ -362,7 +362,7 @@ async def test_flushcache_command_is_admin_only_and_flushes_by_portfolio_and_per
     from lib.features.periods import backfill
     from lib.store import PeriodRow
 
-    st = config.load({**env, "ADMIN_CHAT_IDS": "5", "PERIOD_BACKFILL": "x"})
+    st = config.load({**env, "TELEGRAM_ADMIN_CHAT_IDS": "5", "PERIOD_BACKFILL": "x"})
     ctx = Ctx(st, store, ScriptedBackend([]), registry(), FakeBot())
     store.period_put([PeriodRow(p, f"{y}-0{m}-01", f"{y}-0{m}-28", 1.0, 1.0, "AUD", [])
                       for p in (70007, 80008) for y in (2024, 2025) for m in (1, 2)])
diff --git a/tests/test_handlers.py b/tests/test_handlers.py
index c8e7ad1..9ac02a9 100644
--- a/tests/test_handlers.py
+++ b/tests/test_handlers.py
@@ -274,7 +274,7 @@ def test_dm_buttons_hash_changes_with_presets(monkeypatch):
 
 # ---- admin-only (the default) ---------------------------------------------------------
 async def test_dm_from_a_non_admin_is_logged_but_ignored(env, store):
-    ctx, backend, h = make_ctx(env, store, [step("hi")], ADMIN_ONLY="on", ADMIN_CHAT_IDS="5")
+    ctx, backend, h = make_ctx(env, store, [step("hi")], ADMIN_ONLY="on", TELEGRAM_ADMIN_CHAT_IDS="5")
     stranger = user_msg(ctx.bot, "hello", chat_id=9, user_id=9)
     await run(h, stranger)
     assert stranger.replies == [] and backend.seen == [] and store.history(9, 20)
@@ -299,7 +299,7 @@ async def test_group_admin_can_dm_but_admin_only_off_lets_everyone_in(env, store
 async def test_non_admin_gets_no_portfolio_data_in_a_group(env, store):
     sharesight = FakeMcp("sharesight", gate="portfolio", tools=("list_portfolios",))
     ctx, backend, h = make_ctx(env, store, [step("ok")], servers=[sharesight], ADMIN_ONLY="on",
-                               ADMIN_CHAT_IDS="5")
+                               TELEGRAM_ADMIN_CHAT_IDS="5")
     asked = user_msg(ctx.bot, "@stockbot how is my portfolio doing?", user_id=9)
     await run(h, asked)
     assert asked.replies[0]["text"] == "Portfolio data is for admins only." and backend.seen == []
@@ -459,7 +459,7 @@ async def test_clear_only_touches_this_chat_and_this_bot(env, store):
 
 
 async def test_typing_starts_before_the_bot_logs_the_message_or_checks_who_sent_it(env, store, monkeypatch):
-    ctx, backend, h = make_ctx(env, store, [step("hi")], ADMIN_ONLY="on", ADMIN_CHAT_IDS="5")
+    ctx, backend, h = make_ctx(env, store, [step("hi")], ADMIN_ONLY="on", TELEGRAM_ADMIN_CHAT_IDS="5")
     seen = {}
 
     async def is_admin(msg):