diff --git a/docs/hackbot/actions.md b/docs/hackbot/actions.md index c324b461c6a..1d4753ba2ff 100644 --- a/docs/hackbot/actions.md +++ b/docs/hackbot/actions.md @@ -65,6 +65,29 @@ be turned back on for a recorded message by passing `unfurl=True` to `record_mes `bugzilla.add_comment` appends a feedback-reaction footer to every recorded comment, and `is_private=true` marks it security-group-only. +When hackbot-api posts a Bugzilla comment, it puts a front matter block at the top that +links the comment to its run: + +```text +--- +hackbot: + run: + action: + agent: + url: https://hackbot.moz.tools/runs/ +--- +``` + +BMO's Hackbot extension hides the block from readers, bugmail, webhooks and Pulse, and shows +the agent and a link to the run in the comment header instead. REST `raw_text` keeps it. +The block is added at apply time, not recorded, so a model can't write or change it and the +Hackbot UI and Slack never show it. BMO only reads `run`, `action` and `agent` when they are +identifiers, which run IDs, row IDs and registered agent names always are. When field +updates are merged with a comment, `action` is the ID of the row the comment came from. +Agent text posted without the block (attachment comments and new-bug descriptions) is moved +down a line when it starts with a rule, so BMO doesn't read a block the model wrote as the +comment's metadata. + Adding a type is a declaration in the runtime's domain module plus one line in hackbot-api's handler registry. The dispatch loop never changes. diff --git a/services/hackbot-api/app/action_handlers/base.py b/services/hackbot-api/app/action_handlers/base.py index b407925d994..00687fe04eb 100644 --- a/services/hackbot-api/app/action_handlers/base.py +++ b/services/hackbot-api/app/action_handlers/base.py @@ -34,9 +34,9 @@ def __call__(self, key: str, max_bytes: int | None = None) -> Awaitable[bytes]: class ApplyContext: """Everything an :class:`ActionHandler` needs from the run. - Scoped to a single action application (not the whole run), since only - ``attachments`` varies per action. Handlers never talk to GCS directly — - ``download_artifact`` is provided by the caller (hackbot-api's + Scoped to a single action application (not the whole run), since + ``attachments`` and ``action_id`` vary per action. Handlers never talk to GCS + directly — ``download_artifact`` is provided by the caller (hackbot-api's ``/internal/events/apply-run-actions`` route) — so this package stays free of a dependency on any particular storage backend. Async, matching hackbot-api's own GCS wrappers and ``ActionHandler.apply`` itself. @@ -47,6 +47,9 @@ class ApplyContext: download_artifact: ArtifactDownloader hackbot_client: HackbotClient attachments: list[dict[str, str]] = field(default_factory=list) + # The `RunAction` row ID of the action. When several actions are applied as + # one call, it is the one whose comment is posted. + action_id: int | None = None def artifact_key(self, name: str) -> str | None: """The uploaded key for an attachment recorded under ``name``, if any.""" diff --git a/services/hackbot-api/app/action_handlers/bugzilla_handler.py b/services/hackbot-api/app/action_handlers/bugzilla_handler.py index ff4d78d0fe0..e3c411550db 100644 --- a/services/hackbot-api/app/action_handlers/bugzilla_handler.py +++ b/services/hackbot-api/app/action_handlers/bugzilla_handler.py @@ -57,6 +57,54 @@ def _request(method: str, path: str, json_body: dict[str, Any]) -> dict[str, Any return response.json() +def _front_matter(ctx: ApplyContext) -> str: + """The block that links a comment to the run and action that wrote it. + + BMO's Hackbot extension reads it off the top of comments from the Hackbot + account, hides it, and shows the agent and a link to the run in the comment + header instead. BMO only accepts identifiers for `run`, `action` and `agent`, + which a run ID, a `RunAction` row ID and a registered agent name always are + (`test_agent_names_are_identifiers` keeps it that way). + + Added here rather than by the agent so a model can't write or change it. + Empty when no action is known, such as field updates merged without a + comment. + """ + if ctx.action_id is None: + return "" + url = f"{settings.hackbot_ui_url.rstrip('/')}/runs/{ctx.run_id}" + return "\n".join( + [ + "---", + "hackbot:", + f" run: {ctx.run_id}", + f" action: {ctx.action_id}", + f" agent: {ctx.agent}", + f" url: {url}", + "---", + "", + "", + ] + ) + + +def _escape_front_matter(text: str) -> str: + """Agent text that BMO can't take for front matter. + + BMO reads a block only at the very start of a comment, so text that starts + with a rule is moved down a line: a block the model wrote is then shown as + text rather than taken as the comment's metadata. + """ + return "\n" + text if text.startswith("---") else text + + +def _with_front_matter(comment: dict[str, Any], ctx: ApplyContext) -> dict[str, Any]: + """The comment with its front matter, or with none that BMO would read.""" + front_matter = _front_matter(ctx) + body = comment["body"] if front_matter else _escape_front_matter(comment["body"]) + return {**comment, "body": front_matter + body} + + def _comment_body(params: dict[str, Any]) -> dict[str, Any]: """Build the ``comment`` object for a PUT /bug/{id}. @@ -83,6 +131,10 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult if params.get("comment"): body["comment"] = params["comment"] body["comment_tags"] = [ctx.agent] + # Also a comment the agent put in `changes` itself, which is posted from + # the Hackbot account too. + if body.get("comment"): + body["comment"] = _with_front_matter(body["comment"], ctx) try: _request("PUT", f"bug/{bug_id}", body) except Exception as exc: @@ -94,7 +146,8 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult class AddCommentHandler: async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult: bug_id = params["bug_id"] - body: dict[str, Any] = {"comment": _comment_body(params)} + comment = _with_front_matter(_comment_body(params), ctx) + body: dict[str, Any] = {"comment": comment} body["comment_tags"] = [ctx.agent] try: _request("PUT", f"bug/{bug_id}", body) @@ -126,7 +179,7 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult "is_patch": params.get("is_patch", False), } if params.get("comment"): - body["comment"] = params["comment"] + body["comment"] = _escape_front_matter(params["comment"]) try: data = _request("POST", f"bug/{bug_id}/attachment", body) @@ -146,6 +199,12 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult class CreateBugHandler: async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult: + # The description is comment 0, which gets no front matter. + if isinstance(params.get("description"), str): + params = { + **params, + "description": _escape_front_matter(params["description"]), + } try: data = _request("POST", "bug", params) except Exception as exc: diff --git a/services/hackbot-api/app/actions_applier.py b/services/hackbot-api/app/actions_applier.py index f016f510863..5a29a19232e 100644 --- a/services/hackbot-api/app/actions_applier.py +++ b/services/hackbot-api/app/actions_applier.py @@ -164,6 +164,7 @@ async def _dispatch( params: dict, attachments: list[dict], hackbot_client: HackbotClient, + action_id: int, ) -> ActionResult: """Run one handler call, converting failures into a failed `ActionResult`. @@ -184,6 +185,7 @@ async def _dispatch( attachments=attachments, agent=run.agent, hackbot_client=hackbot_client, + action_id=action_id, ) try: return await handler.apply(params, ctx) @@ -303,18 +305,34 @@ async def _apply_pending_rows( (member.type, resolve_placeholders(member.params, results_by_ref)) for member in member_rows ] + # The front matter names the row whose comment is posted: the + # group's `add_comment`, or else the last update with a comment in + # its `changes` (later changes win). With no comment, nothing reads + # the ID, so the anchor's own will do. + comment_action_id = next( + (m.id for m in member_rows if m.type == "bugzilla.add_comment"), + next( + ( + m.id + for m in reversed(member_rows) + if "comment" in (m.params.get("changes") or {}) + ), + row.id, + ), + ) outcome = await _dispatch( run, "bugzilla.update_bug", merge_resolved(entries), [], hackbot_client, + comment_action_id, ) else: member_rows = [row] params = resolve_placeholders(row.params, results_by_ref) outcome = await _dispatch( - run, row.type, params, attachments, hackbot_client + run, row.type, params, attachments, hackbot_client, row.id ) # Only stamp applied_at on a real success, so a failed row isn't diff --git a/services/hackbot-api/tests/test_actions_applier.py b/services/hackbot-api/tests/test_actions_applier.py index c3fd1976af2..62efef5d964 100644 --- a/services/hackbot-api/tests/test_actions_applier.py +++ b/services/hackbot-api/tests/test_actions_applier.py @@ -670,6 +670,105 @@ async def test_extra_comments_applied_separately(monkeypatch): ] +class _ContextRecordingHandler: + def __init__(self): + self.action_ids = [] + + async def apply(self, params, ctx): + self.action_ids.append(ctx.action_id) + return SimpleNamespace(status="applied", result={}, error=None) + + +async def test_handlers_get_the_action_id_of_the_comment_they_post(monkeypatch): + """A coalesced PUT carries its comment's ID, which goes in the front matter.""" + handler = _ContextRecordingHandler() + monkeypatch.setattr(actions_applier, "get_handler", lambda t: handler) + + update = _row( + 0, + "pending", + action_type="bugzilla.update_bug", + params={"bug_id": 5, "changes": {"status": "RESOLVED"}}, + ) + comment = _row( + 1, + "pending", + action_type="bugzilla.add_comment", + params={"bug_id": 5, "text": "done"}, + ) + other = _row( + 2, + "pending", + action_type="bugzilla.add_comment", + params={"bug_id": 99, "text": "different bug"}, + ) + rows = [(update, []), (comment, []), (other, [])] + + await actions_applier._apply_pending_rows( + _FakeDB(), _FakeRun(status=RunStatus.succeeded.value), rows, _CLIENT + ) + + # Row IDs: the update is 1, its comment 2, the other bug's comment 3. + assert handler.action_ids == [2, 3] + + +async def test_merged_updates_carry_the_id_of_the_comment_in_their_changes( + monkeypatch, +): + """Without an add_comment, the ID is the update whose comment is posted.""" + handler = _ContextRecordingHandler() + monkeypatch.setattr(actions_applier, "get_handler", lambda t: handler) + + rows = [ + ( + _row( + idx, + "pending", + action_type="bugzilla.update_bug", + params={"bug_id": 5, "changes": changes}, + ), + [], + ) + for idx, changes in enumerate( + [ + {"status": "RESOLVED"}, + {"comment": {"body": "fixed"}}, + {"resolution": "FIXED"}, + ] + ) + ] + rows.append( + ( + _row( + 3, + "pending", + action_type="bugzilla.update_bug", + params={"bug_id": 6, "changes": {"a": 1}}, + ), + [], + ) + ) + rows.append( + ( + _row( + 4, + "pending", + action_type="bugzilla.update_bug", + params={"bug_id": 6, "changes": {"b": 2}}, + ), + [], + ) + ) + + await actions_applier._apply_pending_rows( + _FakeDB(), _FakeRun(status=RunStatus.succeeded.value), rows, _CLIENT + ) + + # Bug 5's comment came from the second update (row 2). Bug 6 posts no + # comment and gets its last update's ID (row 5), which nothing reads. + assert handler.action_ids == [2, 5] + + async def test_lone_same_type_actions_on_different_bugs_not_merged(monkeypatch): handler = _RecordingHandler( SimpleNamespace(status="applied", result={}, error=None) diff --git a/services/hackbot-api/tests/test_agents.py b/services/hackbot-api/tests/test_agents.py index 542914659a6..c9e72e5942d 100644 --- a/services/hackbot-api/tests/test_agents.py +++ b/services/hackbot-api/tests/test_agents.py @@ -1,6 +1,7 @@ """Tests for the agent registry and generic env serialization.""" import json +import re import pytest from app.agents import AGENT_REGISTRY, model_to_env @@ -17,6 +18,12 @@ from pydantic import ValidationError +def test_agent_names_are_identifiers(): + """BMO reads the agent name from a comment's front matter only if it is one.""" + for name in AGENT_REGISTRY: + assert re.fullmatch(r"[A-Za-z0-9._-]+", name), name + + def test_model_to_env_uppercases_and_stringifies(): env = model_to_env(BugFixInputs(bug_id=12345, model="claude-opus", max_turns=8)) assert env["BUG_ID"] == "12345" diff --git a/services/hackbot-api/tests/test_bugzilla_handler.py b/services/hackbot-api/tests/test_bugzilla_handler.py index 1b7d855d5af..78a14edfc09 100644 --- a/services/hackbot-api/tests/test_bugzilla_handler.py +++ b/services/hackbot-api/tests/test_bugzilla_handler.py @@ -8,9 +8,10 @@ import base64 from app.action_handlers import ApplyContext, bugzilla_handler +from app.config import settings -def _ctx(attachments=None, artifacts=None, agent="test-agent"): +def _ctx(attachments=None, artifacts=None, agent="test-agent", action_id=None): artifacts = artifacts or {} async def download(key): @@ -22,6 +23,7 @@ async def download(key): attachments=attachments or [], agent=agent, hackbot_client=object(), + action_id=action_id, ) @@ -99,6 +101,125 @@ async def test_comment_handlers_add_acting_agent_tag(monkeypatch): ] +# What BMO's Hackbot extension expects at the top of the comment: identifiers +# for run, action and agent, an https url, then a blank line before the text. +_FRONT_MATTER = ( + "---\n" + "hackbot:\n" + " run: run-1\n" + " action: 42\n" + " agent: frontend-triage\n" + " url: https://hackbot.example/runs/run-1\n" + "---\n" + "\n" +) + + +async def test_comment_handlers_prepend_front_matter(monkeypatch): + monkeypatch.setattr(settings, "hackbot_ui_url", "https://hackbot.example/") + bodies = [] + monkeypatch.setattr( + bugzilla_handler, "_request", lambda _m, _p, body: bodies.append(body) + ) + ctx = _ctx(agent="frontend-triage", action_id=42) + + await bugzilla_handler.AddCommentHandler().apply( + {"bug_id": 5, "text": "standalone"}, ctx + ) + await bugzilla_handler.UpdateBugHandler().apply( + {"bug_id": 5, "changes": {}, "comment": {"body": "coalesced"}}, ctx + ) + + assert [body["comment"]["body"] for body in bodies] == [ + _FRONT_MATTER + "standalone", + _FRONT_MATTER + "coalesced", + ] + + +async def test_update_bug_handler_without_comment_has_no_front_matter(monkeypatch): + bodies = [] + monkeypatch.setattr( + bugzilla_handler, "_request", lambda _m, _p, body: bodies.append(body) + ) + await bugzilla_handler.UpdateBugHandler().apply( + {"bug_id": 5, "changes": {"status": "RESOLVED"}}, + _ctx(action_id=42), + ) + assert bodies == [{"status": "RESOLVED"}] + + +async def test_comment_without_action_id_has_no_front_matter(monkeypatch): + """Without an action to link to, the text is posted as it is.""" + bodies = [] + monkeypatch.setattr( + bugzilla_handler, "_request", lambda _m, _p, body: bodies.append(body) + ) + await bugzilla_handler.AddCommentHandler().apply( + {"bug_id": 5, "text": "hi"}, _ctx(action_id=None) + ) + assert bodies[0]["comment"]["body"] == "hi" + + +async def test_agent_block_is_not_read_as_front_matter(monkeypatch): + """Without our block, a block the model wrote must not reach the top.""" + bodies = [] + monkeypatch.setattr( + bugzilla_handler, "_request", lambda _m, _p, body: bodies.append(body) + ) + forged = "---\nhackbot:\n run: x\n action: y\n agent: z\n---\n\nhi" + await bugzilla_handler.AddCommentHandler().apply( + {"bug_id": 5, "text": forged}, _ctx(action_id=None) + ) + await bugzilla_handler.AddCommentHandler().apply( + {"bug_id": 5, "text": forged}, _ctx(action_id=42) + ) + assert bodies[0]["comment"]["body"] == "\n" + forged + # With our block first, BMO reads ours and shows the model's as text. + assert bodies[1]["comment"]["body"].endswith("---\n\n" + forged) + assert bodies[1]["comment"]["body"].startswith("---\nhackbot:\n run: run-1\n") + + +async def test_other_agent_text_is_not_read_as_front_matter(monkeypatch): + """A block the model wrote can't reach the top of any comment we post.""" + calls = [] + monkeypatch.setattr( + bugzilla_handler, + "_request", + lambda _m, path, body: calls.append((path, body)) or {"ids": [1], "id": 1}, + ) + forged = "---\nhackbot:\n run: x\n action: y\n agent: z\n---\n\nhi" + ctx = _ctx( + attachments=[{"name": "file", "uploaded_key": "k"}], + artifacts={"k": b"data"}, + action_id=42, + ) + + await bugzilla_handler.UpdateBugHandler().apply( + {"bug_id": 5, "changes": {"comment": {"body": forged}}}, ctx + ) + await bugzilla_handler.AddAttachmentHandler().apply( + { + "bug_id": 5, + "file_name": "a.txt", + "summary": "a", + "content_type": "text/plain", + "comment": forged, + }, + ctx, + ) + await bugzilla_handler.CreateBugHandler().apply( + {"product": "P", "summary": "S", "description": forged}, ctx + ) + + update, attachment, new_bug = (body for _path, body in calls) + # A comment in `changes` gets our block first, like any other comment. + assert update["comment"]["body"].startswith("---\nhackbot:\n run: run-1\n") + assert update["comment"]["body"].endswith("\n\n" + forged) + # Attachment comments and descriptions get no block, so they are moved down. + assert attachment["comment"] == "\n" + forged + assert new_bug["description"] == "\n" + forged + + async def test_add_attachment_handler_downloads_and_base64_encodes(monkeypatch): seen = {} monkeypatch.setattr(