Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions docs/hackbot/actions.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,29 @@ be turned back on for a recorded message by passing `unfurl=True` to `record_mes
`bugzilla.add_comment` appends a feedback-reaction footer to every recorded comment, and
`is_private=true` marks it security-group-only.

When hackbot-api posts a Bugzilla comment, it puts a front matter block at the top that
links the comment to its run:

```text
---
hackbot:
run: <run ID>
action: <RunAction row ID>
agent: <agent name>
url: https://hackbot.moz.tools/runs/<run ID>
---
```

BMO's Hackbot extension hides the block from readers, bugmail, webhooks and Pulse, and shows
the agent and a link to the run in the comment header instead. REST `raw_text` keeps it.
The block is added at apply time, not recorded, so a model can't write or change it and the
Hackbot UI and Slack never show it. BMO only reads `run`, `action` and `agent` when they are
identifiers, which run IDs, row IDs and registered agent names always are. When field
updates are merged with a comment, `action` is the ID of the row the comment came from.
Agent text posted without the block (attachment comments and new-bug descriptions) is moved
down a line when it starts with a rule, so BMO doesn't read a block the model wrote as the
comment's metadata.

Adding a type is a declaration in the runtime's domain module plus one line in hackbot-api's
handler registry. The dispatch loop never changes.

Expand Down
9 changes: 6 additions & 3 deletions services/hackbot-api/app/action_handlers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ def __call__(self, key: str, max_bytes: int | None = None) -> Awaitable[bytes]:
class ApplyContext:
"""Everything an :class:`ActionHandler` needs from the run.

Scoped to a single action application (not the whole run), since only
``attachments`` varies per action. Handlers never talk to GCS directly —
``download_artifact`` is provided by the caller (hackbot-api's
Scoped to a single action application (not the whole run), since
``attachments`` and ``action_id`` vary per action. Handlers never talk to GCS
directly — ``download_artifact`` is provided by the caller (hackbot-api's
``/internal/events/apply-run-actions`` route) — so this package stays free of
a dependency on any particular storage backend. Async, matching
hackbot-api's own GCS wrappers and ``ActionHandler.apply`` itself.
Expand All @@ -47,6 +47,9 @@ class ApplyContext:
download_artifact: ArtifactDownloader
hackbot_client: HackbotClient
attachments: list[dict[str, str]] = field(default_factory=list)
# The `RunAction` row ID of the action. When several actions are applied as
# one call, it is the one whose comment is posted.
action_id: int | None = None

def artifact_key(self, name: str) -> str | None:
"""The uploaded key for an attachment recorded under ``name``, if any."""
Expand Down
63 changes: 61 additions & 2 deletions services/hackbot-api/app/action_handlers/bugzilla_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,54 @@ def _request(method: str, path: str, json_body: dict[str, Any]) -> dict[str, Any
return response.json()


def _front_matter(ctx: ApplyContext) -> str:
"""The block that links a comment to the run and action that wrote it.

BMO's Hackbot extension reads it off the top of comments from the Hackbot
account, hides it, and shows the agent and a link to the run in the comment
header instead. BMO only accepts identifiers for `run`, `action` and `agent`,
which a run ID, a `RunAction` row ID and a registered agent name always are
(`test_agent_names_are_identifiers` keeps it that way).

Added here rather than by the agent so a model can't write or change it.
Empty when no action is known, such as field updates merged without a
comment.
"""
if ctx.action_id is None:
return ""
url = f"{settings.hackbot_ui_url.rstrip('/')}/runs/{ctx.run_id}"
return "\n".join(
[
"---",
"hackbot:",
f" run: {ctx.run_id}",
f" action: {ctx.action_id}",
f" agent: {ctx.agent}",
f" url: {url}",
"---",
"",
"",
]
)


def _escape_front_matter(text: str) -> str:
"""Agent text that BMO can't take for front matter.

BMO reads a block only at the very start of a comment, so text that starts
with a rule is moved down a line: a block the model wrote is then shown as
text rather than taken as the comment's metadata.
"""
return "\n" + text if text.startswith("---") else text


def _with_front_matter(comment: dict[str, Any], ctx: ApplyContext) -> dict[str, Any]:
"""The comment with its front matter, or with none that BMO would read."""
front_matter = _front_matter(ctx)
body = comment["body"] if front_matter else _escape_front_matter(comment["body"])
return {**comment, "body": front_matter + body}


def _comment_body(params: dict[str, Any]) -> dict[str, Any]:
"""Build the ``comment`` object for a PUT /bug/{id}.

Expand All @@ -83,6 +131,10 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult
if params.get("comment"):
body["comment"] = params["comment"]
body["comment_tags"] = [ctx.agent]
# Also a comment the agent put in `changes` itself, which is posted from
# the Hackbot account too.
if body.get("comment"):
body["comment"] = _with_front_matter(body["comment"], ctx)
try:
_request("PUT", f"bug/{bug_id}", body)
except Exception as exc:
Expand All @@ -94,7 +146,8 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult
class AddCommentHandler:
async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult:
bug_id = params["bug_id"]
body: dict[str, Any] = {"comment": _comment_body(params)}
comment = _with_front_matter(_comment_body(params), ctx)
body: dict[str, Any] = {"comment": comment}
body["comment_tags"] = [ctx.agent]
try:
_request("PUT", f"bug/{bug_id}", body)
Expand Down Expand Up @@ -126,7 +179,7 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult
"is_patch": params.get("is_patch", False),
}
if params.get("comment"):
body["comment"] = params["comment"]
body["comment"] = _escape_front_matter(params["comment"])

try:
data = _request("POST", f"bug/{bug_id}/attachment", body)
Expand All @@ -146,6 +199,12 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult

class CreateBugHandler:
async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult:
# The description is comment 0, which gets no front matter.
if isinstance(params.get("description"), str):
params = {
**params,
"description": _escape_front_matter(params["description"]),
}
try:
data = _request("POST", "bug", params)
except Exception as exc:
Expand Down
20 changes: 19 additions & 1 deletion services/hackbot-api/app/actions_applier.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ async def _dispatch(
params: dict,
attachments: list[dict],
hackbot_client: HackbotClient,
action_id: int,
) -> ActionResult:
"""Run one handler call, converting failures into a failed `ActionResult`.

Expand All @@ -184,6 +185,7 @@ async def _dispatch(
attachments=attachments,
agent=run.agent,
hackbot_client=hackbot_client,
action_id=action_id,
)
try:
return await handler.apply(params, ctx)
Expand Down Expand Up @@ -303,18 +305,34 @@ async def _apply_pending_rows(
(member.type, resolve_placeholders(member.params, results_by_ref))
for member in member_rows
]
# The front matter names the row whose comment is posted: the
# group's `add_comment`, or else the last update with a comment in
# its `changes` (later changes win). With no comment, nothing reads
# the ID, so the anchor's own will do.
comment_action_id = next(
(m.id for m in member_rows if m.type == "bugzilla.add_comment"),
Comment thread
kyoshino marked this conversation as resolved.
Resolved
next(
(
m.id
for m in reversed(member_rows)
if "comment" in (m.params.get("changes") or {})
),
row.id,
),
)
outcome = await _dispatch(
run,
"bugzilla.update_bug",
merge_resolved(entries),
[],
hackbot_client,
comment_action_id,
)
else:
member_rows = [row]
params = resolve_placeholders(row.params, results_by_ref)
outcome = await _dispatch(
run, row.type, params, attachments, hackbot_client
run, row.type, params, attachments, hackbot_client, row.id
)

# Only stamp applied_at on a real success, so a failed row isn't
Expand Down
99 changes: 99 additions & 0 deletions services/hackbot-api/tests/test_actions_applier.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,105 @@ async def test_extra_comments_applied_separately(monkeypatch):
]


class _ContextRecordingHandler:
def __init__(self):
self.action_ids = []

async def apply(self, params, ctx):
self.action_ids.append(ctx.action_id)
return SimpleNamespace(status="applied", result={}, error=None)


async def test_handlers_get_the_action_id_of_the_comment_they_post(monkeypatch):
"""A coalesced PUT carries its comment's ID, which goes in the front matter."""
handler = _ContextRecordingHandler()
monkeypatch.setattr(actions_applier, "get_handler", lambda t: handler)

update = _row(
0,
"pending",
action_type="bugzilla.update_bug",
params={"bug_id": 5, "changes": {"status": "RESOLVED"}},
)
comment = _row(
1,
"pending",
action_type="bugzilla.add_comment",
params={"bug_id": 5, "text": "done"},
)
other = _row(
2,
"pending",
action_type="bugzilla.add_comment",
params={"bug_id": 99, "text": "different bug"},
)
rows = [(update, []), (comment, []), (other, [])]

await actions_applier._apply_pending_rows(
_FakeDB(), _FakeRun(status=RunStatus.succeeded.value), rows, _CLIENT
)

# Row IDs: the update is 1, its comment 2, the other bug's comment 3.
assert handler.action_ids == [2, 3]


async def test_merged_updates_carry_the_id_of_the_comment_in_their_changes(
monkeypatch,
):
"""Without an add_comment, the ID is the update whose comment is posted."""
handler = _ContextRecordingHandler()
monkeypatch.setattr(actions_applier, "get_handler", lambda t: handler)

rows = [
(
_row(
idx,
"pending",
action_type="bugzilla.update_bug",
params={"bug_id": 5, "changes": changes},
),
[],
)
for idx, changes in enumerate(
[
{"status": "RESOLVED"},
{"comment": {"body": "fixed"}},
{"resolution": "FIXED"},
]
)
]
rows.append(
(
_row(
3,
"pending",
action_type="bugzilla.update_bug",
params={"bug_id": 6, "changes": {"a": 1}},
),
[],
)
)
rows.append(
(
_row(
4,
"pending",
action_type="bugzilla.update_bug",
params={"bug_id": 6, "changes": {"b": 2}},
),
[],
)
)

await actions_applier._apply_pending_rows(
_FakeDB(), _FakeRun(status=RunStatus.succeeded.value), rows, _CLIENT
)

# Bug 5's comment came from the second update (row 2). Bug 6 posts no
# comment and gets its last update's ID (row 5), which nothing reads.
assert handler.action_ids == [2, 5]


async def test_lone_same_type_actions_on_different_bugs_not_merged(monkeypatch):
handler = _RecordingHandler(
SimpleNamespace(status="applied", result={}, error=None)
Expand Down
7 changes: 7 additions & 0 deletions services/hackbot-api/tests/test_agents.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Tests for the agent registry and generic env serialization."""

import json
import re

import pytest
from app.agents import AGENT_REGISTRY, model_to_env
Expand All @@ -17,6 +18,12 @@
from pydantic import ValidationError


def test_agent_names_are_identifiers():
"""BMO reads the agent name from a comment's front matter only if it is one."""
for name in AGENT_REGISTRY:
assert re.fullmatch(r"[A-Za-z0-9._-]+", name), name


def test_model_to_env_uppercases_and_stringifies():
env = model_to_env(BugFixInputs(bug_id=12345, model="claude-opus", max_turns=8))
assert env["BUG_ID"] == "12345"
Expand Down
Loading