Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,29 @@
import os
import subprocess
import tempfile
from collections.abc import Callable
from collections.abc import Callable, Sequence
from contextlib import ExitStack
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Any

from autowebcompat_tools import android_emulator
from autowebcompat_tools.android_browser import AndroidBrowser, AndroidEnvironment
from autowebcompat_tools.browser import ChromeBrowsers, FirefoxBrowsers
from autowebcompat_tools.environment import Environment
from autowebcompat_tools.inputs import AutoWebcompatInput, BugDataInput, BugIdInput
from autowebcompat_tools.mcp_servers import (
build_android_chrome_devtools_server,
build_android_firefox_devtools_server,
build_chrome_devtools_server,
build_firefox_devtools_server,
)
from autowebcompat_tools.result import ResultT
from autowebcompat_tools.task import RunTracker, TaskConfig
from autowebcompat_tools.task import Task as BaseTask
from claude_agent_sdk import McpServerConfig
from claude_agent_sdk.types import McpStdioServerConfig
from pydantic import BaseModel

from .config import BUGZILLA_READ_TOOLS, CHROME_DEVTOOLS_TOOLS, DEVTOOLS_TOOLS
Expand All @@ -44,6 +50,40 @@

PublishFile = Callable[[str, Path, str | None], str]

Browser = Path | AndroidBrowser


def browser_bin(browser: Browser) -> str:
"""BROWSER_BIN for repro scripts: an executable, or an Android debug endpoint."""
return browser.endpoint if isinstance(browser, AndroidBrowser) else str(browser)


def firefox_server(
browser: Browser,
*,
headless: bool,
privileged: bool = False,
extensions: Sequence[str] = (),
) -> McpStdioServerConfig:
"""Firefox DevTools MCP for a desktop binary or an Android browser."""
if isinstance(browser, AndroidBrowser):
return build_android_firefox_devtools_server(
browser, enable_privileged_context=privileged, extensions=extensions
)
return build_firefox_devtools_server(
browser,
headless=headless,
enable_privileged_context=privileged,
profile_path=setup_profile(browser, extensions),
)


def chrome_server(browser: Browser, *, headless: bool) -> McpStdioServerConfig:
"""Chrome DevTools MCP for a desktop binary or an Android browser."""
if isinstance(browser, AndroidBrowser):
return build_android_chrome_devtools_server(browser)
return build_chrome_devtools_server(browser, headless=headless)


class AutowebcompatReproResult(BaseModel):
reproduced: bool
Expand All @@ -61,19 +101,31 @@ class AutowebcompatReproResult(BaseModel):
class Task(BaseTask[ResultT]):
result_server_name = RESULT_SERVER_NAME

def __init__(
self,
task_config: TaskConfig,
run_tracker: RunTracker,
firefox: Browser | None = None,
):
super().__init__(task_config, run_tracker)
self.android = isinstance(firefox, AndroidBrowser)

def system_prompt(self) -> str:
return (HERE / "prompts" / "system.md").read_text()
prompt = (HERE / "prompts" / "system.md").read_text()
if self.android:
prompt += (HERE / "prompts" / "android.md").read_text()
return prompt


def run_confirmation_script(
script_path: Path, firefox_path: Path, headless: bool
script_path: Path,
firefox: Browser,
headless: bool,
) -> ReproductionResult | None:
script_timeout = 5 * 60
env = {
**os.environ,
"BROWSER": "firefox",
"BROWSER_BIN": str(firefox_path),
}
if isinstance(firefox, AndroidBrowser):
firefox.start(fresh=True)
env = {**os.environ, "BROWSER": "firefox", "BROWSER_BIN": browser_bin(firefox)}
if headless:
env["HEADLESS"] = "1"

Expand Down Expand Up @@ -181,38 +233,28 @@ def __init__(
self,
task_config: TaskConfig,
run_tracker: RunTracker,
firefox_path: Path,
profile_path: Path,
firefox: Browser,
input_data: AutoWebcompatInput,
bugzilla_mcp_server: McpServerConfig,
screenshot_dir: Path,
chrome_path: Path,
chrome: Browser,
):
super().__init__(task_config, run_tracker)
super().__init__(task_config, run_tracker, firefox)
self.input_data = input_data
self.firefox_path = firefox_path
self.chrome_path = chrome_path
self.firefox_bin = browser_bin(firefox)
self.chrome_bin = browser_bin(chrome)
self.screenshot_path = make_empty_temp_file(
screenshot_dir, "reproduction=", ".png"
)
self.script_path = make_empty_temp_file(self.repro_dir, "reproduction=", ".mjs")
self.add_mcp_server(
"firefox-devtools",
build_firefox_devtools_server(
firefox_path=firefox_path,
headless=task_config.headless,
enable_script=True,
enable_privileged_context=False,
profile_path=profile_path,
),
firefox_server(firefox, headless=task_config.headless),
DEVTOOLS_TOOLS,
)

self.add_mcp_server(
"chrome-devtools",
build_chrome_devtools_server(
chrome_path=chrome_path, headless=task_config.headless
),
chrome_server(chrome, headless=task_config.headless),
CHROME_DEVTOOLS_TOOLS,
)
if self.input_data.type == "bug_id":
Expand All @@ -223,7 +265,7 @@ def subject(self) -> Any:

def system_prompt(self) -> str:
repro_reference = self.repro_dir / "repro_reference.mjs"
return (
prompt = (
super()
.system_prompt()
.format(
Expand Down Expand Up @@ -255,8 +297,8 @@ def system_prompt(self) -> str:
`{repro_reference}`, write your script to exactly `{self.script_path}`
(read the file before writing), then run it with

`BROWSER=firefox BROWSER_BIN={self.firefox_path} node {self.script_path}`
`BROWSER=chrome BROWSER_BIN={self.chrome_path} node {self.script_path}`
`BROWSER=firefox BROWSER_BIN={self.firefox_bin} node {self.script_path}`
`BROWSER=chrome BROWSER_BIN={self.chrome_bin} node {self.script_path}`

The script checks one browser per run: the difference is demonstrated
when the Firefox run exits with 1 (not working) and the Chrome run exits with
Expand All @@ -277,6 +319,9 @@ def system_prompt(self) -> str:
"""
)
)
if self.android:
prompt += (HERE / "prompts" / "android_bug_reproduction.md").read_text()
return prompt

def user_prompt(self) -> str:
if isinstance(self.input_data, BugDataInput):
Expand All @@ -300,21 +345,14 @@ def __init__(
self,
task_config: TaskConfig,
run_tracker: RunTracker,
firefox_path: Path,
profile_path: Path,
firefox: Browser,
steps: str,
):
super().__init__(task_config, run_tracker)
super().__init__(task_config, run_tracker, firefox)
self.steps = steps
self.add_mcp_server(
"firefox_devtools",
build_firefox_devtools_server(
firefox_path=firefox_path,
headless=task_config.headless,
enable_script=True,
enable_privileged_context=False,
profile_path=profile_path,
),
firefox_server(firefox, headless=task_config.headless),
DEVTOOLS_TOOLS,
)

Expand Down Expand Up @@ -346,20 +384,18 @@ def __init__(
self,
task_config: TaskConfig,
run_tracker: RunTracker,
firefox_path: Path,
profile_path: Path,
firefox: Browser,
steps: str,
) -> None:
super().__init__(task_config, run_tracker)
super().__init__(task_config, run_tracker, firefox)
self.steps = steps
self.add_mcp_server(
"firefox_devtools",
build_firefox_devtools_server(
firefox_path=firefox_path,
firefox_server(
firefox,
headless=task_config.headless,
enable_script=True,
enable_privileged_context=True,
profile_path=profile_path,
privileged=True,
extensions=["chrome-mask"],
),
DEVTOOLS_TOOLS,
)
Expand Down Expand Up @@ -551,7 +587,7 @@ async def run_autowebcompat_repro(
Returns a :class:`AutowebcompatReproResult` on success; raises
:class:`AgentError` if the agent ends in an error.
"""
with Environment() as env:
with Environment() as env, ExitStack() as resources:
if not default_config.headless:
env.start_xvfb()

Expand All @@ -575,13 +611,28 @@ async def run_autowebcompat_repro(
result.failure_reason = "unsupported_ios"
return result

android_requested = "android" in test_plan_result.affects_platforms
android = android_requested and android_emulator.available()

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's currently set up to run either mobile or desktop, depending on what test plan returns, with preference to Android if it's in the list. Thinking we probably want to run it on all platforms that affects_platforms returns?

if android:
logcat = Path(tempfile.mkdtemp(prefix="android-")) / "logcat.txt"

def publish_logcat() -> None:
if logcat.exists():
publish_file("android-logcat.txt", logcat, "text/plain")

resources.callback(publish_logcat)
firefox_browser, chrome_browser = resources.enter_context(
AndroidEnvironment(logcat)
)
elif android_requested:
logger.warning("Android emulator is unavailable; using desktop browsers")

async def next_repro_task(
channel: FirefoxChannel,
extra: str | None = None,
config: TaskConfig = default_config,
) -> None:
browser = getattr(firefox_browser, channel.value)
profile = setup_profile(browser)
logger.info(
"Trying reproduction in %s%s",
channel,
Expand All @@ -594,7 +645,6 @@ async def next_repro_task(
config,
tracker,
browser,
profile,
input_data,
bugzilla_mcp_server,
screenshots_dir,
Expand All @@ -603,7 +653,9 @@ async def next_repro_task(
else:
if initial_repro.script_path is not None:
script_result = run_confirmation_script(
initial_repro.script_path, browser, default_config.headless
initial_repro.script_path,
browser,
default_config.headless,
)
if script_result is not None:
repro_results.set_result(channel, extra, script_result)
Expand All @@ -617,7 +669,6 @@ async def next_repro_task(
config,
tracker,
browser,
profile,
initial_repro.steps,
)
repro_results.set_result(channel, extra, await task.run())
Expand All @@ -640,9 +691,11 @@ async def next_repro_task(
result.failure_reason = "not_firefox_specific"
return result

if not repro_results.reproduced and test_plan_result.affects_platforms == [
"android"
]:
if (
not repro_results.reproduced
and not android
and test_plan_result.affects_platforms == ["android"]
):
result = repro_results.into_result()
result.summary = "Issue was identified as Android only and didn't reproduce on desktop nightly"
result.failure_reason = "unsupported_android"
Expand All @@ -655,21 +708,21 @@ async def next_repro_task(
):
await next_repro_task(FirefoxChannel.stable)

if repro_results.reproduced or "esr" in test_plan_result.affects_channels:
if not android and (
repro_results.reproduced or "esr" in test_plan_result.affects_channels
):
# If we have any result try ESR as a possible regression baseline,
# otherwise try ESR if we think it's affected
await next_repro_task(FirefoxChannel.esr)

if repro_results.initial_repro is not None:
channel = repro_results.initial_repro.channel
browser = getattr(firefox_browser, channel.value)
profile = setup_profile(browser, extensions=["chrome-mask"])

task = ChromeMaskReproduction(
default_config,
tracker,
browser,
profile,
repro_results.initial_repro.steps,
)
repro_results.set_result(channel, "chrome-mask", await task.run())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"mcp__firefox-devtools__set_viewport_size",
"mcp__firefox-devtools__get_firefox_info",
"mcp__firefox-devtools__get_firefox_output",
"mcp__firefox-devtools__restart_firefox",
]

# Chrome DevTools MCP tools (chrome-devtools-mcp), exposed under the
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
You are testing real Firefox for Android on an emulator. The MCP adapter starts
and activates the browser when its tools are used. The viewport is the phone
screen: do not resize it or substitute desktop mobile emulation.
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Chrome for Android runs on the same emulator. In Chrome, use new_page before
navigate_page to create and select a tab. Keep screenshots of the Firefox issue
before switching to Chrome.
On Android, BROWSER_BIN is the browser's remote debugging endpoint; use the
provided reference, including its connection/disconnection code.
Before running the Firefox script, call restart_firefox once to release Firefox
MCP's WebDriver session; scripts and MCP cannot hold concurrent Firefox WebDriver
sessions. Before running the Chrome script, call a Chrome DevTools tool such as
list_pages so Chrome is in the foreground. After the scripts, Firefox MCP
reconnects on its next tool call; reopen the affected URL before reusing it.
Loading