From 8f7c7fb2605681bd93848ebb83b7ce383c5913f3 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Mon, 16 Nov 2026 23:54:28 +0100 Subject: [PATCH 1/2] Find the tests to run for modified paths at the pushed revision The manifests and tasks selected for the modified paths were found in the working directory, which is not updated to the pushed revisions as it is too slow, so new tests and manifests were missed. Read the files at the head of the push with Mercurial instead: the files of a revision are computed from the ones of the previous revision and the files changed between the two, and only the changed test manifests are parsed again. The manifests at tip are parsed when booting the worker, before it forks for each job. Fixes #5854 --- bugbug/test_scheduling.py | 467 ++++++++++++++++------ http_service/bugbug_http/boot.py | 9 + http_service/bugbug_http/models.py | 9 +- http_service/tests/test_schedule_tests.py | 2 +- tests/test_test_scheduling.py | 221 +++++----- 5 files changed, 463 insertions(+), 245 deletions(-) diff --git a/bugbug/test_scheduling.py b/bugbug/test_scheduling.py index 912b84bda22..d95058e4e01 100644 --- a/bugbug/test_scheduling.py +++ b/bugbug/test_scheduling.py @@ -16,6 +16,7 @@ import shelve import shutil import struct +import subprocess import tomllib from datetime import datetime from pathlib import Path @@ -35,6 +36,8 @@ cast, ) +import hglib +import orjson from tqdm import tqdm from bugbug import db, repository @@ -1385,8 +1388,6 @@ def get_test_info(date: datetime) -> dict[str, Any]: return r.json() -manifest_by_path: dict[str, set[str]] | None = None - # If a modified file is close to more manifests than this, it is too broad # (e.g. dom/moz.build) for the sibling heuristic to be informative, so we # don't schedule any of them and leave the decision to the model. @@ -1450,112 +1451,326 @@ def get_suite_test_dirs(repo_dir: str) -> dict[str, tuple[str, ...]]: MAX_SIBLING_MANIFESTS = 42 -def find_manifests_for_paths(repo_dir_str: str, paths: list[str]) -> set[str]: - global manifest_by_path +class _RevisionFiles: + """The files of a repository at a revision, read from the store with Mercurial. - repo_dir = Path(repo_dir_str) + This avoids updating the working directory, which is slow on a large repository. + """ - manifests = set() + def __init__( + self, repo_dir: str, rev: str, base: "_RevisionFiles | None" = None + ) -> None: + """Get the files at a revision. + + If `base` is given, they are computed from the files at another revision and the files + changed between the two revisions, which is faster than listing all the files. + """ + self.repo_dir = repo_dir + self.rev = rev + self._hg: hglib.client.hgclient | None = None + # The parsed test manifests (None when not a test manifest), by path. + self.parsed_manifests: dict[str, dict | None] | None = None + # The test manifests by the paths they include. + self.manifest_by_path: dict[str, set[str]] | None = None + # The files changed from the base revision (None if all of them are new). + self.changed: set[str] | None = None + + files: set[str] + if base is None: + files = set(self._run("files", "-r", rev, "-0")) + else: + files = set(base._files_set) + self.changed = set() + for entry in self._run( + "status", "--rev", base.rev, "--rev", rev, "-amr", "-0" + ): + status, path = entry.split(" ", 1) + self.changed.add(path) + if status == "R": + files.discard(path) + else: + files.add(path) + if base.parsed_manifests is not None: + self.parsed_manifests = { + path: data + for path, data in base.parsed_manifests.items() + if path not in self.changed + } + + self._files_set: set[str] = files + self.files = sorted(files) + + # The directories of removed files are kept, as empty directories don't affect the + # results, and it is faster than recomputing them all. + self._dirs_set: set[str] = set() if base is None else set(base._dirs_set) + for path in files if self.changed is None else self.changed & files: + parent = os.path.dirname(path) + while parent and parent not in self._dirs_set: + self._dirs_set.add(parent) + parent = os.path.dirname(parent) + self.dirs = sorted(self._dirs_set) + + def _run(self, *args: str) -> list[str]: + """Run a Mercurial command printing NUL-separated paths, relative to the root.""" + # Not using the command server, as it is slow to read a large output. + out = subprocess.run( + ["hg", *args], + cwd=self.repo_dir, + check=True, + capture_output=True, + ).stdout + return [ + entry.decode("utf-8", "surrogateescape") + for entry in out.split(b"\0") + if entry + ] - if manifest_by_path is None: - manifest_by_path = collections.defaultdict(set) + def _command(self, *args, **kwargs) -> bytes: + # The command server is kept open until the end of the "with" block using these files, to + # avoid starting Mercurial for each command. + if self._hg is None: + self._hg = hglib.open(self.repo_dir) + return self._hg.rawcommand(hglib.util.cmdbuilder(*args, **kwargs)) + + def __enter__(self) -> "_RevisionFiles": + return self + + def __exit__(self, *exc_info) -> None: + if self._hg is not None: + self._hg.close() + self._hg = None + + @staticmethod + def _under(sorted_paths: list[str], path: str) -> list[str]: + """The paths in a sorted list which are in the given directory.""" + if not path: + return sorted_paths + prefix = f"{path}/" + start = bisect.bisect_left(sorted_paths, prefix) + # "0" is the character after "/". + end = bisect.bisect_left(sorted_paths, f"{path}0", lo=start) + return sorted_paths[start:end] + + def toml_files(self) -> Iterable[str]: + """All the .toml files.""" + return (path for path in self.files if path.endswith(".toml")) + + def read(self, paths: list[str]) -> dict[str, bytes]: + """The contents of the given files, skipping the ones which don't exist.""" + paths = [path for path in paths if path in self._files_set] + contents: dict[str, bytes] = {} + for i in range(0, len(paths), 1000): + out = self._command( + b"cat", + *(f"path:{path}".encode("utf-8") for path in paths[i : i + 1000]), + rev=self.rev.encode("ascii"), + template=b"json", + ) + for entry in orjson.loads(out): + contents[entry["path"]] = entry["data"].encode( + "utf-8", "surrogateescape" + ) + return contents + + def glob(self, pattern: str) -> list[str]: + """The files matching a recursive glob pattern.""" + regex = re.compile(glob.translate(pattern, recursive=True)) + fixed = pattern.split("*", 1)[0] + return [ + path + for path in self._under(self.files, os.path.dirname(fixed)) + if regex.match(path) + ] - for toml_path in repo_dir.rglob("*.toml"): - # HACK: These are not test manifests, skip them. - if ( - toml_path.name in ("Cargo.toml", "pyproject.toml") - or toml_path.parent.name == "test-manifest-toml" - or "third_party" in toml_path.parts - or "manifestparser" in toml_path.parts - ): - continue + def is_dir(self, path: str) -> bool: + return path in self._dirs_set - with open(toml_path, "rb") as toml_f: - data = tomllib.load(toml_f) + def children(self, path: str) -> list[str]: + """The names of the files and directories directly in a directory.""" + start = len(path) + 1 if path else 0 + return list({p[start:].split("/", 1)[0] for p in self._under(self.files, path)}) - # HACK: If there is no "DEFAULT" key and there is no key that starts with "test", this is unlikely a test manifest. - if "DEFAULT" not in data and not any( - key.startswith("test") for key in data - ): + def subdirs(self, path: str) -> Iterable[str]: + """All the directories (recursively) in a directory, excluding itself.""" + return self._under(self.dirs, path) + + def files_in(self, path: str, suffix: str) -> Iterable[str]: + """All the files (recursively) in a directory with the given suffix.""" + return (p for p in self._under(self.files, path) if p.endswith(suffix)) + + +def _get_repo_files(repo_dir: str, rev: str) -> _RevisionFiles: + global _revision_files + + if _revision_files is None or _revision_files.repo_dir != repo_dir: + _revision_files = _RevisionFiles(repo_dir, rev) + elif _revision_files.rev != rev: + # Compute the files from the ones of the last revision, which is faster. + _revision_files = _RevisionFiles(repo_dir, rev, _revision_files) + + return _revision_files + + +# The files at the last revision analyzed. +_revision_files: _RevisionFiles | None = None + + +def _parse_test_manifest(path: str, content: bytes) -> dict | None: + """Parse a test manifest, returning None if the file is not a test manifest.""" + try: + data = tomllib.loads(content.decode("utf-8")) + except UnicodeDecodeError, tomllib.TOMLDecodeError: + logger.warning("Failed to parse %s", path) + return None + + # HACK: If there is no "DEFAULT" key and there is no key that starts with "test", this is unlikely a test manifest. + if "DEFAULT" not in data and not any(key.startswith("test") for key in data): + return None + + return data + + +def _is_candidate_manifest(path: str) -> bool: + parts = path.split("/") + # HACK: These are not test manifests, skip them. + return not ( + parts[-1] in ("Cargo.toml", "pyproject.toml") + or (len(parts) > 1 and parts[-2] == "test-manifest-toml") + or "third_party" in parts + or "manifestparser" in parts + ) + + +def _normalize(path: str) -> str | None: + """Normalize a path relative to the root of the repository, None if outside it.""" + path = os.path.normpath(path) + if path == ".." or path.startswith("../"): + return None + return path + + +def _build_manifest_by_path( + files: _RevisionFiles, manifests: dict[str, dict] +) -> dict[str, set[str]]: + result: dict[str, set[str]] = collections.defaultdict(set) + + def add(path: str | None, toml_rel: str) -> None: + if path is not None: + result[path].add(toml_rel) + + for toml_rel, data in manifests.items(): + toml_dir = os.path.dirname(toml_rel) + + # The manifest path itself, so we schedule it when it is touched. + add(toml_rel, toml_rel) + + # Collect head files. + head_value = data.get("DEFAULT", {}).get("head", []) + head_files = ( + head_value if isinstance(head_value, list) else head_value.split(" ") + ) + for head_file in head_files: + if not head_file.strip(): continue - toml_rel = toml_path.relative_to(repo_dir) - toml_dir = toml_path.parent + add(_normalize(os.path.join(toml_dir, head_file)), toml_rel) - # The manifest path itself, so we schedule it when it is touched. - manifest_by_path[str(toml_rel)].add(str(toml_rel)) + # Collect support files. + def collect_support_files(value, toml_dir=toml_dir, toml_rel=toml_rel): + support_files = value.get("support-files", []) + if isinstance(support_files, str): + support_files = [support_files] - # Collect head files. - head_value = data.get("DEFAULT", {}).get("head", []) - head_files = ( - head_value if isinstance(head_value, list) else head_value.split(" ") - ) - for head_file in head_files: - if not head_file.strip(): + for support_file in support_files: + if not support_file.strip(): continue - manifest_by_path[ - str((toml_dir / head_file).resolve().relative_to(repo_dir)) - ].add(str(toml_rel)) - - # Collect support files. - def collect_support_files(value, toml_dir=toml_dir, toml_rel=toml_rel): - support_files = value.get("support-files", []) - if isinstance(support_files, str): - support_files = [support_files] - - for support_file in support_files: - if not support_file.strip(): - continue - - if support_file.startswith("!"): - support_file = support_file[1:] - - if support_file.startswith("/"): - support_file_path = (repo_dir / support_file[1:]).resolve() - else: - support_file_path = (toml_dir / support_file).resolve() - - if "*" in support_file: - files = [ - Path(f) - for f in glob.glob(str(support_file_path), recursive=True) - ] - else: - files = [support_file_path] - - for f in files: - manifest_by_path[str(f.relative_to(repo_dir))].add( - str(toml_rel) - ) + if support_file.startswith("!"): + support_file = support_file[1:] - collect_support_files(data.get("DEFAULT", {})) + if support_file.startswith("/"): + support_file_path = _normalize(support_file[1:]) + else: + support_file_path = _normalize(os.path.join(toml_dir, support_file)) - # Collect test files. - for key, val in data.items(): - if key != "DEFAULT" and isinstance(val, dict): - collect_support_files(val) + if support_file_path is None: + continue + + if "*" in support_file: + for f in files.glob(support_file_path): + add(f, toml_rel) + else: + add(support_file_path, toml_rel) + + collect_support_files(data.get("DEFAULT", {})) + + # Collect test files. + for key, val in data.items(): + if key != "DEFAULT" and isinstance(val, dict): + collect_support_files(val) + + add(_normalize(os.path.join(toml_dir, key)), toml_rel) + + return result - manifest_by_path[ - str((toml_dir / key).resolve().relative_to(repo_dir)) - ].add(str(toml_rel)) + +def _get_manifest_by_path(files: _RevisionFiles) -> dict[str, set[str]]: + if files.manifest_by_path is not None: + return files.manifest_by_path + + candidates = [p for p in files.toml_files() if _is_candidate_manifest(p)] + # Only read the manifests which changed since the revision the files were computed from. + if files.parsed_manifests is None: + files.parsed_manifests = {} + to_read = [p for p in candidates if p not in files.parsed_manifests] + contents = files.read(to_read) + for path in to_read: + content_or_none = contents.get(path) + files.parsed_manifests[path] = ( + _parse_test_manifest(path, content_or_none) + if content_or_none is not None + else None + ) + + manifests = { + path: data + for path in candidates + if (data := files.parsed_manifests[path]) is not None + } + + files.manifest_by_path = _build_manifest_by_path(files, manifests) + return files.manifest_by_path + + +def find_manifests_for_paths(repo_dir: str, paths: list[str], rev: str) -> set[str]: + """The test manifests and web-platform test directories to run for modified paths. + + The files of the repository at revision `rev` are used, read with Mercurial without + updating the working directory, as it is slow on a large repository. + """ + with _get_repo_files(repo_dir, rev) as files: + return _find_manifests_for_paths(files, paths) + + +def _find_manifests_for_paths(files: _RevisionFiles, paths: list[str]) -> set[str]: + manifest_by_path = _get_manifest_by_path(files) + + manifests = set() for path in paths: + parent = os.path.dirname(path) + # If a manifest, a test, or a support file is modified, run the manifest that includes it. if path in manifest_by_path: manifests.update(manifest_by_path[path]) # Skip root-level files, otherwise we'd walk the whole repository and # schedule every manifest. - elif (repo_dir / path).parent != repo_dir: + elif parent: # Find manifests that are in test subfolders close to a modified file (e.g. if dom/battery/BatteryManager.cpp is modified, we should run dom/battery/test/chrome.toml and dom/battery/test/mochitest.toml). sibling_manifests: set[str] = set() - for sibling in (repo_dir / path).parent.rglob("*"): - if sibling.is_dir() and repository.is_test(f"{str(sibling)}/"): - sibling_manifests.update( - str(f.relative_to(repo_dir)) - for f in sibling.rglob("*.toml") - if f.is_file() - ) + for sibling in files.subdirs(parent): + if repository.is_test(f"/{sibling}/"): + sibling_manifests.update(files.files_in(sibling, ".toml")) if len(sibling_manifests) <= MAX_SIBLING_MANIFESTS: manifests.update(sibling_manifests) @@ -1570,27 +1785,22 @@ def collect_support_files(value, toml_dir=toml_dir, toml_rel=toml_rel): relative = Path(path).relative_to(f"{base}/{sub}") - test_root = repo_dir / base / "tests" + test_root = Path(base) / "tests" cur_dir = test_root / relative.parent - def is_wpt_file(p: Path) -> bool: - return any( - str(p).endswith(suffix) - for suffix in (".html", ".any.js", ".worker.js") - ) + def is_wpt_file(name: str) -> bool: + return name.endswith((".html", ".any.js", ".worker.js")) def has_wpt_files(d: Path) -> bool: - return d.is_dir() and any(is_wpt_file(p) for p in d.iterdir()) + return files.is_dir(str(d)) and any( + is_wpt_file(name) for name in files.children(str(d)) + ) while cur_dir != test_root and not has_wpt_files(cur_dir): cur_dir = cur_dir.parent - # Ignore files in top level "meta" or "tests". - if len(cur_dir.parts) == 1: - break - if has_wpt_files(cur_dir): - manifests.add(str(cur_dir.relative_to(repo_dir))) + manifests.add(str(cur_dir)) break @@ -1606,23 +1816,36 @@ def has_wpt_files(d: Path) -> bool: } -def _get_cppunit_test_names(repo_dir: Path) -> set[str]: - try: - with open(repo_dir / "testing" / "cppunittest.toml", "rb") as f: - data = tomllib.load(f) - return {f"{key}.cpp" for key in data if key != "DEFAULT"} - except FileNotFoundError: +def _get_cppunit_test_names(files: _RevisionFiles) -> set[str]: + content = files.read(["testing/cppunittest.toml"]).get("testing/cppunittest.toml") + if content is None: logger.error( "testing/cppunittest.toml wasn't found, cppunit heuristic won't work" ) return set() + data = tomllib.loads(content.decode("utf-8")) + return {f"{key}.cpp" for key in data if key != "DEFAULT"} + def find_tasks_for_paths( - repo_dir_str: str, known_tasks: tuple[str, ...], paths: list[str] + repo_dir: str, + known_tasks: tuple[str, ...], + paths: list[str], + rev: str, ) -> list[str]: - repo_dir = Path(repo_dir_str) + """The tasks to run for modified paths, according to some heuristics. + + The files of the repository at revision `rev` are used, read with Mercurial without + updating the working directory, as it is slow on a large repository. + """ + with _get_repo_files(repo_dir, rev) as files: + return _find_tasks_for_paths(files, known_tasks, paths) + +def _find_tasks_for_paths( + files: _RevisionFiles, known_tasks: tuple[str, ...], paths: list[str] +) -> list[str]: select_gtest = False select_cppunit = False select_rusttests = False @@ -1642,34 +1865,34 @@ def find_tasks_for_paths( # Any file in a folder close to a gtest folder is modified (e.g. dom/media/CubebUtils.cpp and we have dom/media/gtest/). if not select_gtest: for path in paths: + parent = os.path.dirname(path) # Skip root-level files, otherwise we'd walk the whole repository. - if (repo_dir / path).parent == repo_dir: + if not parent: continue - for sibling in (repo_dir / path).parent.rglob("*"): - if sibling.is_dir() and any( - part in _GTEST_FOLDERS for part in sibling.parts - ): - select_gtest = True - break - if select_gtest: + if any( + part in _GTEST_FOLDERS + for sibling in files.subdirs(parent) + for part in sibling.split("/") + ): + select_gtest = True break # Any C/C++ file containing gtests is modified. if not select_gtest: - for path in paths: - if repository.get_type(path) in ["C/C++", "Objective-C/C++"]: - try: - with open(repo_dir / path, "rb") as f: - select_gtest = _GTEST_RE.search(f.read()) is not None - if select_gtest: - break - except OSError: - pass + cpp_paths = [ + path + for path in paths + if repository.get_type(path) in ["C/C++", "Objective-C/C++"] + ] + select_gtest = any( + _GTEST_RE.search(content) is not None + for content in files.read(cpp_paths).values() + ) # Cppunit: run if infrastructure files are modified, or any .cpp file whose # name matches a stem listed in testing/cppunittest.toml is modified. - cppunit_test_names = _get_cppunit_test_names(repo_dir) + cppunit_test_names = _get_cppunit_test_names(files) for path in paths: if path in _CPPUNIT_INFRA_FILES or Path(path).name in cppunit_test_names: select_cppunit = True diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 5c66258a332..a6bc608ed24 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -306,4 +306,13 @@ def extract_commit_experiences() -> None: clone_autoland_future.result() retrieve_build_platforms_future.result() + # Parse the test manifests once, before the worker forks for each job, so jobs only need to + # parse the manifests which changed since then. + try: + with hglib.open(REPO_DIR) as hg: + tip = hg.log(revrange=b"tip")[0].node.decode("ascii") + test_scheduling.find_manifests_for_paths(REPO_DIR, [], tip) + except Exception: + logger.exception("Failed to parse the test manifests") + logger.info("Worker boot done") diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 10ca854072c..82b924e0fba 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -957,6 +957,9 @@ def _analyze_patch( known_tasks = get_known_tasks() modified_paths = list(set(path for commit in commits for path in commit["files"])) + # The working directory is not updated to the pushed revisions, as it is slow, so the files + # are read at the head of the push. + head_rev = revs[-1].decode("ascii") tasks = ( testlabelselect_model.select_tests( @@ -966,7 +969,7 @@ def _analyze_patch( else {} ) for task in test_scheduling.find_tasks_for_paths( - REPO_DIR, known_tasks, modified_paths + REPO_DIR, known_tasks, modified_paths, head_rev ): tasks[task] = 1.0 for task in backout_tasks: @@ -1010,7 +1013,9 @@ def _analyze_patch( for group, confidence in groups.items() if very_high_confidence is not None and confidence >= very_high_confidence } - for group in test_scheduling.find_manifests_for_paths(REPO_DIR, modified_paths): + for group in test_scheduling.find_manifests_for_paths( + REPO_DIR, modified_paths, head_rev + ): groups[group] = 1.0 risks[group] = 1.0 very_likely_groups.add(group) diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index cfb9db40544..6a16a17862c 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -497,7 +497,7 @@ def select_configs(groups, modified_paths, very_likely_groups, used_configs=()): monkeypatch.setattr( models.test_scheduling, "find_manifests_for_paths", - lambda repo_dir, paths: ["test-group3"], + lambda repo_dir, paths, rev: ["test-group3"], ) assert models.schedule_tests("mozilla-central", rev) == "OK" diff --git a/tests/test_test_scheduling.py b/tests/test_test_scheduling.py index c291353d124..26901c02ed8 100644 --- a/tests/test_test_scheduling.py +++ b/tests/test_test_scheduling.py @@ -6,8 +6,9 @@ import contextlib import math from datetime import datetime -from typing import Any +from typing import Any, Iterator +import hglib import pytest from _pytest.monkeypatch import MonkeyPatch @@ -1208,7 +1209,43 @@ def assert_val(manifest, val): assert_val("browser.ini", 42) +class _Finder: + """Calls the functions finding what to run for paths, at a revision with the current files. + + The files are committed and the working directory is emptied before each call, to check + that they are only read from the repository store. + """ + + def __init__(self, tmp_path) -> None: + self.repo_dir = str(tmp_path) + hglib.init(self.repo_dir) + + @contextlib.contextmanager + def _rev(self) -> Iterator[str]: + with hglib.open(self.repo_dir) as hg: + hg.addremove() + if hg.status(): + hg.commit(message=b"Commit", user=b"bugbug") + rev = hg.log(limit=1)[0].node.decode("ascii") + hg.update(rev=b"null") + try: + yield rev + finally: + hg.update(rev=b"tip") + + def manifests(self, paths: list[str]) -> set[str]: + with self._rev() as rev: + return test_scheduling.find_manifests_for_paths(self.repo_dir, paths, rev) + + def tasks(self, known_tasks: tuple[str, ...], paths: list[str]) -> list[str]: + with self._rev() as rev: + return test_scheduling.find_tasks_for_paths( + self.repo_dir, known_tasks, paths, rev + ) + + def test_find_manifests_for_paths(tmp_path) -> None: + find = _Finder(tmp_path) (tmp_path / "dom" / "battery" / "test").mkdir(parents=True) (tmp_path / "dom" / "battery" / "test" / "mochitest.toml").touch() (tmp_path / "dom" / "battery" / "test" / "chrome.toml").touch() @@ -1242,29 +1279,23 @@ def test_find_manifests_for_paths(tmp_path) -> None: (tmp_path / "absolute_path_with_subdirglob" / "asd.js").touch() (tmp_path / "absolute_path_with_subdirglob" / "subdir" / "asd.js").touch() - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["dom/battery/BatteryManager.cpp"] - ) == { + assert find.manifests(["dom/battery/BatteryManager.cpp"]) == { "dom/battery/test/mochitest.toml", "dom/battery/test/chrome.toml", } - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["dom/battery/BatteryManager.cpp", "test/chrome.toml"] - ) == { + assert find.manifests(["dom/battery/BatteryManager.cpp", "test/chrome.toml"]) == { "dom/battery/test/mochitest.toml", "dom/battery/test/chrome.toml", "test/chrome.toml", } - assert test_scheduling.find_manifests_for_paths(str(tmp_path), ["prova.js"]) == { - "test/chrome.toml" - } + assert find.manifests(["prova.js"]) == {"test/chrome.toml"} # A root-level file that is not referenced by any manifest must not # schedule every manifest in the repository. (tmp_path / "mach").touch() - assert test_scheduling.find_manifests_for_paths(str(tmp_path), ["mach"]) == set() + assert find.manifests(["mach"]) == set() # A file close to too many manifests (e.g. dom/moz.build) must not # schedule all of them. @@ -1275,63 +1306,41 @@ def test_find_manifests_for_paths(tmp_path) -> None: (tmp_path / "hub" / f"component{i}" / "test" / "mochitest.toml").touch() assert ( - len(test_scheduling.find_manifests_for_paths(str(tmp_path), ["hub/moz.build"])) - == test_scheduling.MAX_SIBLING_MANIFESTS + len(find.manifests(["hub/moz.build"])) == test_scheduling.MAX_SIBLING_MANIFESTS ) (tmp_path / "hub" / "one_more" / "test").mkdir(parents=True) (tmp_path / "hub" / "one_more" / "test" / "mochitest.toml").touch() - assert ( - test_scheduling.find_manifests_for_paths(str(tmp_path), ["hub/moz.build"]) - == set() - ) + assert find.manifests(["hub/moz.build"]) == set() # The cap applies per path, so a narrow file is still scheduled when # modified together with a broad one. - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["hub/moz.build", "dom/battery/BatteryManager.cpp"] - ) == { + assert find.manifests(["hub/moz.build", "dom/battery/BatteryManager.cpp"]) == { "dom/battery/test/mochitest.toml", "dom/battery/test/chrome.toml", } - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["test/test_resolve_uris_ipc.js"] - ) == { + assert find.manifests(["test/test_resolve_uris_ipc.js"]) == { "test/chrome.toml", "test/mochitest.toml", } - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["test/relative/path.png"] - ) == {"test/chrome.toml"} + assert find.manifests(["test/relative/path.png"]) == {"test/chrome.toml"} - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["absolute_path_with_glob/asd.js"] - ) == {"test/chrome.toml"} + assert find.manifests(["absolute_path_with_glob/asd.js"]) == {"test/chrome.toml"} - assert ( - test_scheduling.find_manifests_for_paths( - str(tmp_path), ["absolute_path_with_glob/asd.png"] - ) - == set() - ) + assert find.manifests(["absolute_path_with_glob/asd.png"]) == set() - assert ( - test_scheduling.find_manifests_for_paths( - str(tmp_path), ["absolute_path_with_glob/subdir/asd.js"] - ) - == set() - ) + assert find.manifests(["absolute_path_with_glob/subdir/asd.js"]) == set() - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["absolute_path_with_subdirglob/asd.js"] - ) == {"test/chrome.toml"} + assert find.manifests(["absolute_path_with_subdirglob/asd.js"]) == { + "test/chrome.toml" + } - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), ["absolute_path_with_subdirglob/subdir/asd.js"] - ) == {"test/chrome.toml"} + assert find.manifests(["absolute_path_with_subdirglob/subdir/asd.js"]) == { + "test/chrome.toml" + } (tmp_path / "testing/web-platform/tests/html/semantics").mkdir(parents=True) (tmp_path / "testing/web-platform/tests/.gitignore").touch() @@ -1370,46 +1379,44 @@ def test_find_manifests_for_paths(tmp_path) -> None: / "testing/web-platform/tests/encrypted-media/content/content-metadata.js" ).touch() - assert ( - test_scheduling.find_manifests_for_paths( - str(tmp_path), ["testing/web-platform/tests/.gitignore"] - ) - == set() - ) + assert find.manifests(["testing/web-platform/tests/.gitignore"]) == set() - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), + assert find.manifests( ["testing/web-platform/tests/html/semantics/rellist-feature-detection.html"], ) == {"testing/web-platform/tests/html/semantics"} assert ( - test_scheduling.find_manifests_for_paths( - str(tmp_path), ["testing/web-platform/tests/html/semantics/META.yml"] - ) - == set() + find.manifests(["testing/web-platform/tests/html/semantics/META.yml"]) == set() ) - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), + assert find.manifests( [ "testing/web-platform/tests/html/semantics/interactive-elements/contextmenu-historical.html" ], ) == {"testing/web-platform/tests/html/semantics/interactive-elements"} - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), + assert find.manifests( [ "testing/web-platform/mozilla/meta/pointerevents/pointerevent_click_during_parent_capture.html.ini" ], ) == {"testing/web-platform/mozilla/tests/pointerevents"} - assert test_scheduling.find_manifests_for_paths( - str(tmp_path), + assert find.manifests( ["testing/web-platform/tests/encrypted-media/content/content-metadata.js"], ) == {"testing/web-platform/tests/encrypted-media"} + # The files are updated from the ones at the previous revision: a removed manifest isn't run + # anymore, and a modified one is parsed again. + (tmp_path / "test" / "mochitest.toml").unlink() + (tmp_path / "test" / "chrome.toml").write_text( + manifest.replace('"../prova.js"', '""') + ) + assert find.manifests(["test/test_resolve_uris_ipc.js"]) == {"test/chrome.toml"} + assert find.manifests(["prova.js"]) == set() + def test_find_tasks_for_paths(tmp_path) -> None: + find = _Finder(tmp_path) known_tasks = ( "test-linux64/opt-gtest-1proc", "test-linux64/opt-mochitest-browser-chrome-1proc", @@ -1426,51 +1433,35 @@ def test_find_tasks_for_paths(tmp_path) -> None: # Non-C/C++ file containing GTest patterns should not trigger gtest selection. (tmp_path / "script.py").write_bytes(b"TEST(Foo, Bar) {}") - assert ( - test_scheduling.find_tasks_for_paths(str(tmp_path), known_tasks, ["script.py"]) - == [] - ) + assert find.tasks(known_tasks, ["script.py"]) == [] # C/C++ file without GTest patterns should not trigger gtest selection. (tmp_path / "source.cpp").write_bytes(b"int main() { return 0; }") - assert ( - test_scheduling.find_tasks_for_paths(str(tmp_path), known_tasks, ["source.cpp"]) - == [] - ) + assert find.tasks(known_tasks, ["source.cpp"]) == [] # C/C++ file with TEST macro selects tasks containing "gtest". (tmp_path / "test_foo.cpp").write_bytes(b"TEST(FooTest, Bar) {}\n") - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["test_foo.cpp"] - ) == [ + assert find.tasks(known_tasks, ["test_foo.cpp"]) == [ "test-linux64/opt-gtest-1proc", "test-linux64/opt-gtest-e10s", ] # C/C++ file with TEST_F macro also triggers gtest selection. (tmp_path / "test_fixture.cpp").write_bytes(b"TEST_F(FooFixture, Bar) {}\n") - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["test_fixture.cpp"] - ) == [ + assert find.tasks(known_tasks, ["test_fixture.cpp"]) == [ "test-linux64/opt-gtest-1proc", "test-linux64/opt-gtest-e10s", ] - # Non-existent C/C++ file raises OSError which is silently skipped. - assert ( - test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["nonexistent.cpp"] - ) - == [] - ) + # Non-existent C/C++ file is skipped. + assert find.tasks(known_tasks, ["nonexistent.cpp"]) == [] # No paths -> no tasks selected. - assert test_scheduling.find_tasks_for_paths(str(tmp_path), known_tasks, []) == [] + assert find.tasks(known_tasks, []) == [] # known_tasks without any "gtest" task -> empty even when GTest file present. assert ( - test_scheduling.find_tasks_for_paths( - str(tmp_path), + find.tasks( ("test-linux64/opt-mochitest-browser-chrome-1proc",), ["test_foo.cpp"], ) @@ -1478,9 +1469,7 @@ def test_find_tasks_for_paths(tmp_path) -> None: ) # Two paths: one C/C++ with GTest patterns and one without -> gtest tasks selected. - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["source.cpp", "test_foo.cpp"] - ) == [ + assert find.tasks(known_tasks, ["source.cpp", "test_foo.cpp"]) == [ "test-linux64/opt-gtest-1proc", "test-linux64/opt-gtest-e10s", ] @@ -1490,18 +1479,14 @@ def test_find_tasks_for_paths(tmp_path) -> None: (tmp_path / "dom" / "media" / "gtest" / "TestCubeb.cpp").write_bytes( b"// no test macros\n" ) - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["dom/media/gtest/TestCubeb.cpp"] - ) == [ + assert find.tasks(known_tasks, ["dom/media/gtest/TestCubeb.cpp"]) == [ "test-linux64/opt-gtest-1proc", "test-linux64/opt-gtest-e10s", ] # File in a folder adjacent to a gtest subfolder triggers gtest selection. (tmp_path / "dom" / "media" / "CubebUtils.cpp").write_bytes(b"int foo() {}\n") - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["dom/media/CubebUtils.cpp"] - ) == [ + assert find.tasks(known_tasks, ["dom/media/CubebUtils.cpp"]) == [ "test-linux64/opt-gtest-1proc", "test-linux64/opt-gtest-e10s", ] @@ -1509,44 +1494,40 @@ def test_find_tasks_for_paths(tmp_path) -> None: # Rust file triggers rusttests selection. (tmp_path / "servo").mkdir() (tmp_path / "servo" / "lib.rs").write_bytes(b"pub fn foo() {}\n") - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["servo/lib.rs"] - ) == ["test-linux64/opt-rusttests"] + assert find.tasks(known_tasks, ["servo/lib.rs"]) == ["test-linux64/opt-rusttests"] # Non-Rust file does not trigger rusttests selection. - assert "test-linux64/opt-rusttests" not in test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["servo/foo.cpp"] + assert "test-linux64/opt-rusttests" not in find.tasks( + known_tasks, ["servo/foo.cpp"] ) # Modifying testing/cppunittest.toml itself triggers cppunit selection. - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["testing/cppunittest.toml"] - ) == ["test-linux64/opt-cppunit"] + assert find.tasks(known_tasks, ["testing/cppunittest.toml"]) == [ + "test-linux64/opt-cppunit" + ] # Modifying testing/remotecppunittests.py triggers cppunit selection. - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["testing/remotecppunittests.py"] - ) == ["test-linux64/opt-cppunit"] + assert find.tasks(known_tasks, ["testing/remotecppunittests.py"]) == [ + "test-linux64/opt-cppunit" + ] # Modifying testing/runcppunittests.py triggers cppunit selection. - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["testing/runcppunittests.py"] - ) == ["test-linux64/opt-cppunit"] + assert find.tasks(known_tasks, ["testing/runcppunittests.py"]) == [ + "test-linux64/opt-cppunit" + ] # A .cpp file whose stem is listed in cppunittest.toml triggers cppunit selection. - assert test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["mfbt/TestArrayUtils.cpp"] - ) == ["test-linux64/opt-cppunit"] + assert find.tasks(known_tasks, ["mfbt/TestArrayUtils.cpp"]) == [ + "test-linux64/opt-cppunit" + ] # A .cpp file whose stem is not listed in cppunittest.toml does not trigger cppunit. - assert "test-linux64/opt-cppunit" not in test_scheduling.find_tasks_for_paths( - str(tmp_path), known_tasks, ["mfbt/TestUnknown.cpp"] + assert "test-linux64/opt-cppunit" not in find.tasks( + known_tasks, ["mfbt/TestUnknown.cpp"] ) # Empty known_tasks -> always empty. - assert ( - test_scheduling.find_tasks_for_paths(str(tmp_path), (), ["test_foo.cpp"]) == [] - ) + assert find.tasks((), ["test_foo.cpp"]) == [] def test_get_suite_test_dirs(tmp_path) -> None: From 29bb0c897e999808ee153669a8cea59cf5a449d6 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Tue, 17 Nov 2026 00:35:22 +0100 Subject: [PATCH 2/2] Simplify reading the files at the pushed revision Run all the Mercurial commands with subprocess, so the functions don't need to manage a command server, and build the test manifests map in find_manifests_for_paths as before, to keep the diff with the original code small. --- bugbug/test_scheduling.py | 272 +++++++++++++++----------------------- 1 file changed, 108 insertions(+), 164 deletions(-) diff --git a/bugbug/test_scheduling.py b/bugbug/test_scheduling.py index d95058e4e01..8ffb1627a47 100644 --- a/bugbug/test_scheduling.py +++ b/bugbug/test_scheduling.py @@ -36,7 +36,6 @@ cast, ) -import hglib import orjson from tqdm import tqdm @@ -1467,9 +1466,8 @@ def __init__( """ self.repo_dir = repo_dir self.rev = rev - self._hg: hglib.client.hgclient | None = None # The parsed test manifests (None when not a test manifest), by path. - self.parsed_manifests: dict[str, dict | None] | None = None + self.parsed_manifests: dict[str, dict | None] = {} # The test manifests by the paths they include. self.manifest_by_path: dict[str, set[str]] | None = None # The files changed from the base revision (None if all of them are new). @@ -1490,12 +1488,11 @@ def __init__( files.discard(path) else: files.add(path) - if base.parsed_manifests is not None: - self.parsed_manifests = { - path: data - for path, data in base.parsed_manifests.items() - if path not in self.changed - } + self.parsed_manifests = { + path: data + for path, data in base.parsed_manifests.items() + if path not in self.changed + } self._files_set: set[str] = files self.files = sorted(files) @@ -1510,36 +1507,20 @@ def __init__( parent = os.path.dirname(parent) self.dirs = sorted(self._dirs_set) - def _run(self, *args: str) -> list[str]: - """Run a Mercurial command printing NUL-separated paths, relative to the root.""" + def _hg(self, *args: str) -> bytes: # Not using the command server, as it is slow to read a large output. - out = subprocess.run( - ["hg", *args], - cwd=self.repo_dir, - check=True, - capture_output=True, + return subprocess.run( + ["hg", *args], cwd=self.repo_dir, check=True, capture_output=True ).stdout + + def _run(self, *args: str) -> list[str]: + """Run a Mercurial command printing NUL-separated paths, relative to the root.""" return [ entry.decode("utf-8", "surrogateescape") - for entry in out.split(b"\0") + for entry in self._hg(*args).split(b"\0") if entry ] - def _command(self, *args, **kwargs) -> bytes: - # The command server is kept open until the end of the "with" block using these files, to - # avoid starting Mercurial for each command. - if self._hg is None: - self._hg = hglib.open(self.repo_dir) - return self._hg.rawcommand(hglib.util.cmdbuilder(*args, **kwargs)) - - def __enter__(self) -> "_RevisionFiles": - return self - - def __exit__(self, *exc_info) -> None: - if self._hg is not None: - self._hg.close() - self._hg = None - @staticmethod def _under(sorted_paths: list[str], path: str) -> list[str]: """The paths in a sorted list which are in the given directory.""" @@ -1560,11 +1541,13 @@ def read(self, paths: list[str]) -> dict[str, bytes]: paths = [path for path in paths if path in self._files_set] contents: dict[str, bytes] = {} for i in range(0, len(paths), 1000): - out = self._command( - b"cat", - *(f"path:{path}".encode("utf-8") for path in paths[i : i + 1000]), - rev=self.rev.encode("ascii"), - template=b"json", + out = self._hg( + "cat", + "-r", + self.rev, + "-T", + "json", + *(f"path:{path}" for path in paths[i : i + 1000]), ) for entry in orjson.loads(out): contents[entry["path"]] = entry["data"].encode( @@ -1615,145 +1598,111 @@ def _get_repo_files(repo_dir: str, rev: str) -> _RevisionFiles: _revision_files: _RevisionFiles | None = None -def _parse_test_manifest(path: str, content: bytes) -> dict | None: - """Parse a test manifest, returning None if the file is not a test manifest.""" - try: - data = tomllib.loads(content.decode("utf-8")) - except UnicodeDecodeError, tomllib.TOMLDecodeError: - logger.warning("Failed to parse %s", path) - return None - - # HACK: If there is no "DEFAULT" key and there is no key that starts with "test", this is unlikely a test manifest. - if "DEFAULT" not in data and not any(key.startswith("test") for key in data): - return None - - return data - - -def _is_candidate_manifest(path: str) -> bool: - parts = path.split("/") - # HACK: These are not test manifests, skip them. - return not ( - parts[-1] in ("Cargo.toml", "pyproject.toml") - or (len(parts) > 1 and parts[-2] == "test-manifest-toml") - or "third_party" in parts - or "manifestparser" in parts - ) - +def find_manifests_for_paths(repo_dir: str, paths: list[str], rev: str) -> set[str]: + """The test manifests and web-platform test directories to run for modified paths. -def _normalize(path: str) -> str | None: - """Normalize a path relative to the root of the repository, None if outside it.""" - path = os.path.normpath(path) - if path == ".." or path.startswith("../"): - return None - return path + The files of the repository at revision `rev` are used, read with Mercurial without + updating the working directory, as it is slow on a large repository. + """ + repo_files = _get_repo_files(repo_dir, rev) + manifest_by_path = repo_files.manifest_by_path + if manifest_by_path is None: + manifest_by_path = repo_files.manifest_by_path = collections.defaultdict(set) -def _build_manifest_by_path( - files: _RevisionFiles, manifests: dict[str, dict] -) -> dict[str, set[str]]: - result: dict[str, set[str]] = collections.defaultdict(set) + toml_paths = [] + for toml_rel in repo_files.toml_files(): + parts = toml_rel.split("/") + # HACK: These are not test manifests, skip them. + if ( + parts[-1] in ("Cargo.toml", "pyproject.toml") + or (len(parts) > 1 and parts[-2] == "test-manifest-toml") + or "third_party" in parts + or "manifestparser" in parts + ): + continue - def add(path: str | None, toml_rel: str) -> None: - if path is not None: - result[path].add(toml_rel) + toml_paths.append(toml_rel) - for toml_rel, data in manifests.items(): - toml_dir = os.path.dirname(toml_rel) + # Only parse the manifests which changed since the revision the files were computed from. + to_parse = [p for p in toml_paths if p not in repo_files.parsed_manifests] + for toml_rel, content in repo_files.read(to_parse).items(): + try: + data = tomllib.loads(content.decode("utf-8")) + except UnicodeDecodeError, tomllib.TOMLDecodeError: + logger.warning("Failed to parse %s", toml_rel) + data = None + + # HACK: If there is no "DEFAULT" key and there is no key that starts with "test", this is unlikely a test manifest. + if data is not None and ( + "DEFAULT" not in data + and not any(key.startswith("test") for key in data) + ): + data = None - # The manifest path itself, so we schedule it when it is touched. - add(toml_rel, toml_rel) + repo_files.parsed_manifests[toml_rel] = data - # Collect head files. - head_value = data.get("DEFAULT", {}).get("head", []) - head_files = ( - head_value if isinstance(head_value, list) else head_value.split(" ") - ) - for head_file in head_files: - if not head_file.strip(): + for toml_rel in toml_paths: + data = repo_files.parsed_manifests.get(toml_rel) + if data is None: continue - add(_normalize(os.path.join(toml_dir, head_file)), toml_rel) + toml_dir = os.path.dirname(toml_rel) - # Collect support files. - def collect_support_files(value, toml_dir=toml_dir, toml_rel=toml_rel): - support_files = value.get("support-files", []) - if isinstance(support_files, str): - support_files = [support_files] + # The manifest path itself, so we schedule it when it is touched. + manifest_by_path[toml_rel].add(toml_rel) - for support_file in support_files: - if not support_file.strip(): - continue - - if support_file.startswith("!"): - support_file = support_file[1:] - - if support_file.startswith("/"): - support_file_path = _normalize(support_file[1:]) - else: - support_file_path = _normalize(os.path.join(toml_dir, support_file)) - - if support_file_path is None: + # Collect head files. + head_value = data.get("DEFAULT", {}).get("head", []) + head_files = ( + head_value if isinstance(head_value, list) else head_value.split(" ") + ) + for head_file in head_files: + if not head_file.strip(): continue - if "*" in support_file: - for f in files.glob(support_file_path): - add(f, toml_rel) - else: - add(support_file_path, toml_rel) - - collect_support_files(data.get("DEFAULT", {})) - - # Collect test files. - for key, val in data.items(): - if key != "DEFAULT" and isinstance(val, dict): - collect_support_files(val) + manifest_by_path[ + os.path.normpath(os.path.join(toml_dir, head_file)) + ].add(toml_rel) - add(_normalize(os.path.join(toml_dir, key)), toml_rel) - - return result + # Collect support files. + def collect_support_files(value, toml_dir=toml_dir, toml_rel=toml_rel): + support_files = value.get("support-files", []) + if isinstance(support_files, str): + support_files = [support_files] + for support_file in support_files: + if not support_file.strip(): + continue -def _get_manifest_by_path(files: _RevisionFiles) -> dict[str, set[str]]: - if files.manifest_by_path is not None: - return files.manifest_by_path - - candidates = [p for p in files.toml_files() if _is_candidate_manifest(p)] - # Only read the manifests which changed since the revision the files were computed from. - if files.parsed_manifests is None: - files.parsed_manifests = {} - to_read = [p for p in candidates if p not in files.parsed_manifests] - contents = files.read(to_read) - for path in to_read: - content_or_none = contents.get(path) - files.parsed_manifests[path] = ( - _parse_test_manifest(path, content_or_none) - if content_or_none is not None - else None - ) + if support_file.startswith("!"): + support_file = support_file[1:] - manifests = { - path: data - for path in candidates - if (data := files.parsed_manifests[path]) is not None - } + if support_file.startswith("/"): + support_file_path = os.path.normpath(support_file[1:]) + else: + support_file_path = os.path.normpath( + os.path.join(toml_dir, support_file) + ) - files.manifest_by_path = _build_manifest_by_path(files, manifests) - return files.manifest_by_path + if "*" in support_file: + files = repo_files.glob(support_file_path) + else: + files = [support_file_path] + for f in files: + manifest_by_path[f].add(toml_rel) -def find_manifests_for_paths(repo_dir: str, paths: list[str], rev: str) -> set[str]: - """The test manifests and web-platform test directories to run for modified paths. - - The files of the repository at revision `rev` are used, read with Mercurial without - updating the working directory, as it is slow on a large repository. - """ - with _get_repo_files(repo_dir, rev) as files: - return _find_manifests_for_paths(files, paths) + collect_support_files(data.get("DEFAULT", {})) + # Collect test files. + for key, val in data.items(): + if key != "DEFAULT" and isinstance(val, dict): + collect_support_files(val) -def _find_manifests_for_paths(files: _RevisionFiles, paths: list[str]) -> set[str]: - manifest_by_path = _get_manifest_by_path(files) + manifest_by_path[os.path.normpath(os.path.join(toml_dir, key))].add( + toml_rel + ) manifests = set() @@ -1768,9 +1717,9 @@ def _find_manifests_for_paths(files: _RevisionFiles, paths: list[str]) -> set[st elif parent: # Find manifests that are in test subfolders close to a modified file (e.g. if dom/battery/BatteryManager.cpp is modified, we should run dom/battery/test/chrome.toml and dom/battery/test/mochitest.toml). sibling_manifests: set[str] = set() - for sibling in files.subdirs(parent): + for sibling in repo_files.subdirs(parent): if repository.is_test(f"/{sibling}/"): - sibling_manifests.update(files.files_in(sibling, ".toml")) + sibling_manifests.update(repo_files.files_in(sibling, ".toml")) if len(sibling_manifests) <= MAX_SIBLING_MANIFESTS: manifests.update(sibling_manifests) @@ -1792,8 +1741,8 @@ def is_wpt_file(name: str) -> bool: return name.endswith((".html", ".any.js", ".worker.js")) def has_wpt_files(d: Path) -> bool: - return files.is_dir(str(d)) and any( - is_wpt_file(name) for name in files.children(str(d)) + return repo_files.is_dir(str(d)) and any( + is_wpt_file(name) for name in repo_files.children(str(d)) ) while cur_dir != test_root and not has_wpt_files(cur_dir): @@ -1839,13 +1788,8 @@ def find_tasks_for_paths( The files of the repository at revision `rev` are used, read with Mercurial without updating the working directory, as it is slow on a large repository. """ - with _get_repo_files(repo_dir, rev) as files: - return _find_tasks_for_paths(files, known_tasks, paths) + files = _get_repo_files(repo_dir, rev) - -def _find_tasks_for_paths( - files: _RevisionFiles, known_tasks: tuple[str, ...], paths: list[str] -) -> list[str]: select_gtest = False select_cppunit = False select_rusttests = False