From f2d6fac64bd0c8b1a54d55d15a8166f8ec093c24 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:29:43 +0800 Subject: [PATCH] fix(ci): preserve pytest deadline diagnostics and abort slow collection (#8787) Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .github/workflows/python-tests.yml | 26 +- scripts/ci/pytest_timing.py | 396 ++++++++++++++++++++++++++++- tests/test_ci_pytest_timing.py | 245 ++++++++++++++++++ tests/test_python_ci_workflow.py | 24 +- 4 files changed, 679 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index dc7b2ed781..f38b784203 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -615,7 +615,9 @@ jobs: cancel-in-progress: false # All shards were still progressing at 83-94% when the 45m job # deadline cancelled run 37916979857. Reserve setup/reporting headroom - # without changing test-level deadlines, collection or worker count. + # without changing test-level deadlines, collection or worker count; an inner + # 55m cooperative session deadline preserves partial JUnit and diagnostics + # before the 60m runner limit terminates the job. timeout-minutes: 60 strategy: fail-fast: false @@ -647,8 +649,10 @@ jobs: # Ubuntu 24.04 may need an executable-scoped userns profile for bwrap. # Do not disable AppArmor or the system-wide unprivileged-userns guard. run: | - sudo apt-get update - sudo apt-get install -y bubblewrap + if ! command -v bwrap >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends bubblewrap + fi if ! bwrap --unshare-all --ro-bind / / /bin/true; then sudo tee /etc/apparmor.d/loopx-qualification-bwrap >/dev/null <<'PROFILE' abi , @@ -663,12 +667,14 @@ jobs: - name: Run test shard # Split the whole collection, not a hand-maintained list of directories. # Without timing history least_duration alternates equal-weight tests. - # Each runner retains the measured two-worker pool. - # Cooperative per-test Python stacks diagnose stalls without terminating tests. + # Each runner retains the measured two-worker pool; cooperative stacks + # and the 55m inner deadline diagnose slow/blocked cases without masking failures. run: >- python -m pytest -v -n 2 -m "not stage2c_e2e" -p scripts.ci.pytest_timing --ci-stall-after 120 + --ci-session-deadline 3300 + --ci-diagnostics-file ci-shard-diagnostics.json --splits 6 --group ${{ matrix.shard }} --splitting-algorithm least_duration --durations=25 --durations-min=1 @@ -676,13 +682,21 @@ jobs: --cov=loopx --cov-report=term - name: Upload full-shard outcomes for selection audit - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: python-junit-${{ matrix.shard }} path: junit.xml if-no-files-found: error retention-days: 7 + - name: Upload shard timeout and progress diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: python-diagnostics-${{ matrix.shard }} + path: ci-shard-diagnostics.json + if-no-files-found: warn + retention-days: 7 - name: Upload shard coverage uses: actions/upload-artifact@v7 with: diff --git a/scripts/ci/pytest_timing.py b/scripts/ci/pytest_timing.py index 98698e8ff7..9173edde53 100644 --- a/scripts/ci/pytest_timing.py +++ b/scripts/ci/pytest_timing.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from collections import deque import json import hashlib import math @@ -15,6 +16,7 @@ import time import sys import threading +import _thread from collections.abc import Generator, Sequence from typing import Any @@ -26,10 +28,12 @@ PREFIX = "LOOPX_PYTEST_TIMING " +DEADLINE_PREFIX = "LOOPX_PYTEST_DEADLINE " _STACK_FD = pytest.StashKey[int]() # Tests may replace public subprocess helpers. The observer owns this transport; # subprocess.run would still resolve a test's replacement Popen at call time. _STACK_POPEN = subprocess.Popen +_PROC_POPEN = subprocess.Popen _STACK_WRITE_PROGRAM = ( "import os,sys,threading;" "deadline=threading.Timer(0.1,lambda:os._exit(0));" @@ -38,9 +42,99 @@ ) +def _sanitize_comm(raw_comm: str) -> str: + token = raw_comm.strip().split()[0] if raw_comm.strip() else "" + base = os.path.basename(token) + cleaned = re.sub(r"[^A-Za-z0-9_.+-]", "_", base) + return cleaned[:64] or "unknown" + + +def _collect_descendant_processes(root_pid: int, limit: int = 16) -> list[dict[str, Any]]: + """Collect bounded public-safe metadata (no argv/paths/env) for descendant processes.""" + if root_pid <= 0 or limit <= 0 or os.name == "nt": + return [] + children_by_ppid: dict[int, list[int]] = {} + meta: dict[int, dict[str, Any]] = {} + helper_pid = -1 + try: + if os.path.isdir("/proc"): + for entry in os.listdir("/proc"): + if not entry.isdigit(): + continue + try: + with open(f"/proc/{entry}/stat", encoding="utf-8", errors="replace") as handle: + stat_text = handle.read().strip() + except OSError: + continue + rparen = stat_text.rfind(")") + lparen = stat_text.find("(") + if lparen < 0 or rparen <= lparen: + continue + pid_part = stat_text[:lparen].strip() + comm_raw = stat_text[lparen + 1:rparen] + rest = stat_text[rparen + 1:].split() + if not pid_part.isdigit() or len(rest) < 2 or not rest[1].lstrip("-").isdigit(): + continue + pid = int(pid_part) + state = re.sub(r"[^A-Za-z0-9+]", "", rest[0])[:16] or "?" + ppid = int(rest[1]) + meta[pid] = {"pid": pid, "ppid": ppid, "state": state, "comm": _sanitize_comm(comm_raw)} + children_by_ppid.setdefault(ppid, []).append(pid) + else: + proc = _PROC_POPEN( + ["ps", "-eo", "pid=,ppid=,stat=,comm="], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + helper_pid = proc.pid + try: + stdout_bytes, _ = proc.communicate(timeout=0.3) + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=0.2) + for line in stdout_bytes.decode("utf-8", errors="replace").splitlines(): + parts = line.split(None, 3) + if len(parts) != 4: + continue + pid_str, ppid_str, state_raw, comm_raw = parts + if not (pid_str.isdigit() and ppid_str.lstrip("-").isdigit()): + continue + pid = int(pid_str) + if pid == helper_pid: + continue + ppid = int(ppid_str) + state = re.sub(r"[^A-Za-z0-9+]", "", state_raw)[:16] or "?" + meta[pid] = {"pid": pid, "ppid": ppid, "state": state, "comm": _sanitize_comm(comm_raw)} + children_by_ppid.setdefault(ppid, []).append(pid) + except (OSError, subprocess.SubprocessError, ValueError): + return [] + + descendants: list[dict[str, Any]] = [] + seen = {root_pid} + queue: deque[int] = deque(sorted(children_by_ppid.get(root_pid, ()))) + while queue and len(descendants) < limit: + current = queue.popleft() + if current in seen or current == helper_pid: + continue + seen.add(current) + row = meta.get(current) + if row is not None: + descendants.append(row) + for child_pid in sorted(children_by_ppid.get(current, ())): + if child_pid not in seen: + queue.append(child_pid) + return descendants + + def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption("--ci-stall-after", type=float, default=0, help="Print the executing test thread's Python stack after this many seconds; never stop the test.") + parser.addoption("--ci-session-deadline", type=float, default=0, + help="Cooperatively interrupt the session after this many seconds so partial JUnit and diagnostics flush before runner timeout.") + parser.addoption("--ci-diagnostics-file", type=str, default="", + help="Write bounded live and final JSON progress/timeout diagnostics on the controller.") @pytest.hookimpl(wrapper=True, trylast=True) @@ -60,6 +154,12 @@ def dump() -> None: rows.append(f" {frame.f_code.co_filename}:{frame.f_lineno} in {frame.f_code.co_name}") frame = frame.f_back del frame + if not finished.is_set() and (os.path.isdir("/proc") or delay >= 0.05): + for child in _collect_descendant_processes(os.getpid(), limit=8): + rows.append( + f" [child pid={child['pid']} ppid={child['ppid']} " + f"state={child['state']} comm={child['comm']}]" + ) if not finished.is_set(): try: # A duplicate shares stderr's flags: never change them or @@ -112,6 +212,9 @@ def pytest_configure(config: pytest.Config) -> None: delay = config.getoption("ci_stall_after") if not math.isfinite(delay) or delay < 0: raise pytest.UsageError("--ci-stall-after must be finite and nonnegative") + deadline = config.getoption("ci_session_deadline") + if not math.isfinite(deadline) or deadline < 0: + raise pytest.UsageError("--ci-session-deadline must be finite and nonnegative") if delay: # Capture the original descriptor before per-test fd capture starts, # so a blocked worker's diagnostic is visible before it returns. @@ -144,9 +247,172 @@ def __init__(self, config: pytest.Config) -> None: self.config = config self.started = time.perf_counter() self.collection: tuple[str, ...] | None = None + getoption = getattr(config, "getoption", None) + raw_deadline = getoption("ci_session_deadline") if callable(getoption) else 0.0 + raw_diag = getoption("ci_diagnostics_file") if callable(getoption) else "" + self.deadline_seconds = float(raw_deadline or 0.0) + self.diagnostics_path = str(raw_diag or "").strip() + self._lock = threading.Lock() + self._finished = False + self._collection_active = False + self._loop_active = False + self._runtestloop_finished = False + self._deadline_timer: threading.Timer | None = None + self.deadline_exceeded = False + self.deadline_elapsed_seconds: float | None = None + self.deadline_in_flight: list[dict[str, Any]] = [] + self.deadline_children: list[dict[str, Any]] = [] + self.in_flight: dict[str, dict[str, Any]] = {} + self.cases: dict[str, dict[str, Any]] = {} + self.phase_seconds = {"setup": 0.0, "call": 0.0, "teardown": 0.0} + self.finished_cases = 0 + self.invalid_duration_count = 0 + self._last_diag_flush_elapsed = 0.0 + self._last_diag_flush_finished = 0 + + @staticmethod + def _case_identity(nodeid: str) -> tuple[str, str]: + test_id = hashlib.sha256(nodeid.encode("utf-8", errors="surrogatepass")).hexdigest() + display = nodeid.partition("[")[0][:256] + return test_id, display + + def _elapsed(self) -> float: + return round(time.perf_counter() - self.started, 6) + + def _in_flight_snapshot_locked(self, now_elapsed: float) -> list[dict[str, Any]]: + rows = [ + { + "test_id": test_id, + "display_nodeid": entry["display_nodeid"], + "started_elapsed_seconds": entry["started_elapsed_seconds"], + "running_seconds": round(max(0.0, now_elapsed - entry["started_elapsed_seconds"]), 6), + } + for test_id, entry in self.in_flight.items() + ] + rows.sort(key=lambda row: (-row["running_seconds"], row["test_id"])) + return rows[:16] + + def _build_diagnostics_locked( + self, + *, + execution_complete: bool, + exit_code: int | None, + children: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + now_elapsed = self._elapsed() + longest = sorted( + ( + { + "test_id": test_id, + "display_nodeid": entry["display_nodeid"], + "measured_seconds": round(entry["measured_seconds"], 6), + "finished": entry["finished"], + } + for test_id, entry in self.cases.items() + ), + key=lambda row: (-row["measured_seconds"], row["test_id"]), + )[:25] + in_flight_rows = ( + list(self.deadline_in_flight) + if self.deadline_exceeded and self.deadline_in_flight + else self._in_flight_snapshot_locked(now_elapsed) + ) + active_children = children if children is not None else self.deadline_children + return { + "schema_version": "loopx_ci_pytest_diagnostics_v1", + "execution_complete": execution_complete, + "deadline_exceeded": self.deadline_exceeded, + "deadline_seconds": self.deadline_seconds, + "deadline_elapsed_seconds": self.deadline_elapsed_seconds, + "pytest_exit_code": exit_code, + "selected_cases": len(self.collection) if self.collection is not None else None, + "observed_cases": len(self.cases), + "finished_cases": self.finished_cases, + "in_flight_count": len(self.in_flight), + "in_flight_cases": in_flight_rows, + "invalid_duration_count": self.invalid_duration_count, + "observed_elapsed_seconds": now_elapsed, + "measured_worker_seconds": round(sum(self.phase_seconds.values()), 6), + "phase_seconds": {key: round(value, 6) for key, value in self.phase_seconds.items()}, + "longest_cases": longest, + "active_child_processes": list(active_children[:16]), + } + + def _write_diagnostics( + self, + *, + execution_complete: bool = False, + exit_code: int | None = None, + children: list[dict[str, Any]] | None = None, + ) -> None: + if not self.diagnostics_path: + return + with self._lock: + payload = self._build_diagnostics_locked( + execution_complete=execution_complete, + exit_code=exit_code, + children=children, + ) + self._last_diag_flush_elapsed = payload["observed_elapsed_seconds"] + self._last_diag_flush_finished = self.finished_cases + try: + parent = os.path.dirname(os.path.abspath(self.diagnostics_path)) + if parent: + os.makedirs(parent, exist_ok=True) + tmp_path = f"{self.diagnostics_path}.{os.getpid()}.tmp" + with open(tmp_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=True, allow_nan=False, indent=2) + handle.write("\n") + os.replace(tmp_path, self.diagnostics_path) + except OSError: + # Optional diagnostics cannot mask test outcomes. + pass + + def _record_deadline(self, session: pytest.Session, *, interrupt: bool) -> None: + with self._lock: + if self._finished or self._runtestloop_finished or self.deadline_exceeded: + return + children = _collect_descendant_processes(os.getpid(), limit=16) + reason = f"CI session deadline exceeded ({self.deadline_seconds:g}s)" + with self._lock: + if self._finished or self._runtestloop_finished or self.deadline_exceeded: + return + now_elapsed = self._elapsed() + self.deadline_exceeded = True + self.deadline_elapsed_seconds = now_elapsed + self.deadline_in_flight = self._in_flight_snapshot_locked(now_elapsed) + self.deadline_children = children + session.shouldstop = reason + dsession = self.config.pluginmanager.get_plugin("dsession") + if dsession is not None and hasattr(dsession, "shuttingdown") and not getattr(dsession, "shouldstop", False): + dsession.shouldstop = reason + self._write_diagnostics(execution_complete=False, exit_code=None, children=children) + if interrupt: + with self._lock: + if not self._finished and (self._collection_active or self._loop_active): + # Wake collection or pytest_runtestloop immediately even when blocked in IO/sleep. + _thread.interrupt_main() + + def _on_deadline(self, session: pytest.Session) -> None: + self._record_deadline(session, interrupt=True) + + def _abort_if_deadline_exceeded(self, session: pytest.Session) -> None: + if self.deadline_seconds <= 0: + return + with self._lock: + exceeded = self.deadline_exceeded or self._elapsed() >= self.deadline_seconds + if not exceeded: + return + self._record_deadline(session, interrupt=False) + reason = ( + session.shouldstop + if isinstance(session.shouldstop, str) and session.shouldstop + else f"CI session deadline exceeded ({self.deadline_seconds:g}s)" + ) + raise pytest.Session.Interrupted(reason) def emit(self, *, _prefix: str = PREFIX, **row: Any) -> None: - row["elapsed_seconds"] = round(time.perf_counter() - self.started, 6) + row["elapsed_seconds"] = self._elapsed() line = _prefix + json.dumps(row, ensure_ascii=True, allow_nan=False, separators=(",", ":")) reporter = self.config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: @@ -161,11 +427,50 @@ def emit(self, *, _prefix: str = PREFIX, **row: Any) -> None: @pytest.hookimpl(trylast=True) def pytest_sessionstart(self, session: pytest.Session) -> None: self.emit(event="start", version=1) + if self.deadline_seconds > 0: + remaining = max(0.0, self.deadline_seconds - self._elapsed()) + timer = threading.Timer(remaining, self._on_deadline, args=(session,)) + timer.daemon = True + with self._lock: + self._deadline_timer = timer + timer.start() + + @pytest.hookimpl(wrapper=True, tryfirst=True) + def pytest_collection(self, session: pytest.Session) -> Generator[None, object, object]: + if self.deadline_seconds <= 0: + return (yield) + self._abort_if_deadline_exceeded(session) + with self._lock: + self._collection_active = True + try: + return (yield) + finally: + with self._lock: + self._collection_active = False + + @pytest.hookimpl(wrapper=True, tryfirst=True) + def pytest_runtestloop(self, session: pytest.Session) -> Generator[None, object, object]: + if self.deadline_seconds <= 0: + return (yield) + self._abort_if_deadline_exceeded(session) + with self._lock: + self._loop_active = True + completed_normally = False + try: + result = yield + completed_normally = True + return result + finally: + with self._lock: + self._loop_active = False + if completed_normally and not self.deadline_exceeded: + self._runtestloop_finished = True def record_collection(self, ids: Sequence[str]) -> None: if self.collection is None: self.collection = tuple(ids) self.emit(event="collection", selected=len(ids)) + self._write_diagnostics(execution_complete=False, exit_code=None) elif self.collection != tuple(ids): # xdist owns rejecting different worker collections. Preserve the # observation without changing that decision or claiming coverage. @@ -179,9 +484,22 @@ def pytest_collection_finish(self, session: pytest.Session) -> None: if not getattr(self.config.option, "numprocesses", None): self.record_collection([item.nodeid for item in session.items]) + def pytest_runtest_logstart(self, nodeid: str, location: tuple[str, int | None, str]) -> None: + test_id, display = self._case_identity(nodeid) + with self._lock: + if test_id not in self.in_flight: + self.in_flight[test_id] = { + "display_nodeid": display, + "started_elapsed_seconds": self._elapsed(), + } + + def pytest_runtest_logfinish(self, nodeid: str, location: tuple[str, int | None, str]) -> None: + test_id, _ = self._case_identity(nodeid) + with self._lock: + self.in_flight.pop(test_id, None) + def pytest_runtest_logreport(self, report: Any) -> None: - test_id = hashlib.sha256(report.nodeid.encode("utf-8", errors="surrogatepass")).hexdigest() - display = report.nodeid.partition("[")[0][:256] + test_id, display = self._case_identity(report.nodeid) if report.failed: try: self.emit(_prefix=FAILURE_PREFIX, version=1, test_id=test_id, @@ -198,13 +516,81 @@ def pytest_runtest_logreport(self, report: Any) -> None: if isinstance(report, SubtestReport): return duration = report.duration + valid_duration = duration if math.isfinite(duration) and duration >= 0 else None + should_flush_diag = False + now_elapsed = self._elapsed() + with self._lock: + if report.when == "setup" and test_id not in self.in_flight: + self.in_flight[test_id] = { + "display_nodeid": display, + "started_elapsed_seconds": now_elapsed, + } + entry = self.cases.get(test_id) + if entry is None: + entry = {"display_nodeid": display, "phases": {}, "measured_seconds": 0.0, "finished": False} + self.cases[test_id] = entry + if report.when in self.phase_seconds and report.when not in entry["phases"]: + entry["phases"][report.when] = report.outcome + if valid_duration is None: + self.invalid_duration_count += 1 + else: + entry["measured_seconds"] += valid_duration + self.phase_seconds[report.when] += valid_duration + if report.when == "teardown": + self.in_flight.pop(test_id, None) + if not entry["finished"] and ("call" in entry["phases"] or entry["phases"].get("setup") != "passed"): + entry["finished"] = True + self.finished_cases += 1 + if self.diagnostics_path and ( + self.finished_cases - self._last_diag_flush_finished >= 100 + or now_elapsed - self._last_diag_flush_elapsed >= 15.0 + ): + should_flush_diag = True + # Parameter payloads can be huge or trigger hosted secret masking, # corrupting JSON. Hash full identity; display only the parent name. self.emit(event="phase", test_id=test_id, display_nodeid=display, phase=report.when, outcome=report.outcome, - seconds=duration if math.isfinite(duration) and duration >= 0 else None) + seconds=valid_duration) + if should_flush_diag: + self._write_diagnostics(execution_complete=False, exit_code=None) @pytest.hookimpl(trylast=True) def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None: - self.emit(event="end", exit_code=int(session.exitstatus)) + timer: threading.Timer | None = None + with self._lock: + self._finished = True + self._collection_active = False + self._loop_active = False + timer = self._deadline_timer + self._deadline_timer = None + if timer is not None: + timer.cancel() + timer.join() + + if self.deadline_exceeded: + session.exitstatus = pytest.ExitCode.INTERRUPTED + self.emit( + _prefix=DEADLINE_PREFIX, + version=1, + deadline_seconds=self.deadline_seconds, + deadline_elapsed_seconds=self.deadline_elapsed_seconds, + selected_cases=len(self.collection) if self.collection is not None else None, + finished_cases=self.finished_cases, + in_flight_cases=self.deadline_in_flight, + active_child_processes=self.deadline_children, + ) + + final_exit = int(session.exitstatus) + selected = len(self.collection) if self.collection is not None else None + complete = ( + not self.deadline_exceeded + and final_exit in (0, 1) + and selected is not None + and selected > 0 + and self.finished_cases == selected + and self.invalid_duration_count == 0 + ) + self.emit(event="end", exit_code=final_exit) + self._write_diagnostics(execution_complete=complete, exit_code=final_exit) diff --git a/tests/test_ci_pytest_timing.py b/tests/test_ci_pytest_timing.py index 7d934aa65a..3e4959752f 100644 --- a/tests/test_ci_pytest_timing.py +++ b/tests/test_ci_pytest_timing.py @@ -9,6 +9,7 @@ import subprocess import sys import time +import xml.etree.ElementTree as ET import pytest @@ -575,3 +576,247 @@ def test_real_oversized_parameter_ids_keep_distinct_identity_and_bounded_display assert all("Authorization" not in line and "$TOKEN" not in line for line in log.read_text().splitlines() if line.startswith(PREFIX)) assert max(len(line) for line in log.read_text().splitlines() if line.startswith(PREFIX)) < 1200 + + +@pytest.mark.parametrize("deadline", ["-1", "nan", "inf", "-inf"]) +def test_session_deadline_rejects_invalid_values_before_running_tests(tmp_path, deadline): + (tmp_path / "test_subject.py").write_text( + "from pathlib import Path\n" + "def test_unreachable(): Path('ran').touch()\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + sys.executable, "-m", "pytest", "-q", + "-p", "scripts.ci.pytest_timing", + "--ci-session-deadline=" + deadline, + ], + cwd=tmp_path, + env=environment(), + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == 4 + assert "--ci-session-deadline must be finite and nonnegative" in result.stderr + assert not (tmp_path / "ran").exists() + + +@pytest.mark.parametrize("workers,deadline", [(0, "0.6"), (2, "2.5")]) +def test_session_deadline_flushes_partial_junit_and_in_flight_diagnostics( + tmp_path, workers, deadline, +): + (tmp_path / "test_subject.py").write_text( + "import time\nfrom pathlib import Path\n" + "def test_fast():\n" + " Path('fast.done').touch()\n" + "def test_blocked():\n" + " waited = 0.0\n" + " while not Path('fast.done').exists() and waited < 5.0:\n" + " time.sleep(0.02)\n" + " waited += 0.02\n" + " Path('blocked.entered').touch()\n" + " time.sleep(25)\n", + encoding="utf-8", + ) + log = tmp_path / "deadline.log" + diag_path = tmp_path / "ci-shard-diagnostics.json" + junit_path = tmp_path / "junit.xml" + with log.open("w") as output: + result = subprocess.run( + [ + sys.executable, "-m", "pytest", "-v", "-n", str(workers), + "-p", "scripts.ci.pytest_timing", + "--ci-session-deadline", deadline, + "--ci-diagnostics-file", str(diag_path), + "--junitxml", str(junit_path), + ], + cwd=tmp_path, + env=environment(), + stdout=output, + stderr=subprocess.STDOUT, + timeout=30, + check=False, + ) + assert result.returncode == 2, log.read_text() + assert (tmp_path / "blocked.entered").exists(), log.read_text() + assert junit_path.is_file(), "partial JUnit must be flushed on cooperative deadline" + cases = { + name + for case in ET.parse(junit_path).findall(".//testcase") + if (name := case.attrib.get("name")) + } + assert "test_fast" in cases + diag = json.loads(diag_path.read_text(encoding="utf-8")) + assert diag["schema_version"] == "loopx_ci_pytest_diagnostics_v1" + assert diag["deadline_exceeded"] is True + assert diag["execution_complete"] is False + assert diag["pytest_exit_code"] == 2 + assert diag["selected_cases"] == 2 + assert diag["finished_cases"] == 1 + assert any(row["display_nodeid"] == "test_subject.py::test_blocked" for row in diag["in_flight_cases"]) + log_text = log.read_text() + assert "LOOPX_PYTEST_DEADLINE " in log_text + report = read_log(log) + assert report["execution_complete"] is False + assert report["pytest_exit_code"] == 2 + assert report["selected_cases"] == 2 + assert report["finished_cases"] == 1 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX descendant ps inspection") +def test_stall_snapshot_reports_descendant_child_process_basename_without_argv(tmp_path): + (tmp_path / "test_subject.py").write_text( + "import subprocess, sys, time\nfrom pathlib import Path\n" + "def test_wait_child():\n" + " proc = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(20)'])\n" + " try:\n" + " Path('entered').touch()\n" + " while not Path('release').exists():\n" + " time.sleep(0.01)\n" + " finally:\n" + " proc.kill()\n" + " proc.wait(timeout=5)\n", + encoding="utf-8", + ) + log = tmp_path / "child-stall.log" + with log.open("w") as output: + process = subprocess.Popen( + [ + sys.executable, "-m", "pytest", "-q", "-n", "0", + "-p", "scripts.ci.pytest_timing", + "--ci-stall-after=0.1", + ], + cwd=tmp_path, + env=environment(), + stdout=output, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 10 + while True: + text = log.read_text() + if (tmp_path / "entered").exists() and "[child pid=" in text: + break + assert process.poll() is None, text + assert time.monotonic() < deadline, "Expected child process metadata in stall snapshot: " + text + time.sleep(0.02) + finally: + (tmp_path / "release").touch() + process.wait(timeout=10) + assert process.returncode == 0 + text = log.read_text() + assert " [child pid=" in text + assert "comm=" in text + assert "time.sleep(20)" not in text + + +@pytest.mark.parametrize("workers", [0, 2]) +def test_collection_exceeding_session_deadline_aborts_and_writes_diagnostics( + tmp_path: Path, + workers: int, +) -> None: + (tmp_path / "conftest.py").write_text( + "import time\n" + "def pytest_collection_modifyitems(items):\n" + " time.sleep(1.5)\n", + encoding="utf-8", + ) + (tmp_path / "test_subject.py").write_text( + "import time\nfrom pathlib import Path\n" + "def test_never_runs():\n" + " Path('ran').touch()\n" + " time.sleep(5.0)\n", + encoding="utf-8", + ) + log = tmp_path / "slow-collection.log" + diag_path = tmp_path / "ci-shard-diagnostics.json" + junit_path = tmp_path / "junit.xml" + started = time.monotonic() + with log.open("w") as output: + result = subprocess.run( + [ + sys.executable, "-m", "pytest", "-v", "-n", str(workers), + "-p", "scripts.ci.pytest_timing", + "--ci-session-deadline", "0.4", + "--ci-diagnostics-file", str(diag_path), + "--junitxml", str(junit_path), + ], + cwd=tmp_path, + env=environment(), + stdout=output, + stderr=subprocess.STDOUT, + timeout=20, + check=False, + ) + wall = time.monotonic() - started + log_text = log.read_text(encoding="utf-8") + assert result.returncode == 2, log_text + assert wall < 4.5, f"Expected early abort before 5s test, took {wall:.2f}s:\n{log_text}" + assert not (tmp_path / "ran").exists(), log_text + assert diag_path.is_file(), log_text + diag = json.loads(diag_path.read_text(encoding="utf-8")) + assert diag["schema_version"] == "loopx_ci_pytest_diagnostics_v1" + assert diag["deadline_exceeded"] is True + assert diag["execution_complete"] is False + assert diag["pytest_exit_code"] == 2 + assert diag["finished_cases"] == 0 + assert "LOOPX_PYTEST_DEADLINE " in log_text + assert junit_path.is_file(), log_text + cases = { + name + for case in ET.parse(junit_path).findall(".//testcase") + if (name := case.attrib.get("name")) + } + assert "test_never_runs" not in cases + report = read_log(log) + assert report["execution_complete"] is False + assert report["pytest_exit_code"] == 2 + + +def test_runtestloop_entry_aborts_when_session_deadline_already_elapsed( + tmp_path: Path, +) -> None: + from types import SimpleNamespace + from scripts.ci.pytest_timing import TimingObserver + + diag_path = tmp_path / "ci-shard-diagnostics.json" + dsession = SimpleNamespace(shuttingdown=False, shouldstop=False) + config = SimpleNamespace( + getoption=lambda key: 0.4 if key == "ci_session_deadline" else (str(diag_path) if key == "ci_diagnostics_file" else 0), + pluginmanager=SimpleNamespace(get_plugin=lambda name: dsession if name == "dsession" else None), + ) + session = SimpleNamespace(shouldstop=False) + observer = TimingObserver(config) # type: ignore[arg-type] + # Simulate deadline firing between collection and pytest_runtestloop (outside active hooks) + observer.started -= 1.0 + observer._on_deadline(session) # type: ignore[arg-type] + assert observer.deadline_exceeded is True + assert isinstance(session.shouldstop, str) and "0.4s" in session.shouldstop + assert dsession.shouldstop == session.shouldstop + assert diag_path.is_file() + diag = json.loads(diag_path.read_text(encoding="utf-8")) + assert diag["deadline_exceeded"] is True + + hook = observer.pytest_runtestloop(session) # type: ignore[arg-type] + with pytest.raises(pytest.Session.Interrupted): + next(hook) + + # Also cover entering pytest_runtestloop when elapsed >= deadline before timer fired. + diag_path2 = tmp_path / "ci-shard-diagnostics-2.json" + dsession2 = SimpleNamespace(shuttingdown=False, shouldstop=False) + config2 = SimpleNamespace( + getoption=lambda key: 0.4 if key == "ci_session_deadline" else (str(diag_path2) if key == "ci_diagnostics_file" else 0), + pluginmanager=SimpleNamespace(get_plugin=lambda name: dsession2 if name == "dsession" else None), + ) + session2 = SimpleNamespace(shouldstop=False) + observer2 = TimingObserver(config2) # type: ignore[arg-type] + observer2.started -= 1.0 + hook2 = observer2.pytest_runtestloop(session2) # type: ignore[arg-type] + with pytest.raises(pytest.Session.Interrupted): + next(hook2) + assert observer2.deadline_exceeded is True + assert dsession2.shouldstop == session2.shouldstop + assert diag_path2.is_file() + assert json.loads(diag_path2.read_text(encoding="utf-8"))["deadline_exceeded"] is True diff --git a/tests/test_python_ci_workflow.py b/tests/test_python_ci_workflow.py index ab82c69360..52fd13280e 100644 --- a/tests/test_python_ci_workflow.py +++ b/tests/test_python_ci_workflow.py @@ -444,8 +444,10 @@ def test_six_shards_execute_each_test_once_and_merge_portable_coverage( ) assert result.returncode == 0, result.stdout + result.stderr assert "LOOPX_PYTEST_STACK " in result.stdout + result.stderr + diag = json.loads((root / "ci-shard-diagnostics.json").read_text(encoding="utf-8")) + assert diag["execution_complete"] is True and diag["deadline_exceeded"] is False cases = ET.parse(root / "results.xml").findall(".//testcase") - seen.append({case.attrib["name"] for case in cases}) + seen.append({name for case in cases if (name := case.attrib.get("name"))}) partial = subprocess.run( [sys.executable, "-m", "coverage", "report", "--fail-under=100"], cwd=root, env=env, capture_output=True, check=False, @@ -543,6 +545,26 @@ def test_shard_names_failed_cases_before_the_remaining_tests_finish(tmp_path: Pa assert len([case for case in cases if case.find("failure") is not None]) == 1 +def test_python_shards_preserve_two_workers_and_deadline_diagnostics() -> None: + shard_job = yaml.safe_load(WORKFLOW)["jobs"]["test-shard"] + assert shard_job["timeout-minutes"] == 60 + steps = {step.get("name", f"idx-{idx}"): step for idx, step in enumerate(shard_job["steps"])} + isolation = steps["Prepare native shell isolation"]["run"] + assert "if ! command -v bwrap >/dev/null 2>&1; then" in isolation + run_args = shlex.split(steps["Run test shard"]["run"]) + assert run_args[run_args.index("-n") + 1] == "2" + assert run_args[run_args.index("--ci-session-deadline") + 1] == "3300" + assert run_args[run_args.index("--ci-diagnostics-file") + 1] == "ci-shard-diagnostics.json" + junit_upload = steps["Upload full-shard outcomes for selection audit"] + assert junit_upload["if"] == "${{ !cancelled() }}" + assert junit_upload["with"]["path"] == "junit.xml" + diag_upload = steps["Upload shard timeout and progress diagnostics"] + assert diag_upload["if"] == "always()" + assert diag_upload["with"]["name"] == "python-diagnostics-${{ matrix.shard }}" + assert diag_upload["with"]["path"] == "ci-shard-diagnostics.json" + assert diag_upload["with"]["if-no-files-found"] == "warn" + + def test_backend_and_mixed_prs_require_the_browser_qualified_artifact() -> None: producer = WORKFLOW.split(" chat-bundle:\n", 1)[1].split(" chat-bundle-browser:\n", 1)[0] assert "needs.changes.outputs.core_tests == 'true'" in producer