diff --git a/src/toad/client/connection.py b/src/toad/client/connection.py index ae7ed7c0d..b4b56e7ab 100644 --- a/src/toad/client/connection.py +++ b/src/toad/client/connection.py @@ -348,6 +348,10 @@ class ObservationRetry: LONGEST: ClassVar[float] = 30.0 losses: int = 0 due: float = 0.0 + # When the last result reached the UI (monotonic), so an outage can say how long the UI + # had gone without one: a relay that ends because its reader took nothing for its write + # deadline reads differently from a connector that went away. + last_arrival: float | None = None def lost(self, now: float) -> None: self.losses += 1 @@ -363,6 +367,17 @@ def waiting(self, now: float) -> bool: def recovered(self) -> None: self.losses, self.due = 0, 0.0 + def arrived(self, now: float) -> None: + """A result reached the UI: the outage, if any, is over.""" + self.recovered() + self.last_arrival = now + + def quiet_note(self, now: float) -> str: + """How long before ``now`` the UI last received a result, for the outage's report.""" + if self.last_arrival is None: + return "No observation result had reached the UI yet." + return f"The last observation result reached the UI {now - self.last_arrival:.1f} s before this." + @dataclass(frozen=True, kw_only=True) class Observation(ABC): @@ -774,7 +789,7 @@ def retire(self, access: CoordinationAccess): def receive(self, result: Observed): if self.is_current(): - self.access.retry.recovered() + self.access.retry.arrived(time.monotonic()) self.access.dispatch_sync(result, self) def tell(self, wanted) -> None: @@ -797,6 +812,7 @@ def receive_failed(self, error: Exception): if self.is_current(): self.failed(error, self.opened_selection()) if self.access.retry.began(): + error.add_note(self.access.retry.quiet_note(time.monotonic())) self.access.app.report_fatal(error) def views_changed(self, event=None, subscription=None): diff --git a/src/toad/fatal_trace.py b/src/toad/fatal_trace.py new file mode 100644 index 000000000..8a94f376d --- /dev/null +++ b/src/toad/fatal_trace.py @@ -0,0 +1,52 @@ +"""A native crash's Python stacks: every thread's, written beside the saved tracebacks. + +A fault in C (a segfault in the interpreter, PySide or a C extension) kills the process +before any Python handler runs, so nothing reaches `paths.save_error`. Python's +`faulthandler` writes each thread's stack from the signal handler itself, into a file that +must already be open. This owner opens that file at a start, before anything native runs, +keeps it open for the life of the process, and removes it at a clean exit when nothing was +written, so the errors folder holds a `fatal_*` file only for a run that crashed. +""" + +from __future__ import annotations + +import atexit +import faulthandler +import os +from pathlib import Path +from typing import ClassVar, TextIO + +from toad import paths + + +class FatalTrace: + """The one fault trace of this process. `enabled` is called once by each entry that + starts a frontend (Android's and desktop Qt's), first thing.""" + + PREFIX: ClassVar[str] = "fatal" + stream: ClassVar[TextIO | None] = None + path: ClassVar[Path | None] = None + + @classmethod + def enabled(cls) -> Path: + """Open this run's trace file and route every fatal signal's stacks into it.""" + if cls.path is None: + path = paths.get_errors() / paths.generate_datetime_filename(cls.PREFIX, ".txt") + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + cls.stream = os.fdopen(descriptor, "w", buffering=1, encoding="utf-8") + cls.path = path + faulthandler.enable(cls.stream, all_threads=True) + atexit.register(cls.closed) + return cls.path + + @classmethod + def closed(cls) -> None: + """A clean exit: stop tracing and drop the file when no fault wrote to it.""" + if cls.stream is None or cls.path is None: + return + faulthandler.disable() + cls.stream.close() + if cls.path.stat().st_size == 0: + cls.path.unlink() + cls.stream = None + cls.path = None diff --git a/src/toad/qt/android.py b/src/toad/qt/android.py index f40ee45f9..93ec04f32 100644 --- a/src/toad/qt/android.py +++ b/src/toad/qt/android.py @@ -26,6 +26,7 @@ from PySide6.QtCore import Qt, QTimer from toad import paths +from toad.fatal_trace import FatalTrace from toad.qt.android_app import KEPT from toad.qt.frame_probe import FrameProbe from toad.qt.splash import QtSplash @@ -118,6 +119,7 @@ def load_client(): def main() -> None: + FatalTrace.enabled() # before Qt: a native crash anywhere after this leaves every thread's stack _install_excepthook() # Qt's accessibility bridge is switched off by the APK's activity before Qt's layout # exists (tools/android/keepalive ToadActivity.onCreate sets QT_ANDROID_DISABLE_ACCESSIBILITY): diff --git a/src/toad/qt/main.py b/src/toad/qt/main.py index e40bed040..f7a4f369f 100644 --- a/src/toad/qt/main.py +++ b/src/toad/qt/main.py @@ -11,12 +11,14 @@ from PySide6.QtWidgets import QApplication +from toad.fatal_trace import FatalTrace from toad.qt.frame_probe import FrameProbe from toad.qt.splash import QtSplash from toad.qt.theme import current_scheme def main(arguments: list[str] = sys.argv) -> None: + FatalTrace.enabled() # before Qt: a native crash anywhere after this leaves every thread's stack application = FrameProbe.application(arguments[:1]) scheme = current_scheme(os.environ) QtSplash.shown(application, scheme) diff --git a/tools/android/shipped_run.py b/tools/android/shipped_run.py index d8bed2d31..5fb635340 100644 --- a/tools/android/shipped_run.py +++ b/tools/android/shipped_run.py @@ -146,6 +146,44 @@ def shipped_qt(app: Path, recipes: Path, trust: Path, profile: Path, out: Path) env=os.environ | {"PYTHONOPTIMIZE": str(python_optimize())}) +def shipped_fault(app: Path, recipes: Path, crash: bool) -> subprocess.CompletedProcess: + """The shipped interpreter's fault trace (toad.fatal_trace): Android's entry opens it first + thing, so a native crash leaves every thread's stack under the state's errors folder. + `crash` faults the process on purpose; otherwise it exits cleanly, which leaves no file.""" + code = ( + "import faulthandler, sys\n" + f"sys.path[:0] = [{str(app)!r}, {str(recipes)!r}]\n" + "from toad.qt.android_app import enter\n" + "enter()\n" + "import toad.qt.android, toad.qt.main\n" + "from toad.fatal_trace import FatalTrace\n" + "missing = [entry.__name__ for entry in (toad.qt.android, toad.qt.main) if 'FatalTrace' not in entry.main.__code__.co_names]\n" + "missing and sys.exit(f'error: entries that do not open the fault trace: {missing}')\n" + "FatalTrace.enabled()\n" + + ("faulthandler._sigsegv()\n" if crash else "") + ) + return subprocess.run([sys.executable, "-S", "-B", "-c", code], capture_output=True, text=True, + timeout=120, cwd=recipes, check=False, + env=os.environ | {"PYTHONOPTIMIZE": str(python_optimize()), "QT_QPA_PLATFORM": "offscreen"}) + + +def fault_trace_failure(state: Path, app: Path, recipes: Path) -> str | None: + """Why the fault trace would not catch a native crash on the phone, or None.""" + clean = shipped_fault(app, recipes, crash=False) + left = sorted(state.rglob("errors/fatal*.txt")) + if clean.returncode != 0 or left: + return f"a clean run exited {clean.returncode} and left {[path.name for path in left]}: {clean.stderr[-2000:]}" + crashed = shipped_fault(app, recipes, crash=True) + traces = sorted(state.rglob("errors/fatal*.txt")) + text = traces[-1].read_text() if traces else "" + if crashed.returncode != -11 or "Segmentation fault" not in text or "Current thread" not in text: + return (f"a faulted run exited {crashed.returncode} with {len(traces)} trace file(s): " + f"{text[-1000:] or crashed.stderr[-2000:]}") + for trace in traces: + trace.unlink() + return None + + def main(runtime: Path, app: Path, site_packages: Path) -> int: work = Path(tempfile.mkdtemp(prefix="toad-shipped-")) bus = ScratchBus(runtime, work) @@ -179,6 +217,11 @@ def main(runtime: Path, app: Path, site_packages: Path) -> int: f"{len(errors)} error file(s)); TOAD_SHIPPED_KEEP=1 keeps its scratch folder", file=sys.stderr) return 1 print("shipped app ran every view kind and bound key on a scratch bus") + fault = fault_trace_failure(work / "kept" / "state", shipped, recipes) + if fault: + print(f"error: the shipped app's fault trace: {fault}", file=sys.stderr) + return 1 + print("a native crash in the shipped app leaves every thread's stack under its errors folder") return 0 finally: bus.stop()