Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/toad/client/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,10 @@ class ObservationRetry:
LONGEST: ClassVar[float] = 30.0
losses: int = 0
due: float = 0.0
# When the last result reached the UI (monotonic), so an outage can say how long the UI
# had gone without one: a relay that ends because its reader took nothing for its write
# deadline reads differently from a connector that went away.
last_arrival: float | None = None

def lost(self, now: float) -> None:
self.losses += 1
Expand All @@ -363,6 +367,17 @@ def waiting(self, now: float) -> bool:
def recovered(self) -> None:
self.losses, self.due = 0, 0.0

def arrived(self, now: float) -> None:
"""A result reached the UI: the outage, if any, is over."""
self.recovered()
self.last_arrival = now

def quiet_note(self, now: float) -> str:
"""How long before ``now`` the UI last received a result, for the outage's report."""
if self.last_arrival is None:
return "No observation result had reached the UI yet."
return f"The last observation result reached the UI {now - self.last_arrival:.1f} s before this."


@dataclass(frozen=True, kw_only=True)
class Observation(ABC):
Expand Down Expand Up @@ -774,7 +789,7 @@ def retire(self, access: CoordinationAccess):

def receive(self, result: Observed):
if self.is_current():
self.access.retry.recovered()
self.access.retry.arrived(time.monotonic())
self.access.dispatch_sync(result, self)

def tell(self, wanted) -> None:
Expand All @@ -797,6 +812,7 @@ def receive_failed(self, error: Exception):
if self.is_current():
self.failed(error, self.opened_selection())
if self.access.retry.began():
error.add_note(self.access.retry.quiet_note(time.monotonic()))
self.access.app.report_fatal(error)

def views_changed(self, event=None, subscription=None):
Expand Down
52 changes: 52 additions & 0 deletions src/toad/fatal_trace.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""A native crash's Python stacks: every thread's, written beside the saved tracebacks.

A fault in C (a segfault in the interpreter, PySide or a C extension) kills the process
before any Python handler runs, so nothing reaches `paths.save_error`. Python's
`faulthandler` writes each thread's stack from the signal handler itself, into a file that
must already be open. This owner opens that file at a start, before anything native runs,
keeps it open for the life of the process, and removes it at a clean exit when nothing was
written, so the errors folder holds a `fatal_*` file only for a run that crashed.
"""

from __future__ import annotations

import atexit
import faulthandler
import os
from pathlib import Path
from typing import ClassVar, TextIO

from toad import paths


class FatalTrace:
"""The one fault trace of this process. `enabled` is called once by each entry that
starts a frontend (Android's and desktop Qt's), first thing."""

PREFIX: ClassVar[str] = "fatal"
stream: ClassVar[TextIO | None] = None
path: ClassVar[Path | None] = None

@classmethod
def enabled(cls) -> Path:
"""Open this run's trace file and route every fatal signal's stacks into it."""
if cls.path is None:
path = paths.get_errors() / paths.generate_datetime_filename(cls.PREFIX, ".txt")
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
cls.stream = os.fdopen(descriptor, "w", buffering=1, encoding="utf-8")
cls.path = path
faulthandler.enable(cls.stream, all_threads=True)
atexit.register(cls.closed)
return cls.path

@classmethod
def closed(cls) -> None:
"""A clean exit: stop tracing and drop the file when no fault wrote to it."""
if cls.stream is None or cls.path is None:
return
faulthandler.disable()
cls.stream.close()
if cls.path.stat().st_size == 0:
cls.path.unlink()
cls.stream = None
cls.path = None
2 changes: 2 additions & 0 deletions src/toad/qt/android.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from PySide6.QtCore import Qt, QTimer

from toad import paths
from toad.fatal_trace import FatalTrace
from toad.qt.android_app import KEPT
from toad.qt.frame_probe import FrameProbe
from toad.qt.splash import QtSplash
Expand Down Expand Up @@ -118,6 +119,7 @@ def load_client():


def main() -> None:
FatalTrace.enabled() # before Qt: a native crash anywhere after this leaves every thread's stack
_install_excepthook()
# Qt's accessibility bridge is switched off by the APK's activity before Qt's layout
# exists (tools/android/keepalive ToadActivity.onCreate sets QT_ANDROID_DISABLE_ACCESSIBILITY):
Expand Down
2 changes: 2 additions & 0 deletions src/toad/qt/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@

from PySide6.QtWidgets import QApplication

from toad.fatal_trace import FatalTrace
from toad.qt.frame_probe import FrameProbe
from toad.qt.splash import QtSplash
from toad.qt.theme import current_scheme


def main(arguments: list[str] = sys.argv) -> None:
FatalTrace.enabled() # before Qt: a native crash anywhere after this leaves every thread's stack
application = FrameProbe.application(arguments[:1])
scheme = current_scheme(os.environ)
QtSplash.shown(application, scheme)
Expand Down
43 changes: 43 additions & 0 deletions tools/android/shipped_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,44 @@ def shipped_qt(app: Path, recipes: Path, trust: Path, profile: Path, out: Path)
env=os.environ | {"PYTHONOPTIMIZE": str(python_optimize())})


def shipped_fault(app: Path, recipes: Path, crash: bool) -> subprocess.CompletedProcess:
"""The shipped interpreter's fault trace (toad.fatal_trace): Android's entry opens it first
thing, so a native crash leaves every thread's stack under the state's errors folder.
`crash` faults the process on purpose; otherwise it exits cleanly, which leaves no file."""
code = (
"import faulthandler, sys\n"
f"sys.path[:0] = [{str(app)!r}, {str(recipes)!r}]\n"
"from toad.qt.android_app import enter\n"
"enter()\n"
"import toad.qt.android, toad.qt.main\n"
"from toad.fatal_trace import FatalTrace\n"
"missing = [entry.__name__ for entry in (toad.qt.android, toad.qt.main) if 'FatalTrace' not in entry.main.__code__.co_names]\n"
"missing and sys.exit(f'error: entries that do not open the fault trace: {missing}')\n"
"FatalTrace.enabled()\n"
+ ("faulthandler._sigsegv()\n" if crash else "")
)
return subprocess.run([sys.executable, "-S", "-B", "-c", code], capture_output=True, text=True,
timeout=120, cwd=recipes, check=False,
env=os.environ | {"PYTHONOPTIMIZE": str(python_optimize()), "QT_QPA_PLATFORM": "offscreen"})


def fault_trace_failure(state: Path, app: Path, recipes: Path) -> str | None:
"""Why the fault trace would not catch a native crash on the phone, or None."""
clean = shipped_fault(app, recipes, crash=False)
left = sorted(state.rglob("errors/fatal*.txt"))
if clean.returncode != 0 or left:
return f"a clean run exited {clean.returncode} and left {[path.name for path in left]}: {clean.stderr[-2000:]}"
crashed = shipped_fault(app, recipes, crash=True)
traces = sorted(state.rglob("errors/fatal*.txt"))
text = traces[-1].read_text() if traces else ""
if crashed.returncode != -11 or "Segmentation fault" not in text or "Current thread" not in text:
return (f"a faulted run exited {crashed.returncode} with {len(traces)} trace file(s): "
f"{text[-1000:] or crashed.stderr[-2000:]}")
for trace in traces:
trace.unlink()
return None


def main(runtime: Path, app: Path, site_packages: Path) -> int:
work = Path(tempfile.mkdtemp(prefix="toad-shipped-"))
bus = ScratchBus(runtime, work)
Expand Down Expand Up @@ -179,6 +217,11 @@ def main(runtime: Path, app: Path, site_packages: Path) -> int:
f"{len(errors)} error file(s)); TOAD_SHIPPED_KEEP=1 keeps its scratch folder", file=sys.stderr)
return 1
print("shipped app ran every view kind and bound key on a scratch bus")
fault = fault_trace_failure(work / "kept" / "state", shipped, recipes)
if fault:
print(f"error: the shipped app's fault trace: {fault}", file=sys.stderr)
return 1
print("a native crash in the shipped app leaves every thread's stack under its errors folder")
return 0
finally:
bus.stop()
Expand Down
Loading