From 976d44eb8b7d6738a755907f8d3044c2e1838c51 Mon Sep 17 00:00:00 2001 From: alantangok Date: Tue, 20 Oct 2026 17:14:02 +0800 Subject: [PATCH 1/4] fix: keep stream recovery bounded across transient successes A brief SDK OK or display-layer rendering status currently clears the retry counter, allowing repeated failure/success cycles to restart recovery forever. Retain the three-attempt budget until five continuous healthy seconds, keep the existing retry delays and timeouts, and hold an opaque recovery overlay. Do not accept the SDK network-failure flag as successful recovery. Exercise the production poll and PiP recovery methods with fake dependencies and a controlled clock, and run these regressions in the existing build CI. The baseline fails eight behavior checks; the fix passes all 32 checks and the unsigned Release iphoneos build with Xcode 26.6. Device confirmation covers the older stable candidate. This current-main port still needs physical-device restore/PiP validation. --- .github/workflows/build.yml | 2 + OpenParsec.xcodeproj/project.pbxproj | 4 + OpenParsec/MainView.swift | 2 +- OpenParsec/ParsecBackgroundManager.swift | 4 +- OpenParsec/ParsecView.swift | 34 +-- OpenParsec/PictureInPictureManager.swift | 16 +- OpenParsec/SceneDelegate.swift | 1 + OpenParsec/StreamRecovery.swift | 38 ++++ scripts/check_stream_recovery.py | 278 +++++++++++++++++++++++ 9 files changed, 356 insertions(+), 23 deletions(-) create mode 100644 OpenParsec/StreamRecovery.swift create mode 100644 scripts/check_stream_recovery.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8fd4779c..16ccde0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,8 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive + - name: Check stream recovery + run: python3 scripts/check_stream_recovery.py - name: Set Scheme id: scheme run: | diff --git a/OpenParsec.xcodeproj/project.pbxproj b/OpenParsec.xcodeproj/project.pbxproj index 5526ba57..d1c6e11d 100644 --- a/OpenParsec.xcodeproj/project.pbxproj +++ b/OpenParsec.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + A1B2C3D42F8E000200000002 /* StreamRecovery.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D42F8E000200000001 /* StreamRecovery.swift */; }; 170E0A772BEF5AFF00B0EA32 /* ParsecViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 170E0A762BEF5AFF00B0EA32 /* ParsecViewController.swift */; }; 17374D412CB8E35D00EDCEE7 /* KeyCodeTranslators.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17374D402CB8E35500EDCEE7 /* KeyCodeTranslators.swift */; }; 17840D1D2CB639530097374B /* ParsecUserData.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17840D1C2CB6394D0097374B /* ParsecUserData.swift */; }; @@ -59,6 +60,7 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + A1B2C3D42F8E000200000001 /* StreamRecovery.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamRecovery.swift; sourceTree = ""; }; 170E0A762BEF5AFF00B0EA32 /* ParsecViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ParsecViewController.swift; sourceTree = ""; }; 17374D402CB8E35500EDCEE7 /* KeyCodeTranslators.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyCodeTranslators.swift; sourceTree = ""; }; 17840D1C2CB6394D0097374B /* ParsecUserData.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ParsecUserData.swift; sourceTree = ""; }; @@ -141,6 +143,7 @@ isa = PBXGroup; children = ( E76224A02F8C017700A2F86F /* ParsecBackgroundManager.swift */, + A1B2C3D42F8E000200000001 /* StreamRecovery.swift */, E762250A2F8D0A0000A2F86F /* PictureInPictureManager.swift */, 17374D402CB8E35500EDCEE7 /* KeyCodeTranslators.swift */, 17840D1C2CB6394D0097374B /* ParsecUserData.swift */, @@ -290,6 +293,7 @@ 27AC751829EA339B00E8CAF7 /* URLImage.swift in Sources */, 27B23A242B1B98EF00B52F14 /* ParsecMetalViewController.swift in Sources */, E76224A12F8C017F00A2F86F /* ParsecBackgroundManager.swift in Sources */, + A1B2C3D42F8E000200000002 /* StreamRecovery.swift in Sources */, E762250B2F8D0A0100A2F86F /* PictureInPictureManager.swift in Sources */, 27A923E029E8E53000F54BDA /* TouchHandlingView.swift in Sources */, 27E61A92292965FC00FF6563 /* AppDelegate.swift in Sources */, diff --git a/OpenParsec/MainView.swift b/OpenParsec/MainView.swift index a6b5ad6e..cd43ff9d 100644 --- a/OpenParsec/MainView.swift +++ b/OpenParsec/MainView.swift @@ -680,7 +680,7 @@ struct MainView: View { pollTimer = nil CParsec.initialize() DataManager.model.reset() - ParsecBackgroundManager.shared.reconnectAttempts = 0 + ParsecBackgroundManager.shared.recoveryBudget.reset() connectingToName = who.hostname retryingConnect = retried withAnimation { isConnecting = true } diff --git a/OpenParsec/ParsecBackgroundManager.swift b/OpenParsec/ParsecBackgroundManager.swift index a1e68e9d..b68d9f57 100644 --- a/OpenParsec/ParsecBackgroundManager.swift +++ b/OpenParsec/ParsecBackgroundManager.swift @@ -27,7 +27,7 @@ class ParsecBackgroundManager { } private var didDisconnectDueToBackground = false private(set) var isReconnecting = false - var reconnectAttempts = 0 + var recoveryBudget = StreamRecoveryBudget() var reconnectStartTime = Date() var resumedAt: Date? var isPaused = false @@ -112,7 +112,7 @@ class ParsecBackgroundManager { func endSession() { didDisconnectDueToBackground = false isReconnecting = false - reconnectAttempts = 0 + recoveryBudget.reset() resumeOnLaunch = false } diff --git a/OpenParsec/ParsecView.swift b/OpenParsec/ParsecView.swift index f45d5332..bd25c6fa 100644 --- a/OpenParsec/ParsecView.swift +++ b/OpenParsec/ParsecView.swift @@ -55,6 +55,8 @@ struct ParsecStatusBar: View { return } + let mgr = ParsecBackgroundManager.shared + let now = ProcessInfo.processInfo.systemUptime var pcs = ParsecClientStatus() var status = CParsec.getStatusEx(&pcs) @@ -63,7 +65,7 @@ struct ParsecStatusBar: View { pipActive = PictureInPictureManager.shared.isPiPActive } - if status == PARSEC_OK && pcs.networkFailure && !isReconnecting && !pipActive { + if status == PARSEC_OK && pcs.networkFailure && !pipActive { if networkFailureSince == nil { networkFailureSince = Date() } let lost = Date().timeIntervalSince(networkFailureSince!) let justResumed = Date().timeIntervalSince(ParsecBackgroundManager.shared.resumedAt ?? .distantPast) < 10 @@ -81,6 +83,7 @@ struct ParsecStatusBar: View { } if status != PARSEC_OK { + mgr.recoveryBudget.recordFailure() if ParsecBackgroundManager.shared.isMarkedForReconnect { return } @@ -110,21 +113,16 @@ struct ParsecStatusBar: View { } if SettingsHandler.autoReconnect, !status.isPermanentFailure, let peerId = ParsecBackgroundManager.shared.lastPeerId { - let mgr = ParsecBackgroundManager.shared - if mgr.reconnectAttempts < 3 { - let gap: TimeInterval = mgr.reconnectAttempts == 0 ? 0 : mgr.reconnectAttempts == 1 ? 2 : 5 - if Date().timeIntervalSince(mgr.reconnectStartTime) < gap { - return - } - mgr.reconnectAttempts += 1 - appNote("[reconnect] attempt \(mgr.reconnectAttempts) after status \(status.rawValue)") + if !mgr.recoveryBudget.isExhausted { + guard mgr.recoveryBudget.beginAttempt(at: now) else { return } + appNote("[reconnect] attempt \(mgr.recoveryBudget.attempts) after status \(status.rawValue)") isReconnecting = true mgr.reconnectStartTime = Date() + mgr.glkViewController?.isPaused = true parsecViewController?.resetKeyState() CParsec.reconnect(peerId) return } - mgr.reconnectAttempts = 0 } isReconnecting = false @@ -141,8 +139,13 @@ struct ParsecStatusBar: View { return } + if pcs.networkFailure { + mgr.recoveryBudget.recordFailure() + return + } + let stable = mgr.recoveryBudget.recordHealthy(at: now) if isReconnecting { - ParsecBackgroundManager.shared.reconnectAttempts = 0 + guard stable else { return } isReconnecting = false appNote("[reconnect] live again") ParsecView.getHostUserData() @@ -258,7 +261,7 @@ struct ParsecView: View { .padding(.top, 4) } .frame(maxWidth: .infinity, maxHeight: .infinity) - .background(Color.black.opacity(0.6)) + .background(Color.black) .zIndex(10) } @@ -452,8 +455,11 @@ struct ParsecView: View { } showDCAlert = false isReconnecting = true - ParsecBackgroundManager.shared.reconnectAttempts = 0 - ParsecBackgroundManager.shared.reconnectStartTime = Date() + let mgr = ParsecBackgroundManager.shared + mgr.recoveryBudget.reset() + _ = mgr.recoveryBudget.beginAttempt(at: ProcessInfo.processInfo.systemUptime) + mgr.reconnectStartTime = Date() + mgr.glkViewController?.isPaused = true parsecViewController.resetKeyState() CParsec.reconnect(peerId) }), diff --git a/OpenParsec/PictureInPictureManager.swift b/OpenParsec/PictureInPictureManager.swift index 1860f47f..4768294f 100644 --- a/OpenParsec/PictureInPictureManager.swift +++ b/OpenParsec/PictureInPictureManager.swift @@ -40,7 +40,7 @@ class PictureInPictureManager: NSObject { } private var cachedFormatDescription: CMVideoFormatDescription? - private var displayLayerFlushes = 0 + private var displayLayerRecovery = StreamRecoveryBudget() private var displayLayerGaveUp = false private(set) var isPiPActive = false @@ -283,18 +283,22 @@ class PictureInPictureManager: NSObject { } private func displayLayerTakesFrames(_ displayLayer: AVSampleBufferDisplayLayer) -> Bool { + guard !displayLayerGaveUp else { return false } + let now = ProcessInfo.processInfo.systemUptime if displayLayer.status == .failed { - guard !displayLayerGaveUp else { return false } - guard displayLayerFlushes < 3 else { + displayLayerRecovery.recordFailure() + guard !displayLayerRecovery.isExhausted else { displayLayerGaveUp = true handleTerminalDisplayLayerFailure(displayLayer) return false } - displayLayerFlushes += 1 + guard displayLayerRecovery.beginAttempt(at: now) else { return false } displayLayer.flush() cachedFormatDescription = nil } else if displayLayer.status == .rendering { - displayLayerFlushes = 0 + displayLayerRecovery.recordHealthy(at: now) + } else { + displayLayerRecovery.recordFailure() } return displayLayer.isReadyForMoreMediaData } @@ -413,7 +417,7 @@ class PictureInPictureManager: NSObject { } private func resetDisplayLayerFailureState() { - displayLayerFlushes = 0 + displayLayerRecovery.reset() displayLayerGaveUp = false } diff --git a/OpenParsec/SceneDelegate.swift b/OpenParsec/SceneDelegate.swift index 4337b6e4..2dce358d 100644 --- a/OpenParsec/SceneDelegate.swift +++ b/OpenParsec/SceneDelegate.swift @@ -38,6 +38,7 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate { private var pendingUnpause: DispatchWorkItem? func sceneDidBecomeActive(_ scene: UIScene) { + ParsecBackgroundManager.shared.recoveryBudget.recordFailure() if #available(iOS 15.0, *) { PictureInPictureManager.shared.stopPiP() if PictureInPictureManager.shared.stopFramePump() { diff --git a/OpenParsec/StreamRecovery.swift b/OpenParsec/StreamRecovery.swift new file mode 100644 index 00000000..70ee6bd9 --- /dev/null +++ b/OpenParsec/StreamRecovery.swift @@ -0,0 +1,38 @@ +import Foundation + +struct StreamRecoveryBudget { + private(set) var attempts = 0 + private(set) var lastAttemptAt: TimeInterval? + private var healthySince: TimeInterval? + + var isExhausted: Bool { attempts >= 3 } + + mutating func beginAttempt(at time: TimeInterval) -> Bool { + healthySince = nil + guard !isExhausted else { return false } + if let lastAttemptAt { + let delay: TimeInterval = attempts == 1 ? 2 : 5 + guard time - lastAttemptAt >= delay else { return false } + } + attempts += 1 + lastAttemptAt = time + return true + } + + mutating func recordFailure() { + healthySince = nil + } + + @discardableResult + mutating func recordHealthy(at time: TimeInterval) -> Bool { + if healthySince == nil { healthySince = time } + guard time - healthySince! >= 5 else { return false } + attempts = 0 + lastAttemptAt = nil + return true + } + + mutating func reset() { + self = StreamRecoveryBudget() + } +} diff --git a/scripts/check_stream_recovery.py b/scripts/check_stream_recovery.py new file mode 100644 index 00000000..59c77f79 --- /dev/null +++ b/scripts/check_stream_recovery.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Run production recovery methods with a fake SDK, display layer and clock. + +Usage: python3 scripts/check_stream_recovery.py +Requires macOS swiftc; does not exercise an iOS renderer or a real stream. +""" + +from pathlib import Path +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def method(path, signature): + source = (ROOT / path).read_text() + start = source.index(signature) + opening = source.index("{", start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end].replace("private func", "func") + + +poll = method("OpenParsec/ParsecView.swift", "func poll()") +# Metrics formatting is unrelated to recovery and requires SDK decoder structs. +poll = poll.split("\n\t\tif showMenu || SettingsHandler.alwaysShowStatus {", 1)[0] + "\n}" +pip = method("OpenParsec/PictureInPictureManager.swift", "private func displayLayerTakesFrames") + +fixtures = r''' +import Foundation + +var clock: TimeInterval = 0 +var testDate: Date { Date(timeIntervalSince1970: clock) } +var failures = 0 +var checks = 0 +func check(_ value: Bool, _ message: String) { + checks += 1 + if !value { print("FAIL: \(message)"); failures += 1 } +} +func appNote(_ message: String) {} + +enum Status: Int { + case ok, interrupted, connecting, approval, permanent + var isPermanentFailure: Bool { self == .permanent } + func readable() -> String { "test failure" } +} +let PARSEC_OK = Status.ok +let NETWORK_ERR_INTERRUPTED = Status.interrupted +let PARSEC_CONNECTING = Status.connecting +let CONNECT_WRN_APPROVAL = Status.approval +struct ParsecClientStatus { var networkFailure = false } +enum SettingsHandler { + static var autoReconnect = true + static var errorPrompts = true +} +class RenderLoop { var isPaused = false } +class ParsecBackgroundManager { + static var shared = ParsecBackgroundManager() + var isPaused = false + var isMarkedForReconnect = false + var lastPeerId: String? = "host" + var reconnectAttempts = 0 // Lets this harness also reproduce the old main behavior. + var recoveryBudget = StreamRecoveryBudget() + var reconnectStartTime = Date(timeIntervalSince1970: 0) + var resumedAt: Date? + var glkViewController: RenderLoop? = RenderLoop() + func connectionDidEnd() {} + func markForReconnect() { isMarkedForReconnect = true } +} +struct Bounds { var width: CGFloat = 1024; var height: CGFloat = 768 } +class MockView { var bounds = Bounds() } +class ParsecViewController { + var view = MockView() + func resetKeyState() {} +} +class UIScreen { + static let main = UIScreen() + var scale: CGFloat = 2 +} +enum ParsecView { static func getHostUserData() {} } +enum CParsec { + static var status = Status.ok + static var networkFailure = false + static var reconnects = 0 + static func getStatusEx(_ pcs: inout ParsecClientStatus) -> Status { + pcs.networkFailure = networkFailure + return status + } + static func disconnect() {} + static func reconnect(_ peer: String) { reconnects += 1 } + static func setFrame(_ width: CGFloat, _ height: CGFloat, _ scale: CGFloat) {} +} +class AVAudioSession { + enum Option { case notifyOthersOnDeactivation } + static func sharedInstance() -> AVAudioSession { AVAudioSession() } + func setActive(_ active: Bool, options: Option) throws {} +} +class AVSampleBufferDisplayLayer { + enum State { case failed, rendering, unknown } + var status = State.unknown + var isReadyForMoreMediaData = true + var requiresFlushToResumeDecoding = true + var flushes = 0 + func flush() { flushes += 1; status = .unknown } +} +class PictureInPictureManager { + static var shared = PictureInPictureManager() + var isPiPActive = false + var isPumping = false + var displayLayerFlushes = 0 + var displayLayerRecovery = StreamRecoveryBudget() + var displayLayerGaveUp = false + var cachedFormatDescription: Int? + var terminalFailures = 0 + func handleTerminalDisplayLayerFailure(_ layer: AVSampleBufferDisplayLayer) { + terminalFailures += 1 + } + // PIP_METHOD +} +class PollHarness { + var showDCAlert = false + var DCAlertText = "" + var isReconnecting = false + var connectionLost = false + var networkFailureSince: Date? + var parsecViewController: ParsecViewController? = ParsecViewController() + var wasDisconnected = true + var silentDisconnects = 0 + func onSilentDisconnect() { silentDisconnects += 1 } + // POLL_METHOD +} + +@main struct CheckRecovery { + static func fresh() -> PollHarness { + clock = 0 + CParsec.reconnects = 0 + CParsec.networkFailure = false + CParsec.status = .ok + SettingsHandler.autoReconnect = true + SettingsHandler.errorPrompts = true + ParsecBackgroundManager.shared = ParsecBackgroundManager() + PictureInPictureManager.shared = PictureInPictureManager() + return PollHarness() + } + static func poll(_ harness: PollHarness, _ time: TimeInterval, _ status: Status) { + clock = time + CParsec.status = status + harness.poll() + } + static func main() { + let flapping = fresh() + for time in [0.0, 2, 7, 12, 17] { + poll(flapping, time, .interrupted) + poll(flapping, time + 0.5, .ok) + } + check(CParsec.reconnects == 3, "reconnects stop after three attempts across brief OK samples") + check(flapping.showDCAlert, "exhausted reconnects reach the existing error prompt") + + let stable = fresh() + poll(stable, 0, .interrupted) + check(ParsecBackgroundManager.shared.glkViewController?.isPaused == true, "recovery pauses the shared render loop") + poll(stable, 0.5, .ok) + check(stable.isReconnecting, "one OK poll keeps the recovery overlay visible") + poll(stable, 5, .ok) + check(stable.isReconnecting, "recovery waits for five full healthy seconds") + poll(stable, 5.5, .ok) + check(!stable.isReconnecting, "sustained health completes recovery") + check(ParsecBackgroundManager.shared.glkViewController?.isPaused == false, "stable recovery resumes rendering") + check(ParsecBackgroundManager.shared.recoveryBudget.attempts == 0, "sustained health renews retries") + + let pumping = fresh() + PictureInPictureManager.shared.isPumping = true + poll(pumping, 0, .interrupted) + poll(pumping, 0.5, .ok) + poll(pumping, 5.5, .ok) + check(ParsecBackgroundManager.shared.glkViewController?.isPaused == true, "PiP pumping retains ownership of rendering") + + let network = fresh() + poll(network, 0, .interrupted) + CParsec.networkFailure = true + for time in stride(from: 0.5, through: 4.0, by: 0.5) { poll(network, time, .ok) } + check(CParsec.reconnects == 2, "SDK OK with network failure cannot complete a reconnect") + check(network.isReconnecting, "network failure keeps recovery active") + + let connecting = fresh() + poll(connecting, 0, .interrupted) + poll(connecting, 14, .connecting) + check(CParsec.reconnects == 1, "connecting retains the existing 15-second timeout") + poll(connecting, 15, .connecting) + check(CParsec.reconnects == 2, "connecting retries after its timeout") + + let approval = fresh() + poll(approval, 0, .interrupted) + poll(approval, 59, .approval) + check(CParsec.reconnects == 1, "approval retains its 60-second timeout") + poll(approval, 60, .approval) + check(CParsec.reconnects == 2, "approval retries after its timeout") + + let permanent = fresh() + poll(permanent, 0, .permanent) + check(CParsec.reconnects == 0 && permanent.showDCAlert, "permanent failures are not retried") + let silent = fresh() + SettingsHandler.errorPrompts = false + for time in [0.0, 2, 7, 12] { poll(silent, time, .interrupted) } + check(silent.silentDisconnects == 1 && !silent.showDCAlert, "silent exhaustion stays silent") + + let pip = PictureInPictureManager() + let layer = AVSampleBufferDisplayLayer() + for time in [0.0, 2, 7, 12] { + clock = time + layer.status = .failed + _ = pip.displayLayerTakesFrames(layer) + clock = time + 0.5 + layer.status = .rendering + _ = pip.displayLayerTakesFrames(layer) + } + check(layer.flushes == 3 && pip.terminalFailures == 1, "PiP stops alternating failure/rendering after three flushes") + check(!pip.displayLayerTakesFrames(layer), "terminal PiP recovery stays stopped") + + let delayed = PictureInPictureManager() + let delayedLayer = AVSampleBufferDisplayLayer() + clock = 0 + delayedLayer.status = .failed + _ = delayed.displayLayerTakesFrames(delayedLayer) + clock = 1 + delayedLayer.status = .failed + check(!delayed.displayLayerTakesFrames(delayedLayer), "PiP backoff does not enqueue failed layers") + check(delayedLayer.flushes == 1, "PiP respects retry spacing") + + clock = 2 + delayedLayer.status = .failed + _ = delayed.displayLayerTakesFrames(delayedLayer) + clock = 2.5 + delayedLayer.status = .rendering + _ = delayed.displayLayerTakesFrames(delayedLayer) + clock = 6 + delayedLayer.status = .unknown + _ = delayed.displayLayerTakesFrames(delayedLayer) + clock = 7.5 + delayedLayer.status = .rendering + _ = delayed.displayLayerTakesFrames(delayedLayer) + check(delayed.displayLayerRecovery.attempts == 2, "unknown layer status interrupts the healthy interval") + clock = 12.5 + _ = delayed.displayLayerTakesFrames(delayedLayer) + check(delayed.displayLayerRecovery.attempts == 0, "five healthy PiP seconds renew flush retries") + + var budget = StreamRecoveryBudget() + check(budget.beginAttempt(at: 0), "first recovery attempt starts immediately") + check(!budget.beginAttempt(at: 1), "second attempt waits two seconds") + check(budget.beginAttempt(at: 2), "second attempt starts at its deadline") + check(!budget.beginAttempt(at: 6.9), "third attempt waits five seconds") + check(budget.beginAttempt(at: 7), "third attempt starts at its deadline") + check(!budget.recordHealthy(at: 8), "brief health retains exhausted retries") + budget.recordFailure() + check(!budget.recordHealthy(at: 100), "resume/failure restarts the health interval") + check(budget.recordHealthy(at: 105), "continuous health renews exhausted retries") + budget.reset() + check(budget.attempts == 0 && budget.lastAttemptAt == nil, "a new session gets a fresh budget") + + if failures > 0 { exit(1) } + print("\(checks) stream recovery policy and production-method checks passed") + } +} +''' + +fixtures = fixtures.replace("// PIP_METHOD", pip).replace("// POLL_METHOD", poll) +fixtures = fixtures.replace("ProcessInfo.processInfo.systemUptime", "clock") +fixtures = fixtures.replace("Date()", "testDate") +with tempfile.TemporaryDirectory(prefix="openparsec-recovery-") as directory: + source = Path(directory) / "RecoveryChecks.swift" + binary = Path(directory) / "check-recovery" + source.write_text(fixtures) + subprocess.run(["swiftc", str(ROOT / "OpenParsec/StreamRecovery.swift"), str(source), "-o", str(binary)], check=True) + subprocess.run([str(binary)], check=True) From 5b712609e8839f10bd716c47b4aceb019bc4fa2e Mon Sep 17 00:00:00 2001 From: alantangok Date: Thu, 29 Oct 2026 16:55:05 +0800 Subject: [PATCH 2/4] chore: narrow recovery PR to app changes --- scripts/check_stream_recovery.py | 278 ------------------------------- 1 file changed, 278 deletions(-) delete mode 100644 scripts/check_stream_recovery.py diff --git a/scripts/check_stream_recovery.py b/scripts/check_stream_recovery.py deleted file mode 100644 index 59c77f79..00000000 --- a/scripts/check_stream_recovery.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env python3 -"""Run production recovery methods with a fake SDK, display layer and clock. - -Usage: python3 scripts/check_stream_recovery.py -Requires macOS swiftc; does not exercise an iOS renderer or a real stream. -""" - -from pathlib import Path -import subprocess -import tempfile - - -ROOT = Path(__file__).resolve().parents[1] - - -def method(path, signature): - source = (ROOT / path).read_text() - start = source.index(signature) - opening = source.index("{", start) - depth = 1 - end = opening + 1 - while depth: - depth += (source[end] == "{") - (source[end] == "}") - end += 1 - return source[start:end].replace("private func", "func") - - -poll = method("OpenParsec/ParsecView.swift", "func poll()") -# Metrics formatting is unrelated to recovery and requires SDK decoder structs. -poll = poll.split("\n\t\tif showMenu || SettingsHandler.alwaysShowStatus {", 1)[0] + "\n}" -pip = method("OpenParsec/PictureInPictureManager.swift", "private func displayLayerTakesFrames") - -fixtures = r''' -import Foundation - -var clock: TimeInterval = 0 -var testDate: Date { Date(timeIntervalSince1970: clock) } -var failures = 0 -var checks = 0 -func check(_ value: Bool, _ message: String) { - checks += 1 - if !value { print("FAIL: \(message)"); failures += 1 } -} -func appNote(_ message: String) {} - -enum Status: Int { - case ok, interrupted, connecting, approval, permanent - var isPermanentFailure: Bool { self == .permanent } - func readable() -> String { "test failure" } -} -let PARSEC_OK = Status.ok -let NETWORK_ERR_INTERRUPTED = Status.interrupted -let PARSEC_CONNECTING = Status.connecting -let CONNECT_WRN_APPROVAL = Status.approval -struct ParsecClientStatus { var networkFailure = false } -enum SettingsHandler { - static var autoReconnect = true - static var errorPrompts = true -} -class RenderLoop { var isPaused = false } -class ParsecBackgroundManager { - static var shared = ParsecBackgroundManager() - var isPaused = false - var isMarkedForReconnect = false - var lastPeerId: String? = "host" - var reconnectAttempts = 0 // Lets this harness also reproduce the old main behavior. - var recoveryBudget = StreamRecoveryBudget() - var reconnectStartTime = Date(timeIntervalSince1970: 0) - var resumedAt: Date? - var glkViewController: RenderLoop? = RenderLoop() - func connectionDidEnd() {} - func markForReconnect() { isMarkedForReconnect = true } -} -struct Bounds { var width: CGFloat = 1024; var height: CGFloat = 768 } -class MockView { var bounds = Bounds() } -class ParsecViewController { - var view = MockView() - func resetKeyState() {} -} -class UIScreen { - static let main = UIScreen() - var scale: CGFloat = 2 -} -enum ParsecView { static func getHostUserData() {} } -enum CParsec { - static var status = Status.ok - static var networkFailure = false - static var reconnects = 0 - static func getStatusEx(_ pcs: inout ParsecClientStatus) -> Status { - pcs.networkFailure = networkFailure - return status - } - static func disconnect() {} - static func reconnect(_ peer: String) { reconnects += 1 } - static func setFrame(_ width: CGFloat, _ height: CGFloat, _ scale: CGFloat) {} -} -class AVAudioSession { - enum Option { case notifyOthersOnDeactivation } - static func sharedInstance() -> AVAudioSession { AVAudioSession() } - func setActive(_ active: Bool, options: Option) throws {} -} -class AVSampleBufferDisplayLayer { - enum State { case failed, rendering, unknown } - var status = State.unknown - var isReadyForMoreMediaData = true - var requiresFlushToResumeDecoding = true - var flushes = 0 - func flush() { flushes += 1; status = .unknown } -} -class PictureInPictureManager { - static var shared = PictureInPictureManager() - var isPiPActive = false - var isPumping = false - var displayLayerFlushes = 0 - var displayLayerRecovery = StreamRecoveryBudget() - var displayLayerGaveUp = false - var cachedFormatDescription: Int? - var terminalFailures = 0 - func handleTerminalDisplayLayerFailure(_ layer: AVSampleBufferDisplayLayer) { - terminalFailures += 1 - } - // PIP_METHOD -} -class PollHarness { - var showDCAlert = false - var DCAlertText = "" - var isReconnecting = false - var connectionLost = false - var networkFailureSince: Date? - var parsecViewController: ParsecViewController? = ParsecViewController() - var wasDisconnected = true - var silentDisconnects = 0 - func onSilentDisconnect() { silentDisconnects += 1 } - // POLL_METHOD -} - -@main struct CheckRecovery { - static func fresh() -> PollHarness { - clock = 0 - CParsec.reconnects = 0 - CParsec.networkFailure = false - CParsec.status = .ok - SettingsHandler.autoReconnect = true - SettingsHandler.errorPrompts = true - ParsecBackgroundManager.shared = ParsecBackgroundManager() - PictureInPictureManager.shared = PictureInPictureManager() - return PollHarness() - } - static func poll(_ harness: PollHarness, _ time: TimeInterval, _ status: Status) { - clock = time - CParsec.status = status - harness.poll() - } - static func main() { - let flapping = fresh() - for time in [0.0, 2, 7, 12, 17] { - poll(flapping, time, .interrupted) - poll(flapping, time + 0.5, .ok) - } - check(CParsec.reconnects == 3, "reconnects stop after three attempts across brief OK samples") - check(flapping.showDCAlert, "exhausted reconnects reach the existing error prompt") - - let stable = fresh() - poll(stable, 0, .interrupted) - check(ParsecBackgroundManager.shared.glkViewController?.isPaused == true, "recovery pauses the shared render loop") - poll(stable, 0.5, .ok) - check(stable.isReconnecting, "one OK poll keeps the recovery overlay visible") - poll(stable, 5, .ok) - check(stable.isReconnecting, "recovery waits for five full healthy seconds") - poll(stable, 5.5, .ok) - check(!stable.isReconnecting, "sustained health completes recovery") - check(ParsecBackgroundManager.shared.glkViewController?.isPaused == false, "stable recovery resumes rendering") - check(ParsecBackgroundManager.shared.recoveryBudget.attempts == 0, "sustained health renews retries") - - let pumping = fresh() - PictureInPictureManager.shared.isPumping = true - poll(pumping, 0, .interrupted) - poll(pumping, 0.5, .ok) - poll(pumping, 5.5, .ok) - check(ParsecBackgroundManager.shared.glkViewController?.isPaused == true, "PiP pumping retains ownership of rendering") - - let network = fresh() - poll(network, 0, .interrupted) - CParsec.networkFailure = true - for time in stride(from: 0.5, through: 4.0, by: 0.5) { poll(network, time, .ok) } - check(CParsec.reconnects == 2, "SDK OK with network failure cannot complete a reconnect") - check(network.isReconnecting, "network failure keeps recovery active") - - let connecting = fresh() - poll(connecting, 0, .interrupted) - poll(connecting, 14, .connecting) - check(CParsec.reconnects == 1, "connecting retains the existing 15-second timeout") - poll(connecting, 15, .connecting) - check(CParsec.reconnects == 2, "connecting retries after its timeout") - - let approval = fresh() - poll(approval, 0, .interrupted) - poll(approval, 59, .approval) - check(CParsec.reconnects == 1, "approval retains its 60-second timeout") - poll(approval, 60, .approval) - check(CParsec.reconnects == 2, "approval retries after its timeout") - - let permanent = fresh() - poll(permanent, 0, .permanent) - check(CParsec.reconnects == 0 && permanent.showDCAlert, "permanent failures are not retried") - let silent = fresh() - SettingsHandler.errorPrompts = false - for time in [0.0, 2, 7, 12] { poll(silent, time, .interrupted) } - check(silent.silentDisconnects == 1 && !silent.showDCAlert, "silent exhaustion stays silent") - - let pip = PictureInPictureManager() - let layer = AVSampleBufferDisplayLayer() - for time in [0.0, 2, 7, 12] { - clock = time - layer.status = .failed - _ = pip.displayLayerTakesFrames(layer) - clock = time + 0.5 - layer.status = .rendering - _ = pip.displayLayerTakesFrames(layer) - } - check(layer.flushes == 3 && pip.terminalFailures == 1, "PiP stops alternating failure/rendering after three flushes") - check(!pip.displayLayerTakesFrames(layer), "terminal PiP recovery stays stopped") - - let delayed = PictureInPictureManager() - let delayedLayer = AVSampleBufferDisplayLayer() - clock = 0 - delayedLayer.status = .failed - _ = delayed.displayLayerTakesFrames(delayedLayer) - clock = 1 - delayedLayer.status = .failed - check(!delayed.displayLayerTakesFrames(delayedLayer), "PiP backoff does not enqueue failed layers") - check(delayedLayer.flushes == 1, "PiP respects retry spacing") - - clock = 2 - delayedLayer.status = .failed - _ = delayed.displayLayerTakesFrames(delayedLayer) - clock = 2.5 - delayedLayer.status = .rendering - _ = delayed.displayLayerTakesFrames(delayedLayer) - clock = 6 - delayedLayer.status = .unknown - _ = delayed.displayLayerTakesFrames(delayedLayer) - clock = 7.5 - delayedLayer.status = .rendering - _ = delayed.displayLayerTakesFrames(delayedLayer) - check(delayed.displayLayerRecovery.attempts == 2, "unknown layer status interrupts the healthy interval") - clock = 12.5 - _ = delayed.displayLayerTakesFrames(delayedLayer) - check(delayed.displayLayerRecovery.attempts == 0, "five healthy PiP seconds renew flush retries") - - var budget = StreamRecoveryBudget() - check(budget.beginAttempt(at: 0), "first recovery attempt starts immediately") - check(!budget.beginAttempt(at: 1), "second attempt waits two seconds") - check(budget.beginAttempt(at: 2), "second attempt starts at its deadline") - check(!budget.beginAttempt(at: 6.9), "third attempt waits five seconds") - check(budget.beginAttempt(at: 7), "third attempt starts at its deadline") - check(!budget.recordHealthy(at: 8), "brief health retains exhausted retries") - budget.recordFailure() - check(!budget.recordHealthy(at: 100), "resume/failure restarts the health interval") - check(budget.recordHealthy(at: 105), "continuous health renews exhausted retries") - budget.reset() - check(budget.attempts == 0 && budget.lastAttemptAt == nil, "a new session gets a fresh budget") - - if failures > 0 { exit(1) } - print("\(checks) stream recovery policy and production-method checks passed") - } -} -''' - -fixtures = fixtures.replace("// PIP_METHOD", pip).replace("// POLL_METHOD", poll) -fixtures = fixtures.replace("ProcessInfo.processInfo.systemUptime", "clock") -fixtures = fixtures.replace("Date()", "testDate") -with tempfile.TemporaryDirectory(prefix="openparsec-recovery-") as directory: - source = Path(directory) / "RecoveryChecks.swift" - binary = Path(directory) / "check-recovery" - source.write_text(fixtures) - subprocess.run(["swiftc", str(ROOT / "OpenParsec/StreamRecovery.swift"), str(source), "-o", str(binary)], check=True) - subprocess.run([str(binary)], check=True) From 47886da630f07a6729375f903fe5d360edb9439e Mon Sep 17 00:00:00 2001 From: alantangok Date: Thu, 29 Oct 2026 16:55:10 +0800 Subject: [PATCH 3/4] chore: restore existing build workflow --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16ccde0f..8fd4779c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,8 +33,6 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - - name: Check stream recovery - run: python3 scripts/check_stream_recovery.py - name: Set Scheme id: scheme run: | From e6b25be0bb7b39622a88277d08d8871d4ecb769e Mon Sep 17 00:00:00 2001 From: alantangok Date: Wed, 4 Nov 2026 18:25:07 +0800 Subject: [PATCH 4/4] fix: cover stale stream immediately on PiP restore --- OpenParsec/ParsecView.swift | 18 ++++++++++++++---- OpenParsec/SceneDelegate.swift | 4 ++++ OpenParsec/StreamRecovery.swift | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/OpenParsec/ParsecView.swift b/OpenParsec/ParsecView.swift index bd25c6fa..b5721ba1 100644 --- a/OpenParsec/ParsecView.swift +++ b/OpenParsec/ParsecView.swift @@ -10,18 +10,20 @@ struct ParsecStatusBar: View { @Binding var DCAlertText: String @Binding var isReconnecting: Bool @Binding var connectionLost: Bool + @Binding var restoreGuard: StreamRestoreGuard @State var networkFailureSince: Date? @State var parsecViewController: ParsecViewController? @State var wasDisconnected: Bool = true let timer = Timer.publish(every: 0.5, on: .main, in: .common).autoconnect() let onSilentDisconnect: () -> Void - init(showMenu: Binding, showDCAlert: Binding, DCAlertText: Binding, parsecViewController: ParsecViewController, isReconnecting: Binding, connectionLost: Binding, onSilentDisconnect: @escaping () -> Void) { + init(showMenu: Binding, showDCAlert: Binding, DCAlertText: Binding, parsecViewController: ParsecViewController, isReconnecting: Binding, connectionLost: Binding, restoreGuard: Binding, onSilentDisconnect: @escaping () -> Void) { _showMenu = showMenu _showDCAlert = showDCAlert _DCAlertText = DCAlertText _isReconnecting = isReconnecting _connectionLost = connectionLost + _restoreGuard = restoreGuard self.parsecViewController = parsecViewController self.onSilentDisconnect = onSilentDisconnect } @@ -57,6 +59,7 @@ struct ParsecStatusBar: View { let mgr = ParsecBackgroundManager.shared let now = ProcessInfo.processInfo.systemUptime + restoreGuard.expire(at: now) var pcs = ParsecClientStatus() var status = CParsec.getStatusEx(&pcs) @@ -144,6 +147,7 @@ struct ParsecStatusBar: View { return } let stable = mgr.recoveryBudget.recordHealthy(at: now) + if stable { restoreGuard.end() } if isReconnecting { guard stable else { return } isReconnecting = false @@ -201,6 +205,7 @@ struct ParsecView: View { @State var preferH265: Bool = true @State var isReconnecting: Bool = false @State var connectionLost: Bool = false + @State var restoreGuard = StreamRestoreGuard() @State var resolutions: [ParsecResolution] @State var bitrates: [Int] @@ -241,12 +246,12 @@ struct ParsecView: View { .zIndex(1) .prefersPersistentSystemOverlaysHidden() - ParsecStatusBar(showMenu: $showMenu, showDCAlert: $showDCAlert, DCAlertText: $DCAlertText, parsecViewController: parsecViewController, isReconnecting: $isReconnecting, connectionLost: $connectionLost, onSilentDisconnect: { disconnect() }) + ParsecStatusBar(showMenu: $showMenu, showDCAlert: $showDCAlert, DCAlertText: $DCAlertText, parsecViewController: parsecViewController, isReconnecting: $isReconnecting, connectionLost: $connectionLost, restoreGuard: $restoreGuard, onSilentDisconnect: { disconnect() }) - if isReconnecting || connectionLost { + if isReconnecting || connectionLost || restoreGuard.isActive { VStack(spacing: 12) { ActivityIndicator(isAnimating: .constant(true), style: .large, tint: .white) - Text(isReconnecting ? ParsecBackgroundManager.shared.lastHostname.map { "Reconnecting to \($0)..." } ?? "Reconnecting..." : "Connection lost...") + Text(isReconnecting || restoreGuard.isActive ? ParsecBackgroundManager.shared.lastHostname.map { "Reconnecting to \($0)..." } ?? "Reconnecting..." : "Connection lost...") .font(.system(size: 16, weight: .medium)) .foregroundColor(.white) Button(action: { disconnect() }) { @@ -512,6 +517,11 @@ struct ParsecView: View { } } } + PictureInPictureManager.shared.onRestoreUserInterface = { [self] in + // a stale stream still reports OK for seconds, keep it covered until it proves healthy + ParsecBackgroundManager.shared.recoveryBudget.recordFailure() + restoreGuard.begin(at: ProcessInfo.processInfo.systemUptime) + } PictureInPictureManager.shared.onPiPStartFailed = { if UIApplication.shared.applicationState != .active { ParsecBackgroundManager.shared.glkViewController?.isPaused = true diff --git a/OpenParsec/SceneDelegate.swift b/OpenParsec/SceneDelegate.swift index 2dce358d..95a5b488 100644 --- a/OpenParsec/SceneDelegate.swift +++ b/OpenParsec/SceneDelegate.swift @@ -69,6 +69,10 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate { } func sceneWillEnterForeground(_ scene: UIScene) { + // returning through the app icon skips the PiP restore callback + if #available(iOS 15.0, *), PictureInPictureManager.shared.isPiPActive { + PictureInPictureManager.shared.onRestoreUserInterface?() + } } func sceneDidEnterBackground(_ scene: UIScene) { diff --git a/OpenParsec/StreamRecovery.swift b/OpenParsec/StreamRecovery.swift index 70ee6bd9..dcf66cd8 100644 --- a/OpenParsec/StreamRecovery.swift +++ b/OpenParsec/StreamRecovery.swift @@ -1,5 +1,23 @@ import Foundation +struct StreamRestoreGuard: Equatable { + private var deadline: TimeInterval? + + var isActive: Bool { deadline != nil } + + mutating func begin(at time: TimeInterval) { + deadline = time + 20 + } + + mutating func expire(at time: TimeInterval) { + if let deadline, time >= deadline { end() } + } + + mutating func end() { + deadline = nil + } +} + struct StreamRecoveryBudget { private(set) var attempts = 0 private(set) var lastAttemptAt: TimeInterval?