Skip to content

Capture Go panics in the main app and report them to Crashlytics #215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pappz merged 2 commits into from
Sep 3, 2026
Merged

Capture Go panics in the main app and report them to Crashlytics #215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
48a7906
Capture Go panics in the main app and report them to Crashlytics
pappz Sep 2, 2026
1ad1d0b
Keep netbird.err shared across processes and report backlogs one dump…
pappz Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions NetBird.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@
978FC4712EEDF167002D0EB8 /* AppLogger.swift in Sources */ = {isa = PBXBuildFile; fileRef = 978FC46F2EEDF167002D0EB8 /* AppLogger.swift */; };
978FC4722EEDF167002D0EB8 /* AppLogger.swift in Sources */ = {isa = PBXBuildFile; fileRef = 978FC46F2EEDF167002D0EB8 /* AppLogger.swift */; };
978FC4732EEDF167002D0EB8 /* AppLogger.swift in Sources */ = {isa = PBXBuildFile; fileRef = 978FC46F2EEDF167002D0EB8 /* AppLogger.swift */; };
A7C0DE012F9A0001001A2B3C /* GoCrashCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7C0DE002F9A0001001A2B3C /* GoCrashCapture.swift */; };
A7C0DE022F9A0001001A2B3C /* GoCrashCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7C0DE002F9A0001001A2B3C /* GoCrashCapture.swift */; };
A7C0DE032F9A0001001A2B3C /* GoCrashCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7C0DE002F9A0001001A2B3C /* GoCrashCapture.swift */; };
A7C0DE042F9A0001001A2B3C /* GoCrashCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7C0DE002F9A0001001A2B3C /* GoCrashCapture.swift */; };
978FC4742EEDF168002D0EB8 /* Platform.swift in Sources */ = {isa = PBXBuildFile; fileRef = 443782BD2EDF284A00F9FA94 /* Platform.swift */; };
9B6C03E5ED4245A597C0FBE7 /* iOSConnectionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8DE6A20955914E68BFADDEED /* iOSConnectionView.swift */; };
9CC0E000AE3F165CA72FD465 /* AppLoggerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AA7193B3AE82DF185EDEB1B /* AppLoggerTests.swift */; };
Expand Down Expand Up @@ -368,6 +372,7 @@
8DE6A20955914E68BFADDEED /* iOSConnectionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = iOSConnectionView.swift; sourceTree = "<group>"; };
91FA1F06D3375864C74EAB3B /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS18.0.sdk/System/Library/Frameworks/Foundation.framework; sourceTree = DEVELOPER_DIR; };
978FC46F2EEDF167002D0EB8 /* AppLogger.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppLogger.swift; sourceTree = "<group>"; };
A7C0DE002F9A0001001A2B3C /* GoCrashCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GoCrashCapture.swift; sourceTree = "<group>"; };
9CD257EF78F038560FF3112D /* VPNOnDemandView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VPNOnDemandView.swift; sourceTree = "<group>"; };
A1B2C3D32F4A000100000001 /* VPNToggleView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VPNToggleView.swift; sourceTree = "<group>"; };
BB001A002F99000000000001 /* TroubleshootView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TroubleshootView.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -622,6 +627,7 @@
isa = PBXGroup;
children = (
978FC46F2EEDF167002D0EB8 /* AppLogger.swift */,
A7C0DE002F9A0001001A2B3C /* GoCrashCapture.swift */,
F1B292092EE0BC40001D91B8 /* GlobalConstants.swift */,
A1C3D5E72F000001001A2B3C /* WiFiOnDemandPolicy.swift */,
A1C3D5E82F000002001A2B3C /* CellularOnDemandPolicy.swift */,
Expand Down Expand Up @@ -1110,6 +1116,7 @@
BB3D4E022F4E5A0200D1E2F3 /* TVPreSharedKeyButton.swift in Sources */,
CC5F6A022F4E5A0300D1E2F3 /* TVQRCodeSheet.swift in Sources */,
978FC4732EEDF167002D0EB8 /* AppLogger.swift in Sources */,
A7C0DE012F9A0001001A2B3C /* GoCrashCapture.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1137,6 +1144,7 @@
44F3E3942EE2151100C87FEC /* ConnectionListener.swift in Sources */,
44F3E38C2EE214E300C87FEC /* NetBirdAdapter.swift in Sources */,
978FC4722EEDF167002D0EB8 /* AppLogger.swift in Sources */,
A7C0DE022F9A0001001A2B3C /* GoCrashCapture.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1165,6 +1173,7 @@
505118CF2AD96ECA003027D3 /* x25519.c in Sources */,
F1B292082EE0AC2A001D91B8 /* EnvVarPackager.swift in Sources */,
978FC4712EEDF167002D0EB8 /* AppLogger.swift in Sources */,
A7C0DE032F9A0001001A2B3C /* GoCrashCapture.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1218,6 +1227,7 @@
A1C3D5EC2F000006001A2B3C /* WiFiOnDemandPolicy.swift in Sources */,
A1C3D5F02F00000A001A2B3C /* CellularOnDemandPolicy.swift in Sources */,
978FC4702EEDF167002D0EB8 /* AppLogger.swift in Sources */,
A7C0DE042F9A0001001A2B3C /* GoCrashCapture.swift in Sources */,
978FC4742EEDF168002D0EB8 /* Platform.swift in Sources */,
9B6C03E5ED4245A597C0FBE7 /* iOSConnectionView.swift in Sources */,
962925F1DAA24D40B98D395B /* iOSPeersView.swift in Sources */,
Expand Down
30 changes: 30 additions & 0 deletions NetBird/Source/App/NetBirdApp.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import SwiftUI
import FirebaseCore
import FirebaseCrashlytics
import Combine
import UserNotifications
import NetBirdSDK
Expand Down Expand Up @@ -38,6 +39,31 @@ private func configureFirebaseIfNeeded() {
}
}

/// Forwards Go crash output left behind by a previous run to Crashlytics.
///
/// A Go panic aborts the process, and the crash Crashlytics records for it ends
/// at the Go stack switch with no panicking frames. The panic text and goroutine
/// dump only exist in netbird.err (see GoCrashCapture), so on the next launch
/// they are attached to a non-fatal whose headline is the panic line itself.
private func reportPreviousGoCrashIfNeeded() {
guard FirebaseApp.app() != nil,
let output = GoCrashCapture.takeUnreportedCrashOutput() else { return }

let headline = output
.split(whereSeparator: \.isNewline)
.first { $0.hasPrefix("panic:") || $0.hasPrefix("fatal error:") }
.map(String.init) ?? "Go runtime crash"

let crashlytics = Crashlytics.crashlytics()
crashlytics.log(output)
crashlytics.record(error: NSError(
domain: "io.netbird.GoCrash",
code: 1,
userInfo: [NSLocalizedDescriptionKey: headline]
))
AppLogger.shared.log("Reported Go crash output from a previous session to Crashlytics: \(headline)")
}

#if os(iOS)
extension Notification.Name {
static let netbirdLoginNotificationTapped = Notification.Name("io.netbird.loginNotificationTapped")
Expand All @@ -49,6 +75,7 @@ class AppDelegate: NSObject, UIApplicationDelegate, UNUserNotificationCenterDele
didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
) -> Bool {
configureFirebaseIfNeeded()
reportPreviousGoCrashIfNeeded()

let center = UNUserNotificationCenter.current()
center.delegate = self
Expand Down Expand Up @@ -98,9 +125,12 @@ struct NetBirdApp: App {
#endif

init() {
// Must run before any Go SDK call so a Go panic during startup is captured too.
GoCrashCapture.redirect()
// Configure Firebase on main thread as required by Firebase
#if os(tvOS)
configureFirebaseIfNeeded()
reportPreviousGoCrashIfNeeded()
#endif
}

Expand Down
120 changes: 120 additions & 0 deletions NetbirdKit/GoCrashCapture.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
//
// GoCrashCapture.swift
// NetBird
//

import Foundation

/// Keeps the Go runtime's crash output around after the process is gone.
///
/// The Go runtime writes panic messages and fatal-error goroutine dumps to fd 2
/// right before it aborts the process. Neither the main app nor the network
/// extension keeps stderr anywhere, and the resulting SIGABRT crash report stops
/// at the Go stack switch (`runtime.asmcgocall`) without the panicking Go frames —
/// so after a crash this file is the only place the panic reason can be recovered
/// from. `redirect()` points fd 2 at "netbird.err" in the app group container,
/// next to logfile.log, where the debug bundle generator already picks it up
/// (see BundleGenerator.addLogfile in netbird-core).
///
/// Both processes append to the same file. `takeUnreportedCrashOutput()` lets the
/// main app forward what was written since its previous report without truncating
/// the file, which the debug bundle still relies on.
public enum GoCrashCapture {
private static let fileName = "netbird.err"
private static let maxFileSize: UInt64 = 5 * 1024 * 1024
/// Crashlytics keeps at most 64 KB of logs per session; stay under it so the
/// panic header at the top of the dump is never the part that gets dropped.
private static let maxReportSize = 48 * 1024
private static let reportedOffsetKey = "io.netbird.goCrashCapture.reportedOffset"
private static let crashMarkers = ["panic:", "fatal error:"]

private static var fileURL: URL? {
FileManager.default
.containerURL(forSecurityApplicationGroupIdentifier: GlobalConstants.userPreferencesSuiteName)?
.appendingPathComponent(fileName)
}

private static let redirectOnce: Void = {
guard let errLogURL = fileURL else {
AppLogger.shared.log("stderr redirect: app group container unavailable")
return
}

let fd = open(errLogURL.path, O_WRONLY | O_CREAT | O_APPEND, 0o644)
guard fd >= 0 else {
AppLogger.shared.log("stderr redirect: failed to open \(errLogURL.path), errno=\(errno)")
return
}

var info = stat()
if fstat(fd, &info) == 0, info.st_size > 0 {
let size = UInt64(info.st_size)
// Surface a previous session's crash output before appending to it.
AppLogger.shared.log("stderr redirect: netbird.err has \(size) bytes from a previous session (possible crash dump)")
// Cap growth across sessions once it grows beyond 5 MB. Truncate in place
// rather than unlinking: the other process may hold fd 2 on this inode, and
// unlinking would send its future writes into an orphan nobody can read.
if size > maxFileSize {
AppLogger.shared.log("stderr redirect: netbird.err exceeds 5 MB cap, truncating")
if ftruncate(fd, 0) != 0 {
AppLogger.shared.log("stderr redirect: failed to truncate netbird.err, errno=\(errno)")
}
}
}

dup2(fd, STDERR_FILENO)
if fd != STDERR_FILENO {
close(fd)
}

let marker = "\n=== stderr redirect active pid=\(getpid()) at \(ISO8601DateFormatter().string(from: Date())) ===\n"
marker.withCString { _ = write(STDERR_FILENO, $0, strlen($0)) }
AppLogger.shared.log("stderr redirect: fd 2 -> netbird.err in app group container")
}()

/// Redirects this process's stderr into netbird.err. Safe to call repeatedly;
/// only the first call has an effect. Must run before any Go SDK call so a
/// panic during startup is captured too.
public static func redirect() {
_ = redirectOnce
}

/// Returns the next chunk of crash output appended to netbird.err since the
/// previous call, or nil when nothing new looks like a Go panic or fatal
/// error. The consumed range is remembered so the same dump is never reported
/// twice, even though the file itself is left untouched. Only the returned
/// chunk is consumed: anything after it stays unread for the next call, so a
/// backlog of several dumps is reported one launch at a time instead of being
/// skipped past.
public static func takeUnreportedCrashOutput() -> String? {
guard let errLogURL = fileURL,
let handle = try? FileHandle(forReadingFrom: errLogURL) else {
return nil
}
defer { try? handle.close() }

guard let fileSize = try? handle.seekToEnd() else { return nil }
let defaults = UserDefaults.standard
var offset = UInt64(max(0, defaults.integer(forKey: reportedOffsetKey)))
// The file was reset (or replaced) since the last report; start over.
if offset > fileSize {
offset = 0
}
defer { defaults.set(Int(offset), forKey: reportedOffsetKey) }

while offset < fileSize {
guard (try? handle.seek(toOffset: offset)) != nil,
let data = try? handle.read(upToCount: min(Int(fileSize - offset), maxReportSize)),
!data.isEmpty else {
return nil
}
offset += UInt64(data.count)

let output = String(decoding: data, as: UTF8.self)
if crashMarkers.contains(where: output.contains) {
return output
}
}
return nil
}
}
43 changes: 1 addition & 42 deletions NetbirdNetworkExtension/PacketTunnelProvider.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,52 +12,11 @@ import os
import UserNotifications
import WidgetKit

/// One-time (per process) redirect of stderr (fd 2) into "netbird.err" in the app
/// group container. The Go runtime writes panic messages and fatal-error goroutine
/// dumps to fd 2, which an app extension otherwise discards — after a SIGABRT crash
/// this file is the only place the panic reason can be recovered from.
/// The file lives next to logfile.log, so the debug bundle generator picks it up
/// automatically as "netbird.err" (see BundleGenerator.addLogfile in netbird-core).
private let stderrRedirectOnce: Void = {
let fileManager = FileManager.default
guard let groupURL = fileManager.containerURL(forSecurityApplicationGroupIdentifier: GlobalConstants.userPreferencesSuiteName) else {
AppLogger.shared.log("stderr redirect: app group container unavailable")
return
}
let errLogURL = groupURL.appendingPathComponent("netbird.err")

if let attrs = try? fileManager.attributesOfItem(atPath: errLogURL.path),
let size = attrs[.size] as? UInt64, size > 0 {
// Surface a previous session's crash output before appending to it.
AppLogger.shared.log("stderr redirect: netbird.err has \(size) bytes from a previous session (possible crash dump)")
// Cap growth across sessions: reset once it grows beyond 5 MB.
if size > 5 * 1024 * 1024 {
AppLogger.shared.log("stderr redirect: netbird.err exceeds 5 MB cap, resetting")
try? fileManager.removeItem(at: errLogURL)
}
}

let fd = open(errLogURL.path, O_WRONLY | O_CREAT | O_APPEND, 0o644)
guard fd >= 0 else {
AppLogger.shared.log("stderr redirect: failed to open \(errLogURL.path), errno=\(errno)")
return
}
dup2(fd, STDERR_FILENO)
if fd != STDERR_FILENO {
close(fd)
}

let marker = "\n=== stderr redirect active pid=\(getpid()) at \(ISO8601DateFormatter().string(from: Date())) ===\n"
marker.withCString { _ = write(STDERR_FILENO, $0, strlen($0)) }
AppLogger.shared.log("stderr redirect: fd 2 -> netbird.err in app group container")
}()


class PacketTunnelProvider: NEPacketTunnelProvider {

override init() {
// Must run before any Go SDK call so a Go panic during startup is captured too.
_ = stderrRedirectOnce
GoCrashCapture.redirect()
super.init()
}

Expand Down
Loading