diff --git a/CLAUDE.md b/CLAUDE.md index 4e5de07..351a16d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -242,10 +242,15 @@ The codebase follows a modular architecture: ### Core Components 1. **main.swift** - Entry point using Swift ArgumentParser - - Reads from stdin and coordinates parsing/output + - Reads stdin through `StreamingLineReader` and coordinates parsing/output - Outputs JSON or TOON format (controlled by `--format` / `-f` flag) +1b. **StreamingLineReader.swift** - Bounded stdin reader + - Reads 64 KiB chunks and frames lines on the newline byte (CRLF input keeps a trailing `\r`) + - Lines longer than `LineParser.maximumLineBytes` (64 KiB) are ignored and reported to stderr + 2. **OutputParser.swift** - Core parsing logic + - `StreamingOutputParser` accumulates events line-by-line; `OutputParser` wraps it for complete input - `OutputParser` class with regex-based line parsing - Defines data structures: `BuildResult`, `BuildSummary`, `BuildError`, `BuildWarning`, `FailedTest`, `SlowTest`, `CodeCoverage`, `FileCoverage`, `BuildInfo`, `TargetBuildInfo`, `Executable` - Pattern matching for various Xcode/SPM output formats diff --git a/Sources/XCSiftCore/LineParser.swift b/Sources/XCSiftCore/LineParser.swift index 543d5b0..ce92669 100644 --- a/Sources/XCSiftCore/LineParser.swift +++ b/Sources/XCSiftCore/LineParser.swift @@ -67,8 +67,8 @@ public enum LineResult { /// /// ```swift /// var parser = LineParser() -/// for line in output.split(separator: "\n") { -/// if case .consumed(let event) = parser.feed(String(line)) { +/// for line in lines { +/// if case .consumed(let event) = parser.feed(line) { /// handle(event) /// } /// } @@ -76,8 +76,10 @@ public enum LineResult { /// ``` public struct LineParser: Sendable { - /// Maximum UTF-8 byte length accepted for one input line. - public static let maximumLineBytes = 5_000 + /// Maximum UTF-8 byte length accepted for one input line. Longer lines are ignored. + /// + /// The budget is in bytes, so a non-ASCII diagnostic spends two to four bytes per character. + public static let maximumLineBytes = 64 * 1024 // MARK: - Multi-line linker state private var currentLinkerArchitecture: String? @@ -753,9 +755,10 @@ public struct LineParser: Sendable { private func hasPotentialLinkerPrefix(_ line: String) -> Bool { for byte in line.utf8 { switch byte { - case 0x20, 0x09: + case UInt8(ascii: " "), UInt8(ascii: "\t"): continue - case 0x22, 0x55, 0x64, 0x66, 0x6C: + // First byte of `"symbol"`, `Undefined symbols`, `duplicate symbol`, `ld:`. + case UInt8(ascii: "\""), UInt8(ascii: "U"), UInt8(ascii: "d"), UInt8(ascii: "l"): return true default: return false @@ -983,10 +986,25 @@ public struct LineParser: Sendable { // MARK: - Error / Warning Parsing - private static let warningFormatUTF8 = Array(XcodebuildSymbols.warningFormat.utf8) + /// A literal marker kept in both forms so the search never re-encodes it per line. + struct UTF8Needle: Sendable { + let bytes: [UInt8] + let text: String + + init(_ text: String) { + self.bytes = Array(text.utf8) + self.text = text + } + } + + static let warningFormatNeedle = UTF8Needle(XcodebuildSymbols.warningFormat) + static let errorFormatNeedle = UTF8Needle(XcodebuildSymbols.errorFormat) + static let xctestBundleNeedle = UTF8Needle(".xctest") - private func warningFormatRange(in line: String) -> Range? { - let marker = Self.warningFormatUTF8 + /// Byte-exact substring search. `String.range(of:)` is Unicode-aware and dominates the + /// profile when every line of a large log runs several searches. + static func range(of needle: UTF8Needle, in line: String) -> Range? { + let marker = needle.bytes if let byteOffset = line.utf8.withContiguousStorageIfAvailable({ bytes in guard bytes.count >= marker.count else { return -1 } @@ -1006,7 +1024,31 @@ public struct LineParser: Sendable { return lowerBound ..< upperBound } - return line.range(of: XcodebuildSymbols.warningFormat, options: .literal) + return line.range(of: needle.text, options: .literal) + } + + static func contains(_ needle: UTF8Needle, in line: String) -> Bool { + range(of: needle, in: line) != nil + } + + /// Splits the `file:line` prefix that precedes a diagnostic marker. + private func parseFileAndLine(_ prefix: Substring) -> (file: Substring, line: Int?) { + guard let finalColon = prefix.lastIndex(of: ":"), + let number = Int(prefix[prefix.index(after: finalColon)...]) + else { + return (prefix, nil) + } + return (prefix[.. (file: Substring, line: Int?, column: Int?) { + let (withoutFinal, finalNumber) = parseFileAndLine(prefix) + guard let column = finalNumber else { return (prefix, nil, nil) } + + let (file, lineNumber) = parseFileAndLine(withoutFinal) + guard let lineNumber else { return (withoutFinal, column, nil) } + return (file, lineNumber, column) } private func isJSONLikeLine(_ line: String) -> Bool { @@ -1047,41 +1089,36 @@ public struct LineParser: Sendable { if isRuntimeLogNoise(line) { return nil } if line.hasPrefix(" "), line.contains("|") || line.contains("`") { return nil } - if let errorRange = line.range(of: XcodebuildSymbols.errorFormat) { - let beforeError = String(line[..= 3, let lineNum = Int(components[components.count - 2]), - let colNum = Int(components[components.count - 1]) - { - let file = components[0 ..< (components.count - 2)].joined(separator: ":") - return BuildError(file: file, line: lineNum, message: message, column: colNum) - } else if components.count >= 2, let lineNum = Int(components[components.count - 1]) { - let file = components[0 ..< (components.count - 1)].joined(separator: ":") - return BuildError(file: file, line: lineNum, message: message, column: nil) - } else { - return BuildError(file: beforeError, line: nil, message: message, column: nil) - } + if let errorRange = Self.range(of: Self.errorFormatNeedle, in: line) { + let location = parseLocation(line[..= 2, let lineNum = Int(components[components.count - 1]) { - let file = components[0 ..< (components.count - 1)].joined(separator: ":") - return BuildError(file: file, line: lineNum, message: message, column: nil) - } else { - return BuildError(file: beforeError, line: nil, message: message, column: nil) - } + let (file, lineNumber) = parseFileAndLine(line[..= 2, let lineNum = Int(components[components.count - 1]) { - let file = components[0 ..< (components.count - 1)].joined(separator: ":") - return BuildError(file: file, line: lineNum, message: "Fatal error", column: nil) + let beforeFatal = line.dropLast(XcodebuildSymbols.fatalErrorSuffix.count) + let (file, lineNumber) = parseFileAndLine(beforeFatal) + if let lineNumber { + return BuildError( + file: String(file), + line: lineNumber, + message: "Fatal error", + column: nil + ) } } @@ -1105,33 +1142,14 @@ public struct LineParser: Sendable { if isRuntimeLogNoise(line) { return nil } if line.hasPrefix(" "), line.contains("|") || line.contains("`") { return nil } - if let warningRange = warningFormatRange(in: line) { - let beforeWarning = line[.. = [] var seenWarnings: Set = [] - var seenCompactWarnings: Set = [] - var lastCountOnlyWarning: WarningKey? var warningCount = 0 var seenErrors: Set = [] var seenLinkerErrors: Set = [] @@ -156,7 +121,9 @@ public struct StreamingOutputParser { /// Feeding after ``finish(coverage:)`` is a programmer error. public mutating func feed(_ line: String) { precondition(finishedResult == nil, "Cannot feed a finished StreamingOutputParser") - if shouldDiscoverTestedTarget, testedTarget == nil { + if shouldDiscoverTestedTarget, testedTarget == nil, + LineParser.contains(LineParser.xctestBundleNeedle, in: line) + { testedTarget = Self.extractTestedTarget(fromLine: line) } if case .consumed(let event) = lineParser.feed(line) { @@ -327,15 +294,7 @@ public struct StreamingOutputParser { case .warning(let w): let key = WarningKey(file: w.file, line: w.line, message: w.message) - let inserted: Bool - if shouldRetainWarnings { - inserted = state.seenWarnings.insert(key).inserted - } else { - guard state.lastCountOnlyWarning != key else { return } - state.lastCountOnlyWarning = key - inserted = state.seenCompactWarnings.insert(CompactWarningKey(w)).inserted - } - guard inserted else { return } + guard state.seenWarnings.insert(key).inserted else { return } state.warningCount += 1 if shouldRetainWarnings { state.warnings.append(w) @@ -530,6 +489,13 @@ public class OutputParser { public init() {} + /// Splits on the newline byte. `String.split(separator: "\n")` never matches a CRLF line + /// ending, because Swift treats `\r\n` as one `Character`. + private static func lines(of input: String) -> [String] { + input.utf8.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false) + .map { String(decoding: $0, as: UTF8.self) } + } + /// Parses raw xcodebuild or SPM output and returns a structured ``BuildResult``. /// /// Each invocation uses a fresh streaming session, so an `OutputParser` instance can be reused @@ -566,8 +532,8 @@ public class OutputParser { xcbeautify: xcbeautify ) - for line in input.split(separator: "\n", omittingEmptySubsequences: false) { - parser.feed(String(line)) + for line in Self.lines(of: input) { + parser.feed(line) } let result = parser.finish(coverage: coverage) @@ -579,10 +545,8 @@ public class OutputParser { /// /// A `.xctest` suite name such as `MyAppTests.xctest` resolves to `MyApp`. public func extractTestedTarget(from input: String) -> String? { - for line in input.split(separator: "\n") { - if let testedTarget = StreamingOutputParser.extractTestedTarget( - fromLine: String(line) - ) { + for line in Self.lines(of: input) { + if let testedTarget = StreamingOutputParser.extractTestedTarget(fromLine: line) { return testedTarget } } diff --git a/Sources/xcsift/main.swift b/Sources/xcsift/main.swift index 42b11e4..9c68d42 100644 --- a/Sources/xcsift/main.swift +++ b/Sources/xcsift/main.swift @@ -289,6 +289,13 @@ struct XCSift: ParsableCommand { throw ExitCode.failure } + if inputScan.oversizedLinesDropped > 0 { + writeToStderr( + "hint: Ignored \(inputScan.oversizedLinesDropped) input line(s) longer than " + + "\(LineParser.maximumLineBytes) bytes.\n" + ) + } + // Check if input is empty if !inputScan.containsNonWhitespace { throw ValidationError( diff --git a/Sources/xcsift/xcsift.docc/Usage.md b/Sources/xcsift/xcsift.docc/Usage.md index efa9cd5..2d5c3ed 100644 --- a/Sources/xcsift/xcsift.docc/Usage.md +++ b/Sources/xcsift/xcsift.docc/Usage.md @@ -329,6 +329,13 @@ xcodebuild build 2>&1 | xcsift --Werror --exit-on-failure swift test 2>&1 | xcsift -W -E -f toon ``` +## Large Build Logs + +xcsift parses stdin as it arrives, so memory stays bounded no matter how large the log is. + +One input line is limited to 64 KiB of UTF-8. Longer lines are ignored, and xcsift reports the +count to stderr. + ## Exit Codes - `0` — Build succeeded (or xcsift completed normally without `--exit-on-failure`) diff --git a/Tests/XCSiftCoreTests/LineParserTests.swift b/Tests/XCSiftCoreTests/LineParserTests.swift index d2d3447..db4eb4b 100644 --- a/Tests/XCSiftCoreTests/LineParserTests.swift +++ b/Tests/XCSiftCoreTests/LineParserTests.swift @@ -17,11 +17,22 @@ final class LineParserTests: XCTestCase { XCTAssertEqual(parser.feed("note: some note message"), .ignored) } - func testLineLengthLimitUsesUTF8Bytes() { + func testLongNonASCIIDiagnosticStaysWithinTheLineBudget() { var parser = LineParser() let line = "main.swift:1:1: error: " + String(repeating: "é", count: 3_000) - XCTAssertEqual(parser.feed(line), .ignored) + guard case .consumed(let event) = parser.feed(line), case .error(let error) = event else { + return XCTFail("Expected .consumed(.error), got \(parser.feed(line))") + } + XCTAssertEqual(error.file, "main.swift") + XCTAssertEqual(error.message.count, 3_000) + } + + func testLineLengthLimitUsesUTF8Bytes() { + var parser = LineParser() + let padding = String(repeating: "é", count: LineParser.maximumLineBytes / 2) + + XCTAssertEqual(parser.feed("main.swift:1:1: error: " + padding), .ignored) } // MARK: - Error @@ -102,6 +113,19 @@ final class LineParserTests: XCTestCase { } } + func testErrorMarkerRequiresExactASCIIBytes() { + let lines = [ + "Foo.swift:1:1: error:no space", + "Foo.swift:1:1: Error: wrong case", + "Foo.swift:1:1: error:\u{301} combining mark before space", + ] + + for line in lines { + var parser = LineParser() + XCTAssertEqual(parser.feed(line), .ignored, "Unexpected error for: \(line)") + } + } + // MARK: - Failed test func testFailedTest() { diff --git a/Tests/XCSiftCoreTests/StreamingOutputParserTests.swift b/Tests/XCSiftCoreTests/StreamingOutputParserTests.swift index 11a76ee..2356a37 100644 --- a/Tests/XCSiftCoreTests/StreamingOutputParserTests.swift +++ b/Tests/XCSiftCoreTests/StreamingOutputParserTests.swift @@ -147,6 +147,21 @@ final class StreamingOutputParserTests: XCTestCase { XCTAssertEqual(try encoder.encode(first), try encoder.encode(second)) } + func testCompleteInputParserSplitsCRLFLineEndings() { + let parser = OutputParser() + let input = [ + "First.swift:1:1: error: broken", + "Second.swift:2:1: error: also broken", + "** BUILD FAILED **", + ].joined(separator: "\r\n") + + let result = parser.parse(input: input) + + XCTAssertEqual(result.status, "failed") + XCTAssertEqual(result.summary.errors, 2) + XCTAssertEqual(result.errors.first?.file, "First.swift") + } + func testCompleteInputParserDoesNotLeakStateAcrossCalls() { let parser = OutputParser() diff --git a/Tests/xcsiftTests/StreamingLineReaderTests.swift b/Tests/xcsiftTests/StreamingLineReaderTests.swift index e92c03f..ba47974 100644 --- a/Tests/xcsiftTests/StreamingLineReaderTests.swift +++ b/Tests/xcsiftTests/StreamingLineReaderTests.swift @@ -93,6 +93,21 @@ final class StreamingLineReaderTests: XCTestCase { XCTAssertTrue(scan.containsNonWhitespace) } + func testInvalidUTF8BytesDoNotDiscardTheLine() throws { + var source = ChunkSource( + chunks: [Data("main.swift:1:1: error: bad ".utf8) + Data([0xFF]) + Data(" byte\n".utf8)], + log: EventLog() + ) + var reader = StreamingLineReader() + var lines: [String] = [] + + let scan = try reader.consume(from: &source) { lines.append($0) } + + XCTAssertEqual(lines.first?.hasPrefix("main.swift:1:1: error: bad"), true) + XCTAssertEqual(lines.first?.contains("byte"), true) + XCTAssertTrue(scan.containsNonWhitespace) + } + func testPreservesEmptyLinesAndUnterminatedFinalLine() throws { var source = ChunkSource( chunks: [Data("\n\nlast line".utf8)],