From e60b4af27e335c05789c3f57cc7989faadc73f8d Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Sun, 25 Oct 2026 16:05:58 +0000 Subject: [PATCH] Preserve UTF-8 characters in process log output Validate and preserve complete UTF-8 sequences in standard JSON and XML process logs instead of replacing every high-bit byte. Retain question-mark replacement for invalid bytes, add regression coverage, and document the supported encoding. Addresses #121. --- .../LogMonitorTests/LogMonitorTests.cpp | 89 +++++++++++++++++++ LogMonitor/docs/README.md | 6 ++ LogMonitor/src/LogMonitor/ProcessMonitor.cpp | 29 +++++- 3 files changed, 121 insertions(+), 3 deletions(-) diff --git a/LogMonitor/LogMonitorTests/LogMonitorTests.cpp b/LogMonitor/LogMonitorTests/LogMonitorTests.cpp index b85b7380..631f484c 100644 --- a/LogMonitor/LogMonitorTests/LogMonitorTests.cpp +++ b/LogMonitor/LogMonitorTests/LogMonitorTests.cpp @@ -79,4 +79,93 @@ namespace LogMonitorTests Assert::AreEqual(expectedOutput, realOutput); } }; + + TEST_CLASS(ProcessMonitorTests) + { + std::wstring previousLogFormat; + std::wstring previousCustomFormat; + + const std::string utf8Message = + "ASCII \xc2\xa1 \xe2\x82\xac \xe6\x97\xa5\xe6\x9c\xac \xf0\x9f\x98\x80 \xef\xbf\xbd"; + + void CheckStandardFormats(const std::string& input, const std::string& expected) + { + loggingformat = L"JSON"; + std::string formatted = FormatProcessLog(input); + Assert::AreEqual( + std::string("{\"Source\":\"Process\",\"LogEntry\":{\"Logline\":\"") + + expected + "\"},\"SchemaVersion\":\"1.0.0\"}\n", + formatted); + nlohmann::json parsed = nlohmann::json::parse(formatted); + Assert::AreEqual(expected, parsed.at("LogEntry").at("Logline").get()); + + loggingformat = L"XML"; + Assert::AreEqual( + std::string("Process") + + expected + "\n", + FormatProcessLog(input)); + } + + public: + TEST_METHOD_INITIALIZE(InitializeProcessMonitorTests) + { + previousLogFormat = loggingformat; + previousCustomFormat = processCustomLogFormat; + } + + TEST_METHOD_CLEANUP(CleanupProcessMonitorTests) + { + loggingformat = previousLogFormat; + processCustomLogFormat = previousCustomFormat; + } + + TEST_METHOD(TestStandardFormatsPreserveUtf8) + { + CheckStandardFormats(utf8Message, utf8Message); + std::string longMessage = std::string(4095, 'a') + utf8Message; + CheckStandardFormats(longMessage, longMessage); + } + + TEST_METHOD(TestStandardFormatsPreserveAsciiAndNullReplacement) + { + CheckStandardFormats("", ""); + CheckStandardFormats("plain ASCII 123", "plain ASCII 123"); + std::string withNull = "before"; + withNull.push_back('\0'); + withNull += "after"; + CheckStandardFormats(withNull, "before?after"); + } + + TEST_METHOD(TestStandardFormatsReplaceInvalidUtf8) + { + struct TestCase + { + const char* input; + const char* expected; + }; + const TestCase cases[] = { + {"\x80", "?"}, + {"\xff", "?"}, + {"\xc0\xaf", "??"}, + {"\xed\xa0\x80", "???"}, + {"\xf4\x90\x80\x80", "????"}, + {"\xe2\x28\xa1", "?(?"}, + {"\xf0\x9f\x98", "???"}, + }; + for (const auto& testCase : cases) + { + CheckStandardFormats(testCase.input, testCase.expected); + } + CheckStandardFormats(utf8Message + "\xff" + utf8Message, + utf8Message + "?" + utf8Message); + } + + TEST_METHOD(TestCustomFormatPreservesUtf8) + { + loggingformat = L"Custom"; + processCustomLogFormat = L"%Source%: %Message%"; + Assert::AreEqual(std::string("Process: ") + utf8Message + "\n", + FormatProcessLog(utf8Message)); + } + }; } diff --git a/LogMonitor/docs/README.md b/LogMonitor/docs/README.md index af845944..9e3b95b3 100644 --- a/LogMonitor/docs/README.md +++ b/LogMonitor/docs/README.md @@ -365,6 +365,12 @@ WARNING: Failed to parse configuration file. Error retrieving source attributes. Currently, this one does not need any configuration, it basically streams the output of tthe process/command that is provided as the argument to `LogMonitor.exe`. For the given _entryproint_ command, a child process is created and links its STDIN and STDOUT to the `LogMonitor` process (through a _pipe_). +Child processes should write UTF-8 output. JSON and XML process logs preserve valid +UTF-8 characters, including characters split across pipe reads. Invalid UTF-8 +bytes are replaced with `?`; valid characters elsewhere in the line are +preserved. Process monitoring does not automatically detect or convert legacy +Windows code pages. + ### Examples ```dockerfile diff --git a/LogMonitor/src/LogMonitor/ProcessMonitor.cpp b/LogMonitor/src/LogMonitor/ProcessMonitor.cpp index 2dc4328f..1160af18 100644 --- a/LogMonitor/src/LogMonitor/ProcessMonitor.cpp +++ b/LogMonitor/src/LogMonitor/ProcessMonitor.cpp @@ -222,11 +222,34 @@ std::string FormatStandardLog(const std::string& inputLine) { suffix = "\"},\"SchemaVersion\":\"1.0.0\"}\n"; } - // Sanitize the log line + // Preserve complete UTF-8 sequences; retain '?' for NUL and invalid bytes. std::string sanitized; sanitized.reserve(inputLine.size()); - for (char c : inputLine) { - sanitized += (c > 0) ? c : '?'; + for (size_t i = 0; i < inputLine.size();) { + unsigned char c = static_cast(inputLine[i]); + if (c > 0 && c < 0x80) { + sanitized += inputLine[i++]; + continue; + } + + int sequenceLength = 0; + if (c >= 0xc2 && c <= 0xdf) { + sequenceLength = 2; + } else if (c >= 0xe0 && c <= 0xef) { + sequenceLength = 3; + } else if (c >= 0xf0 && c <= 0xf4) { + sequenceLength = 4; + } + + if (sequenceLength > 0 && + inputLine.size() - i >= static_cast(sequenceLength) && + Utility::IsTextUTF8(inputLine.data() + i, sequenceLength)) { + sanitized.append(inputLine, i, sequenceLength); + i += sequenceLength; + } else { + sanitized += '?'; + ++i; + } } std::string logLine = prefix + sanitized + suffix;