Preserve UTF-8 characters in process log output

LogMonitor/LogMonitorTests/LogMonitorTests.cpp

@@ -79,4 +79,93 @@ namespace LogMonitorTests
             Assert::AreEqual(expectedOutput, realOutput);
         }
     };
+
+    TEST_CLASS(ProcessMonitorTests)
+    {
+        std::wstring previousLogFormat;
+        std::wstring previousCustomFormat;
+
+        const std::string utf8Message =
+            "ASCII \xc2\xa1 \xe2\x82\xac \xe6\x97\xa5\xe6\x9c\xac \xf0\x9f\x98\x80 \xef\xbf\xbd";
+
+        void CheckStandardFormats(const std::string& input, const std::string& expected)
+        {
+            loggingformat = L"JSON";
+            std::string formatted = FormatProcessLog(input);
+            Assert::AreEqual(
+                std::string("{\"Source\":\"Process\",\"LogEntry\":{\"Logline\":\"") +
+                    expected + "\"},\"SchemaVersion\":\"1.0.0\"}\n",
+                formatted);
+            nlohmann::json parsed = nlohmann::json::parse(formatted);
+            Assert::AreEqual(expected, parsed.at("LogEntry").at("Logline").get<std::string>());
+
+            loggingformat = L"XML";
+            Assert::AreEqual(
+                std::string("<Log><Source>Process</Source><LogEntry><Logline>") +
+                    expected + "</Logline></LogEntry></Log>\n",
+                FormatProcessLog(input));
+        }
+
+    public:
+        TEST_METHOD_INITIALIZE(InitializeProcessMonitorTests)
+        {
+            previousLogFormat = loggingformat;
+            previousCustomFormat = processCustomLogFormat;
+        }
+
+        TEST_METHOD_CLEANUP(CleanupProcessMonitorTests)
+        {
+            loggingformat = previousLogFormat;
+            processCustomLogFormat = previousCustomFormat;
+        }
+
+        TEST_METHOD(TestStandardFormatsPreserveUtf8)
+        {
+            CheckStandardFormats(utf8Message, utf8Message);
+            std::string longMessage = std::string(4095, 'a') + utf8Message;
+            CheckStandardFormats(longMessage, longMessage);
+        }
+
+        TEST_METHOD(TestStandardFormatsPreserveAsciiAndNullReplacement)
+        {
+            CheckStandardFormats("", "");
+            CheckStandardFormats("plain ASCII 123", "plain ASCII 123");
+            std::string withNull = "before";
+            withNull.push_back('\0');
+            withNull += "after";
+            CheckStandardFormats(withNull, "before?after");
+        }
+
+        TEST_METHOD(TestStandardFormatsReplaceInvalidUtf8)
+        {
+            struct TestCase
+            {
+                const char* input;
+                const char* expected;
+            };
+            const TestCase cases[] = {
+                {"\x80", "?"},
+                {"\xff", "?"},
+                {"\xc0\xaf", "??"},
+                {"\xed\xa0\x80", "???"},
+                {"\xf4\x90\x80\x80", "????"},
+                {"\xe2\x28\xa1", "?(?"},
+                {"\xf0\x9f\x98", "???"},
+            };
+            for (const auto& testCase : cases)
+            {
+                CheckStandardFormats(testCase.input, testCase.expected);
+            }
+            CheckStandardFormats(utf8Message + "\xff" + utf8Message,
+                utf8Message + "?" + utf8Message);
+        }
+
+        TEST_METHOD(TestCustomFormatPreservesUtf8)
+        {
+            loggingformat = L"Custom";
+            processCustomLogFormat = L"%Source%: %Message%";
+            Assert::AreEqual(std::string("Process: ") + utf8Message + "\n",
+                FormatProcessLog(utf8Message));
+        }
+    };
 }

LogMonitor/docs/README.md

@@ -365,6 +365,12 @@ WARNING: Failed to parse configuration file. Error retrieving source attributes.
 
 Currently, this one does not need any configuration, it basically streams the output of tthe process/command that is provided as the argument to `LogMonitor.exe`. For the given _entryproint_ command, a child process is created and links its STDIN and STDOUT to the `LogMonitor` process (through a _pipe_).
 
+Child processes should write UTF-8 output. JSON and XML process logs preserve valid
+UTF-8 characters, including characters split across pipe reads. Invalid UTF-8
+bytes are replaced with `?`; valid characters elsewhere in the line are
+preserved. Process monitoring does not automatically detect or convert legacy
+Windows code pages.
+
 ### Examples
 
 ```dockerfile

LogMonitor/src/LogMonitor/ProcessMonitor.cpp

@@ -222,11 +222,34 @@ std::string FormatStandardLog(const std::string& inputLine) {
         suffix = "\"},\"SchemaVersion\":\"1.0.0\"}\n";
     }
 
-    // Sanitize the log line
+    // Preserve complete UTF-8 sequences; retain '?' for NUL and invalid bytes.
     std::string sanitized;
     sanitized.reserve(inputLine.size());
-    for (char c : inputLine) {
-        sanitized += (c > 0) ? c : '?';
+    for (size_t i = 0; i < inputLine.size();) {
+        unsigned char c = static_cast<unsigned char>(inputLine[i]);
+        if (c > 0 && c < 0x80) {
+            sanitized += inputLine[i++];
+            continue;
+        }
+
+        int sequenceLength = 0;
+        if (c >= 0xc2 && c <= 0xdf) {
+            sequenceLength = 2;
+        } else if (c >= 0xe0 && c <= 0xef) {
+            sequenceLength = 3;
+        } else if (c >= 0xf0 && c <= 0xf4) {
+            sequenceLength = 4;
+        }
+
+        if (sequenceLength > 0 &&
+            inputLine.size() - i >= static_cast<size_t>(sequenceLength) &&
+            Utility::IsTextUTF8(inputLine.data() + i, sequenceLength)) {
+            sanitized.append(inputLine, i, sequenceLength);
+            i += sequenceLength;
+        } else {
+            sanitized += '?';
+            ++i;
+        }
     }
 
     std::string logLine = prefix + sanitized + suffix;