Fix unnecessarily and erroneous whitespace changes when using --anonymize-passwords and --sensitive-words

netconan/sensitive_item_removal.py

@@ -162,7 +162,7 @@ def __init__(self, sensitive_words, salt, reserved_words=default_reserved_words)
     def anonymize(self, line):
         """Anonymize sensitive words from the input line."""
         if self.sens_regex.search(line) is not None:
-            leading, words, trailing = _split_line(line)
+            leading, words, trailing, whitespace_strings = _split_line(line)
             # Anonymize only words that do not match the conflicting (reserved) words
             words = [
                 (
@@ -174,6 +174,10 @@ def anonymize(self, line):
             ]
             # Restore leading and trailing whitespace since those were removed when splitting into words
             line = leading + " ".join(words) + trailing
+
+            if whitespace_strings:
+                line = _restore_spaces(line, whitespace_strings, leading, trailing)
+
         return line
 
     def _generate_conflicting_reserved_word_list(self, sensitive_words):
@@ -359,8 +363,8 @@ def replace_matching_item(
     reserved_words=default_reserved_words,
 ):
     """If line matches a regex, anonymize or remove the line."""
-    # Collapse whitespace to simplify regexes, also preserve leading and trailing whitespace
-    leading, words, trailing = _split_line(input_line)
+    # Collapse whitespace to simplify regexes, also preserve leading and trailing whitespace and store space counts
+    leading, words, trailing, whitespace_strings = _split_line(input_line)
     # Save enclosing text (like quotes) to avoid removing during anonymization
     leading, output_line, trailing = _extract_enclosing_text(
         " ".join(words), leading, trailing
@@ -404,10 +408,64 @@ def replace_matching_item(
         if match_found:
             break
 
-    # Restore leading and trailing whitespace for readability and context
-    return leading + output_line + trailing
+    if whitespace_strings:
+        line_to_return = _restore_spaces(
+            output_line, whitespace_strings, leading, trailing
+        )
+    else:
+        line_to_return = leading + output_line + trailing
+
+    return line_to_return
 
 
 def _split_line(line):
-    """Split line into leading whitespace, list of words, and trailing whitespace."""
-    return line[: -len(line.lstrip())], line.split(), line[len(line.rstrip()) :]
+    """Split a non-empty/non-blank line into leading and trailing whitespace, list of words, and list of whitespace strings between words."""
+    leading = line[: -len(line.lstrip())]
+    trailing = line[len(line.rstrip()) :]
+    words = line.strip().split()
+    whitespace_strings = []
+    i = 0
+    while i < len(line):
+        ws = ""
+        while i < len(line) and line[i] in " \t":
+            ws += line[i]
+            i += 1
+        if ws:
+            whitespace_strings.append(ws)
+        while i < len(line) and line[i] not in " \t":
+            i += 1
+    return leading, words, trailing, whitespace_strings
+
+
+def _restore_spaces(line, target_whitespace_strings, leading, trailing):
+    """Restore whitespace between words according to target_whitespace_strings. Leading/trailing characters are preserved."""
+    parts = line.split()
+    # Rebuild starting with leading
+    rebuilt = leading
+    # Determine whether to skip the first value in target_whitespace_strings due to leading
+    skip = 0
+    if leading and target_whitespace_strings:
+        # Only skip if all leading characters are whitespace and match the first string
+        if (
+            leading.replace(" ", "").replace("\t", "") == ""
+            and target_whitespace_strings[0] == leading
+        ):
+            skip = 1
+    # Rebuild the remaining words
+    for i, word in enumerate(parts):
+        rebuilt += word
+        if i < len(parts) - 1:
+            idx = i + skip
+            # TODO: For now, the else ' ' guard below is currently needed in a few
+            # tests due to both _split_line and _extract_enclosing_text doing
+            # some handling of leading/trailing. It would be good to remove the
+            # need for it.
+            spaces = (
+                target_whitespace_strings[idx]
+                if idx < len(target_whitespace_strings)
+                else " "
+            )
+            rebuilt += spaces
+    # Add on trailing
+    rebuilt += trailing
+    return rebuilt

tests/unit/test_sensitive_item_removal.py

@@ -623,7 +623,7 @@ def test_pwd_removal_preserve_context(regexes, config_line, anon_line):
     assert anon_line == replace_matching_item(regexes, config_line, pwd_lookup, SALT)
 
 
-@pytest.mark.parametrize("whitespace", [" ", "\t", "\n", " \t\n"])
+@pytest.mark.parametrize("whitespace", [" ", "  ", "\t", "\n", " \t\n"])
 def test_pwd_removal_preserve_leading_whitespace(regexes, whitespace):
     """Test leading whitespace is preserved in config lines."""
     config_line = "{whitespace}{line}".format(
@@ -634,7 +634,7 @@ def test_pwd_removal_preserve_leading_whitespace(regexes, whitespace):
     assert processed_line.startswith(whitespace)
 
 
-@pytest.mark.parametrize("whitespace", [" ", "\t", "\n", " \t\n"])
+@pytest.mark.parametrize("whitespace", [" ", "  ", "\t", "\n", " \t\n"])
 def test_pwd_removal_preserve_trailing_whitespace(regexes, whitespace):
     """Test trailing whitespace is preserved in config lines."""
     config_line = "{line}{whitespace}".format(
@@ -645,6 +645,80 @@ def test_pwd_removal_preserve_trailing_whitespace(regexes, whitespace):
     assert processed_line.endswith(whitespace)
 
 
+@pytest.mark.parametrize(
+    "whitespace", [" ", "  ", "   ", "          ", "\t", "  \t \t"]
+)
+def test_pwd_removal_preserve_single_inline_whitespace(regexes, whitespace):
+    """Test single inline whitespace including tabs is preserved in config lines."""
+    word = "foobar"
+    config_line = f"{word}{whitespace}{word}"
+    pwd_lookup = {}
+    processed_line = replace_matching_item(regexes, config_line, pwd_lookup, SALT)
+    assert processed_line == config_line
+
+
+@pytest.mark.parametrize(
+    "whitespace", [" ", "  ", "   ", "          ", "\t", "  \t \t"]
+)
+def test_pwd_removal_preserve_multiple_inline_whitespace(regexes, whitespace):
+    """Test multiple inline whitespace including tabs is preserved in config lines."""
+    word = "foobar"
+    config_line = f"{word}{whitespace}{word}{whitespace}{word}"
+    pwd_lookup = {}
+    processed_line = replace_matching_item(regexes, config_line, pwd_lookup, SALT)
+    assert processed_line == config_line
+
+
+@pytest.mark.parametrize(
+    "ws1, ws2",
+    [
+        (" ", "  "),
+        ("    ", "   "),
+        ("      ", "       "),
+        ("\t", "\t"),
+        ("\t", "\t "),
+        (" \t", "\t"),
+        (" \t ", "  \t  "),
+    ],
+)
+def test_pwd_removal_preserve_different_inline_whitespace(regexes, ws1, ws2):
+    """Test different inline whitespace sizes, including tabs, are preserved in config lines."""
+    word = "foobar"
+    config_line = f"{word}{ws1}{word}{ws2}{word}"
+    pwd_lookup = {}
+    processed_line = replace_matching_item(regexes, config_line, pwd_lookup, SALT)
+    assert processed_line == config_line
+
+
+@pytest.mark.parametrize(
+    "leading, ws1, ws2, trailing",
+    [
+        ("", " ", "  ", ""),
+        ("", "    ", "   ", ""),
+        ("", "      ", "       ", ""),
+        ("", "\t", "\t", ""),
+        ("", "\t", "\t ", ""),
+        ("", " \t", "\t", ""),
+        ("", " \t ", "  \t  ", ""),
+        ("  ", " ", "  ", ""),
+        ("\t", "    ", "   ", ""),
+        ("   ", "\t", "\t ", "\t"),
+        (" \t ", " \t", "\t", "  "),
+        ("", " ", "  ", "   "),
+        ("\t\t", "\t", "\t", "\t\t"),
+    ],
+)
+def test_pwd_removal_preserve_leading_inline_and_trailing_whitespace(
+    regexes, leading, ws1, ws2, trailing
+):
+    """Test leading, inline, and trailing whitespace including tabs are preserved in config lines."""
+    word = "foobar"
+    config_line = f"{leading}{word}{ws1}{word}{ws2}{word}{trailing}"
+    pwd_lookup = {}
+    processed_line = replace_matching_item(regexes, config_line, pwd_lookup, SALT)
+    assert processed_line == config_line
+
+
 @pytest.mark.parametrize("config_line,sensitive_text", sensitive_lines)
 @pytest.mark.parametrize(
     "prepend_text",