Fix encrypted-password anonymization to replace instead of scrub

netconan/default_pwd_regexes.py

@@ -122,7 +122,9 @@
     [(r"(key-string \d?)(.*)", None)],
     [(r"(message-digest-key \d+ md5 (7|encrypted)) (.*)", None)],
     [(r"(.*?neighbor.*?) (\S*) password (.*)", None)],
-    [(r"(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)", None)],
+    [(r"(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)", 1)],
+    # Juniper encrypted-password with capture group for hash-preserving anonymization.
+    [(r"(?P<prefix>(\S* )*encrypted-password )([^ ;]+)", 3)],
     # These are regexes for JUNOS
     # TODO(https://github.com/intentionet/netconan/issues/4):
     # Follow-up on these.  They were modified from RANCID's regexes and currently:
@@ -132,7 +134,6 @@
     #     (to make sure the regex handles different syntaxes allowed in the line)
     [(r"(\S* )*md5 \d+ key [^ ;]+(.*)", None)],
     [(r"(\S* )*(secret|simple-password) [^ ;]+(.*)", None)],
-    [(r"(\S* )*encrypted-password [^ ;]+(.*)", None)],
     [(r"(\S* )*ssh-(rsa|dsa) \"(.*)", None)],
     [(r"(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)", None)],
 ]

netconan/sensitive_item_removal.py

@@ -22,7 +22,7 @@
 from hashlib import md5
 
 # Using passlib for digests not supported by hashlib
-from passlib.hash import cisco_type7, md5_crypt, sha512_crypt
+from passlib.hash import cisco_type7, md5_crypt, sha256_crypt, sha512_crypt
 
 from netconan.utils import juniper_secrets
 
@@ -81,7 +81,7 @@
 # These are extra regexes to find lines that seem like they might contain
 # sensitive info (these are not already caught by RANCID default regexes)
 extra_password_regexes: list[list[RegexRule]] = [
-    [(r"(?<=encrypted-password )(\S+)", None)],
+    [(r"(?<=encrypted-password )(\S+)", 1)],
     [(r'(?<=key ")([^"]+)', 1)],
     [(r"(?<=key-hash sha256 )(\S+)", 1)],
     # Replace communities that do not look like well-known BGP communities
@@ -92,6 +92,8 @@
     # Catch-all's matching what looks like hashed passwords
     [(r'("?\$9\$[^\s;"]+)', 1)],
     [(r'("?\$1\$[^\s;"]+)', 1)],
+    [(r'("?\$5\$[^\s;"]+)', 1)],
+    [(r'("?\$6\$[^\s;"]+)', 1)],
 ]
 
 
@@ -237,6 +239,7 @@ class _sensitive_item_formats(Enum):
     text = 5
     sha512 = 6
     juniper_type9 = 7
+    sha256 = 8
 
 
 def anonymize_as_numbers(anonymizer: AsNumberAnonymizer, line: str) -> str:
@@ -301,6 +304,9 @@ def _anonymize_value(
         # identify anonymized lines
         anon_val = md5_crypt.using(salt="0" * old_salt_size).hash(anon_val)
 
+    if item_format == _sensitive_item_formats.sha256:
+        anon_val = sha256_crypt.using(rounds=5000).hash(anon_val)
+
     if item_format == _sensitive_item_formats.sha512:
         # Hash anon_val w/standard rounds=5000 to omit rounds parameter from hash output
         anon_val = sha512_crypt.using(rounds=5000).hash(anon_val)
@@ -323,6 +329,8 @@ def _check_sensitive_item_format(val: str) -> _sensitive_item_formats:
     # specific format so it should override hex or text)
     if re.match(r"^\$9\$[\S]+$", val):
         item_format = _sensitive_item_formats.juniper_type9
+    if re.match(r"^\$5\$[\S]+$", val):
+        item_format = _sensitive_item_formats.sha256
     if re.match(r"^\$6\$[\S]+$", val):
         item_format = _sensitive_item_formats.sha512
     if re.match(r"^\$1\$[\S]+\$[\S]+$", val):

tests/end_to_end/test_e2e_encrypted_password.py

@@ -0,0 +1,33 @@
+"""End-to-end tests for encrypted-password anonymization."""
+
+import re
+
+from netconan.netconan import main
+
+
+def test_end_to_end_encrypted_password_sha512(tmpdir):
+    """Test that encrypted-password with $6$ hash is anonymized, not scrubbed."""
+    filename = "test.txt"
+    # Hash of "netconanExamplePassword" using sha512_crypt
+    original_hash = "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0"
+    input_line = 'set system root-authentication encrypted-password "{}"\n'.format(
+        original_hash
+    )
+
+    input_dir = tmpdir.mkdir("input")
+    input_dir.join(filename).write(input_line)
+
+    output_dir = tmpdir.mkdir("output")
+    args = ["-s", "TESTSALT", "-p", "-i", str(input_dir), "-o", str(output_dir)]
+    main(args)
+
+    with open(str(output_dir.join(filename))) as f:
+        output = f.read()
+
+    # Original hash must not appear in output
+    assert original_hash not in output
+    # Line must not be scrubbed
+    assert "SCRUBBED" not in output
+    # Context must be preserved and output must contain a $6$ hash
+    assert "encrypted-password" in output
+    assert re.search(r'\$6\$[^\s"]+\$[^\s"]+', output)

tests/unit/test_sensitive_item_removal.py

@@ -37,7 +37,8 @@
 arista_password_lines = [
     (
         "username noc secret sha512 {}",
-        "$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0",
+        # Hash of "netconanExamplePassword" using sha512_crypt
+        "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0",
     ),
     ("   vrrp 2 authentication text {}", "RemoveMe"),
 ]
@@ -197,6 +198,21 @@
     ),
     ("set snmp community {} authorization read-only", "SECRETTEXT"),
     ("set snmp trap-group {} otherstuff", "SECRETTEXT"),
+    (
+        'set system root-authentication encrypted-password "{}"',
+        # Hash of "netconanExamplePassword" using sha512_crypt
+        "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0",
+    ),
+    (
+        'set system login user admin authentication encrypted-password "{}"',
+        # Hash of "netconanExamplePassword" using sha512_crypt
+        "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0",
+    ),
+    (
+        'encrypted-password "{}";',
+        # Hash of "netconanExamplePassword" using sha512_crypt
+        "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0",
+    ),
     ("key hexadecimal {}", "ABCDEF123456"),
     (
         'authentication-key "{}";',
@@ -303,7 +319,13 @@
         _sensitive_item_formats.juniper_type9,
     ),
     (
-        "$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0",
+        # Hash of "netconanExamplePassword" using sha256_crypt
+        "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8",
+        _sensitive_item_formats.sha256,
+    ),
+    (
+        # Hash of "netconanExamplePassword" using sha512_crypt
+        "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0",
         _sensitive_item_formats.sha512,
     ),
 ]
@@ -327,6 +349,8 @@
     "PasswordThree",
     "$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t",
     "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/",
+    # Hash of "netconanExamplePassword" using sha256_crypt
+    "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8",
     "$6$NQJRTiqxZiNR0aWI$hU1EPleWl6wGcMtDxaMEqNhN8WnxEqmeFjWC5h8oh5USSn5P9ZgFXbf2giO8nEtM.yBXO3O6b.76LQ1zlmG3B0",
 ]
 
@@ -457,6 +481,44 @@ def test__anonymize_value_unique():
         unique_anon_vals.add(anon_val)
 
 
+@pytest.mark.parametrize(
+    "original_val, hash_module",
+    [
+        (
+            # Hash of "netconanExamplePassword" using sha256_crypt
+            "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8",
+            "sha256_crypt",
+        ),
+        (
+            # Hash of "netconanExamplePassword" using sha512_crypt
+            "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0",
+            "sha512_crypt",
+        ),
+        (
+            "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/",
+            "md5_crypt",
+        ),
+    ],
+)
+def test__anonymize_value_produces_verifiable_hash(original_val, hash_module):
+    """Test that anonymized crypt hashes are verifiable against their plaintext."""
+    from passlib.hash import md5_crypt, sha256_crypt, sha512_crypt
+
+    hash_modules = {
+        "md5_crypt": md5_crypt,
+        "sha256_crypt": sha256_crypt,
+        "sha512_crypt": sha512_crypt,
+    }
+    pwd_lookup = {}
+    anon_val = _anonymize_value(original_val, pwd_lookup, {}, SALT)
+
+    # _anonymize_value generates "netconanRemoved0" as the plaintext (first
+    # entry in an empty lookup) and hashes it. Verify the hash is valid.
+    plaintext = "netconanRemoved0"
+    hasher = hash_modules[hash_module]
+    assert hasher.verify(plaintext, anon_val)
+
+
 @pytest.mark.parametrize("val, format_", sensitive_items_and_formats)
 def test__check_sensitive_item_format(val, format_):
     """Test sensitive item format detection."""
@@ -680,13 +742,13 @@ def test_pwd_removal_preserve_trailing_whitespace(regexes, whitespace):
 
 
 @pytest.mark.parametrize("whitespace", [" ", "\t", "\n", " \t\n"])
-def test_line_scrub_preserve_trailing_whitespace(regexes, whitespace):
-    """Test trailing whitespace is preserved when line is scrubbed (encrypted-password case)."""
-    # encrypted-password triggers line scrubbing (sensitive_item_num=None)
+def test_encrypted_password_preserve_trailing_whitespace(regexes, whitespace):
+    """Test trailing whitespace is preserved when encrypted-password is anonymized."""
     config_line = "        encrypted-password SECRET{}".format(whitespace)
     pwd_lookup = {}
     processed_line = replace_matching_item(regexes, config_line, pwd_lookup, SALT)
-    assert _LINE_SCRUBBED_MESSAGE in processed_line
+    assert "SECRET" not in processed_line
+    assert _LINE_SCRUBBED_MESSAGE not in processed_line
     assert processed_line.endswith(whitespace)