Merge pull request blacklanternsecurity#1495 from blacklanternsecurit…

…y/remove-parse-liststring

remove parse_list_string

bbot/core/helpers/misc.py

@@ -1189,7 +1189,14 @@ def str_or_file(s):
 split_regex = re.compile(r"[\s,]")
 
 
-def chain_lists(l, try_files=False, msg=None, remove_blank=True):
+def chain_lists(
+    l,
+    try_files=False,
+    msg=None,
+    remove_blank=True,
+    validate=False,
+    validate_chars='<>:"/\\|?*)',
+):
     """Chains together list elements, allowing for entries separated by commas.
 
     This function takes a list `l` and flattens it by splitting its entries on commas.
@@ -1202,10 +1209,15 @@ def chain_lists(l, try_files=False, msg=None, remove_blank=True):
         try_files (bool, optional): Whether to try to open entries as files. Defaults to False.
         msg (str, optional): An optional message to log when reading from a file. Defaults to None.
         remove_blank (bool, optional): Whether to remove blank entries from the list. Defaults to True.
+        validate (bool, optional): Whether to perform validation for undesirable characters. Defaults to False.
+        validate_chars (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:"/\\|?*)'
 
     Returns:
         list: The list of chained elements.
 
+    Raises:
+        ValueError: If the input string contains invalid characters, when enabled (off by default).
+
     Examples:
         >>> chain_lists(["a", "b,c,d"])
         ['a', 'b', 'c', 'd']
@@ -1219,6 +1231,9 @@ def chain_lists(l, try_files=False, msg=None, remove_blank=True):
     for entry in l:
         for s in split_regex.split(entry):
             f = s.strip()
+            if validate:
+                if any((c in validate_chars) or (ord(c) < 32 and c != " ") for c in f):
+                    raise ValueError(f"Invalid character in string: {f}")
             f_path = Path(f).resolve()
             if try_files and f_path.is_file():
                 if msg is not None:
@@ -2612,39 +2627,6 @@ def parse_port_string(port_string):
     return ports
 
 
-def parse_list_string(list_string):
-    """
-    Parses a comma-separated string into a list, removing invalid characters.
-
-    Args:
-        list_string (str): The string containing elements separated by commas.
-
-    Returns:
-        list: A list of individual elements parsed from the input string.
-
-    Raises:
-        ValueError: If the input string contains invalid characters.
-
-    Examples:
-        >>> parse_list_string("html,js,css")
-        ['html', 'js', 'css']
-
-        >>> parse_list_string("png,jpg,gif")
-        ['png', 'jpg', 'gif']
-
-        >>> parse_list_string("invalid<>char")
-        ValueError: Invalid character in string: invalid<>char
-    """
-    elements = list_string.split(",")
-    result = []
-
-    for element in elements:
-        if any((c in '<>:"/\\|?*') or (ord(c) < 32 and c != " ") for c in element):
-            raise ValueError(f"Invalid character in string: {element}")
-        result.append(element)
-    return result
-
-
 async def as_completed(coros):
     """
     Async generator that yields completed Tasks as they are completed.

bbot/modules/deadly/ffuf.py

@@ -1,5 +1,4 @@
 from bbot.modules.base import BaseModule
-from bbot.core.helpers.misc import parse_list_string
 
 import random
 import string
@@ -56,7 +55,7 @@ async def setup(self):
         self.tempfile, tempfile_len = self.generate_templist()
         self.verbose(f"Generated dynamic wordlist with length [{str(tempfile_len)}]")
         try:
-            self.extensions = parse_list_string(self.config.get("extensions", ""))
+            self.extensions = self.helpers.chain_lists(self.config.get("extensions", ""), validate=True)
             self.debug(f"Using custom extensions: [{','.join(self.extensions)}]")
         except ValueError as e:
             self.warning(f"Error parsing extensions: {e}")

bbot/modules/ffuf_shortnames.py

@@ -3,7 +3,6 @@
 import string
 
 from bbot.modules.deadly.ffuf import ffuf
-from bbot.core.helpers.misc import parse_list_string
 
 
 def find_common_prefixes(strings, minimum_set_length=4):
@@ -93,7 +92,7 @@ async def setup(self):
         self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions)
 
         try:
-            self.extensions = parse_list_string(self.config.get("extensions", ""))
+            self.extensions = self.helpers.chain_lists(self.config.get("extensions", ""), validate=True)
             self.debug(f"Using custom extensions: [{','.join(self.extensions)}]")
         except ValueError as e:
             self.warning(f"Error parsing extensions: {e}")

bbot/test/test_step_1/test_helpers.py

@@ -720,15 +720,24 @@ def test_portparse(helpers):
     assert str(e.value) == "Invalid port or port range: foo"
 
 
-def test_liststring(helpers):
-    assert helpers.parse_list_string("hello,world,bbot") == ["hello", "world", "bbot"]
+# test chain_lists helper
 
+
+def test_liststring_valid_strings(helpers):
+    assert helpers.chain_lists("hello,world,bbot") == ["hello", "world", "bbot"]
+
+
+def test_liststring_invalid_string(helpers):
     with pytest.raises(ValueError) as e:
-        helpers.parse_list_string("hello,world,\x01")
+        helpers.chain_lists("hello,world,\x01", validate=True)
     assert str(e.value) == "Invalid character in string: \x01"
 
-    assert helpers.parse_list_string("hello") == ["hello"]
 
+def test_liststring_singleitem(helpers):
+    assert helpers.chain_lists("hello") == ["hello"]
+
+
+def test_liststring_invalidfnchars(helpers):
     with pytest.raises(ValueError) as e:
-        helpers.parse_list_string("hello,world,bbot|test")
+        helpers.chain_lists("hello,world,bbot|test", validate=True)
     assert str(e.value) == "Invalid character in string: bbot|test"