diff --git a/Dockerfile b/Dockerfile index d8304a5..07fad61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get update \ WORKDIR /joern RUN curl -L "https://github.com/joernio/joern/releases/latest/download/joern-install.sh" -o joern-install.sh \ && chmod u+x joern-install.sh \ - && ./joern-install.sh --version=v2.0.290 + && ./joern-install.sh --version=v4.0.383 # Copy Quack contents COPY . /quack/ diff --git a/README.md b/README.md index 3ca6f87..86edf9e 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,12 @@ to the `unserialize` function call. ## Requirements and Setup Quack depends on: -* [Joern code analysis platform](https://joern.io/) version 2.0.290 +* [Joern code analysis platform](https://joern.io/) version 4.0.383 * Java Development Kit 19 (Joern dependency) * Python3 -Quack depends on features in Joern version 2.0.290. Older versions -of Joern will not work. Versions newer than 2.0.290 will likely work, +Quack depends on features in Joern version 4.0.383. Older versions +of Joern will not work. Versions newer than 4.0.383 will likely work, although have not been explicitly tested. @@ -56,7 +56,7 @@ $ sudo apt-get update && sudo apt-get install -y openjdk-19-jdk openjdk-19-jre ``` $ curl -L "https://github.com/joernio/joern/releases/latest/download/joern-install.sh" -o joern-install.sh $ chmod u+x joern-install.sh -$ ./joern-install.sh --version=v2.0.290 +$ ./joern-install.sh --version=v4.0.383 ``` 3. Install Python packages (optionally, in a virtual environment). diff --git a/deduce_allowed_classes.py b/deduce_allowed_classes.py index 1fa54cf..1cf02c1 100644 --- a/deduce_allowed_classes.py +++ b/deduce_allowed_classes.py @@ -8,6 +8,7 @@ """ import itertools +from collections import defaultdict # PHP primitive types. # Don't treat string as native because of __toString @@ -29,40 +30,89 @@ class Leak(Exception): pass -def deduce_allowed_classes(avail_classes, evidence): +def deduce_allowed_classes(avail_classes, grouped_evidence): # Extract all type deductions from the evidence collected about an object. - # (Duck and Exact type matching rules) - type_entries_all = [x for x in evidence if x["condType"] in ("Duck", "Exact")] - type_entries_no_tostring = [x for x in type_entries_all if x["reason"] != "HasToString"] - # Check if the type leaks because it's being used in a dynamic call, and if it - # does, return all available classes - leaks = any([x["reason"] == "DynamicCall" for x in type_entries_all]) - if leaks: - raise Leak() - - # Parse the deduced types - types_all = list(itertools.chain(*[x["type"].split("|") for x in type_entries_all])) - types_no_tostring = list(itertools.chain(*[x["type"].split("|") for x in type_entries_no_tostring])) + all_obj_types = [] + all_obj_types_no_tostring = [] + # If we have more than one evidence for a single object, take the intersection + # of all deduced types + for obj_evidence in grouped_evidence: + obj_evidence_no_tostring = [ + x for x in obj_evidence if x["reason"] != "HasToString" + ] + + # Check if the type leaks because it's being used in a dynamic call, and if it + # does, return all available classes + leaks = any([x["reason"] == "DynamicCall" for x in obj_evidence]) + if leaks: + raise Leak() + + obj_types = list(map( + set, + [ + evidence["type"].split("|") + for evidence in obj_evidence + if len(evidence["type"]) != 0 + ], + )) + obj_types_no_tostring = list(map( + set, + [ + evidence["type"].split("|") + for evidence in obj_evidence_no_tostring + if len(evidence["type"]) != 0 + ], + )) + + if len(obj_types) > 0: + obj_types_intersect = set.intersection(*obj_types) + all_obj_types.append(obj_types_intersect) + + if len(obj_types_no_tostring) > 0: + obj_types_intersect_no_tostring = set.intersection( + *obj_types_no_tostring) + all_obj_types_no_tostring.append(obj_types_intersect_no_tostring) + + # No types found + if len(all_obj_types) == 0: + return [], [], [] # Check if we have evidence indicating a native type - have_native_evidence = any([x in native_types for x in types_all]) + have_native_evidence = any([x in native_types for x in all_obj_types]) # Allowed types are the intersection of the set of types deduced for an # object, and the set of available classes. - allowed_types_all = set([x for x in types_all if x in avail_classes]) - allowed_types_no_tostring = set([x for x in types_no_tostring if x in avail_classes]) + allowed_types_all = set( + [ + obj_type + for obj_type_set in all_obj_types + for obj_type in obj_type_set + if obj_type in avail_classes + ] + ) + allowed_types_no_tostring = set( + [ + obj_type + for obj_type_set in all_obj_types_no_tostring + for obj_type in obj_type_set + if obj_type in avail_classes + ] + ) # We should not have found evidence that the object is both a native and # a class type. if have_native_evidence and len(allowed_types_no_tostring) > 0: - raise Exception(f"Found evidence for native type but also the following allowed types: {allowed_types_all}") + raise Exception( + f"Found evidence for native type but also the following allowed types: {allowed_types_all}" + ) # Determine whether the deduced types are useful for constraining the # available classes into allowed classes. # Filter out the evidence for types that don't give us any useful type # information (e.g., if type is 'array', we don't actually know what the # types of each element are, unless we collected more evidence) + types_all = set.union(*all_obj_types) useful_types = [x for x in types_all if x not in not_useful_types] useful_types = [x for x in useful_types if "." not in x and "->" not in x] @@ -77,7 +127,7 @@ def deduce_allowed_classes(avail_classes, evidence): elif len(useful_types) == 0: allowed_types_no_tostring = avail_classes - return types_all, allowed_types_all, allowed_types_no_tostring + return list(types_all), list(allowed_types_all), list(allowed_types_no_tostring) def compute_allowed_classes(evidence_entries, avail_classes_entries): @@ -90,37 +140,92 @@ def compute_allowed_classes(evidence_entries, avail_classes_entries): # Get the call location filename = unser_call["filename"] line_no = unser_call["lineNumber"] - # Get the collected type evidence - evidence = unser_call["conditions"] print(f"Working on: File[{filename}],Line[{line_no}]") + # Get the collected type evidence + evidence = [ + x for x in unser_call["conditions"] if x["condType"] in ("Duck", "Exact") + ] + + # Group by object + grouped_evidence_dict = defaultdict(list) + for item in evidence: + grouped_evidence_dict[item["nodeId"]].append(item) + grouped_evidence = list(grouped_evidence_dict.values()) + # Get the available classes at that callsite - avail_classes_entry = [x for x in avail_classes_entries if x["filename"] == filename] - assert (len(avail_classes_entry) == 1), f"No avail classes entries found for {filename}!" + avail_classes_entry = [ + x for x in avail_classes_entries if x["filename"] == filename + ] + assert ( + len(avail_classes_entry) == 1 + ), f"No avail classes entries found for {filename}!" avail_classes_entry = avail_classes_entry[0] - avail_classes_lines = list(set(avail_classes_entry['line_numbers']).intersection({line_no})) + avail_classes_lines = list( + set(avail_classes_entry["line_numbers"]).intersection({line_no}) + ) # Make sure we actually have available classes for this line - assert (len(avail_classes_lines) >= 1), f"No avail classes entries found for {line_no} in {filename}!" + assert ( + len(avail_classes_lines) >= 1 + ), f"No avail classes entries found for {line_no} in {filename}!" avail_classes = avail_classes_entry["avail_classes"] - result_entries.append({"filename" : filename, "lineNumber" : line_no, "allowedTypes" : None, "allowedClasses" : None}) + result_entries.append( + { + "filename": filename, + "lineNumber": line_no, + "allowedTypes": None, + "allowedClasses": None, + } + ) try: # Call the main script that consolidates the available classes # with the inferred types and produces the final set of allowed classes - types_all, allowed_classes_all, allowed_types_no_tostring = deduce_allowed_classes(avail_classes, evidence) + filtered_grouped_evidence = [ + [{key: d.get(key) for key in ("type", "reason")} + for d in obj_evidence] + for obj_evidence in grouped_evidence + ] + types_all, allowed_classes_all, allowed_types_no_tostring = ( + deduce_allowed_classes( + avail_classes, filtered_grouped_evidence) + ) print(f"All types collected from evidence: {types_all}") print(f"All allowed classes: {allowed_types_no_tostring}") result_entries[-1]["allowedTypes"] = types_all - result_entries[-1]["allowedClasses"] = list(allowed_types_no_tostring) + result_entries[-1]["allowedClasses"] = list( + allowed_types_no_tostring) except Leak as e: # we identify that the analysis is "leaking" (e.g., flows in to a dynamic # call we can't track)=> we need to just return available classes' gadgets. # When not taking into account the available classes (NOAVAIL), we just return all the gadgets in the project, # else only return the gadgets in the available classes - print(f"Project analysis for [{filename}]:[{line_no}] resulted in a leak. " - f"See docs about how to continue from here") + print( + f"Project analysis for [{filename}]:[{line_no}] resulted in a leak. " + f"See docs about how to continue from here" + ) except Exception as e: print(f"{e.__class__.__name__}:{e}") return result_entries + + +if __name__ == "__main__": + + import argparse + import json + + parser = argparse.ArgumentParser() + parser.add_argument("--analysis-results-path", required=True) + parser.add_argument("--availclass-results-path", required=True) + args = parser.parse_args() + + with open(args.analysis_results_path) as f: + evidence_entries = json.load(f) + + with open(args.availclass_results_path) as f: + avail_classes_entries = json.load(f) + + results = compute_allowed_classes(evidence_entries, avail_classes_entries) + print(results) diff --git a/pytests/php-samples/duck_test/duck_test.php b/pytests/php-samples/duck_test/duck_test.php new file mode 100644 index 0000000..73d086d --- /dev/null +++ b/pytests/php-samples/duck_test/duck_test.php @@ -0,0 +1,25 @@ +swim(); + $animal->fly(); +} + +?> diff --git a/pytests/php-samples/duck_tests/availclass.json b/pytests/php-samples/duck_tests/availclass.json new file mode 100644 index 0000000..bfb9363 --- /dev/null +++ b/pytests/php-samples/duck_tests/availclass.json @@ -0,0 +1 @@ +[{"filename":"/home/neo/quack/pytests/php-samples/duck_tests/pass_to_func.php","line_numbers":[9],"avail_classes":["Whale","Duck"]},{"filename":"/home/neo/quack/pytests/php-samples/duck_tests/method.php","line_numbers":[20],"avail_classes":["Whale","Duck"]},{"filename":"/home/neo/quack/pytests/php-samples/duck_tests/instanceof.php","line_numbers":[8],"avail_classes":["Whale","Duck"]},{"filename":"/home/neo/quack/pytests/php-samples/duck_tests/field.php","line_numbers":[11],"avail_classes":["Whale","Duck"]}] \ No newline at end of file diff --git a/pytests/php-samples/duck_tests/availclass.json.errors b/pytests/php-samples/duck_tests/availclass.json.errors new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/pytests/php-samples/duck_tests/availclass.json.errors @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pytests/php-samples/duck_tests/availclass.json.files_to_classes b/pytests/php-samples/duck_tests/availclass.json.files_to_classes new file mode 100644 index 0000000..ac22cdc --- /dev/null +++ b/pytests/php-samples/duck_tests/availclass.json.files_to_classes @@ -0,0 +1,4 @@ +/home/neo/quack/pytests/php-samples/duck_tests/pass_to_func.php -> ListBuffer(Duck, Whale) +/home/neo/quack/pytests/php-samples/duck_tests/method.php -> ListBuffer(Duck, Whale) +/home/neo/quack/pytests/php-samples/duck_tests/field.php -> ListBuffer(Duck, Whale) +/home/neo/quack/pytests/php-samples/duck_tests/instanceof.php -> ListBuffer(Duck, Whale) \ No newline at end of file diff --git a/pytests/php-samples/duck_tests/availclass.json.included_files b/pytests/php-samples/duck_tests/availclass.json.included_files new file mode 100644 index 0000000..e69de29 diff --git a/pytests/php-samples/duck_tests/availclass.json.warnings b/pytests/php-samples/duck_tests/availclass.json.warnings new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/pytests/php-samples/duck_tests/availclass.json.warnings @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pytests/php-samples/duck_tests/availclass_fixed.json b/pytests/php-samples/duck_tests/availclass_fixed.json new file mode 100644 index 0000000..b95d68c --- /dev/null +++ b/pytests/php-samples/duck_tests/availclass_fixed.json @@ -0,0 +1,42 @@ +[ + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/pass_to_func.php", + "line_numbers": [ + 9 + ], + "avail_classes": [ + "Whale", + "Duck" + ] + }, + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/method.php", + "line_numbers": [ + 20 + ], + "avail_classes": [ + "Whale", + "Duck" + ] + }, + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/instanceof.php", + "line_numbers": [ + 8 + ], + "avail_classes": [ + "Whale", + "Duck" + ] + }, + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/field.php", + "line_numbers": [ + 11 + ], + "avail_classes": [ + "Whale", + "Duck" + ] + } +] \ No newline at end of file diff --git a/pytests/php-samples/duck_tests/field.php b/pytests/php-samples/duck_tests/field.php new file mode 100644 index 0000000..d81081c --- /dev/null +++ b/pytests/php-samples/duck_tests/field.php @@ -0,0 +1,14 @@ +feather_color"; + +?> diff --git a/pytests/php-samples/duck_tests/instanceof.php b/pytests/php-samples/duck_tests/instanceof.php new file mode 100644 index 0000000..e6e8640 --- /dev/null +++ b/pytests/php-samples/duck_tests/instanceof.php @@ -0,0 +1,14 @@ + diff --git a/pytests/php-samples/duck_tests/joe_analyze.out.warnings b/pytests/php-samples/duck_tests/joe_analyze.out.warnings new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/pytests/php-samples/duck_tests/joe_analyze.out.warnings @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pytests/php-samples/duck_tests/joe_analyze_pretty.json b/pytests/php-samples/duck_tests/joe_analyze_pretty.json new file mode 100644 index 0000000..e6d0bf0 --- /dev/null +++ b/pytests/php-samples/duck_tests/joe_analyze_pretty.json @@ -0,0 +1,113 @@ +[ + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/field.php", + "lineNumber": 11, + "conditions": [ + { + "reason": "Scalar", + "condType": "Exact", + "type": "string", + "nodeId": "30064771076" + }, + { + "condType": "Scalar", + "type": "string" + }, + { + "condType": "ArgToFuncIdx", + "argIdx": "0", + "callerFullName": "echo", + "callerName": "echo" + }, + { + "condType": "FieldAccess", + "fieldName": "feather_color" + }, + { + "reason": "HasToString", + "condType": "Duck", + "type": "", + "nodeId": "25769803778" + }, + { + "condType": "Duck", + "field": "feather_color", + "reason": "HasField", + "type": "Duck", + "nodeId": "94489280512" + }, + { + "condType": "Exact", + "reason": "FuncArg", + "function": "echo", + "type": "string", + "nodeId": "30064771075" + }, + { + "reason": "HasToString", + "condType": "Duck", + "type": "", + "nodeId": "30064771074" + } + ] + }, + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/instanceof.php", + "lineNumber": 8, + "conditions": [ + { + "condType": "InstanceOf", + "type": "Duck" + } + ] + }, + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/method.php", + "lineNumber": 20, + "conditions": [ + { + "methodName": "swim", + "condType": "CallsMethod", + "methodFullName": "mixed.swim" + }, + { + "condType": "Duck", + "method": "swim", + "reason": "HasMethod", + "type": "Duck|Whale", + "nodeId": "94489280517" + }, + { + "methodName": "fly", + "condType": "CallsMethod", + "methodFullName": "mixed.fly" + }, + { + "condType": "Duck", + "method": "fly", + "reason": "HasMethod", + "type": "Duck", + "nodeId": "94489280517" + } + ] + }, + { + "filename": "/home/neo/quack/pytests/php-samples/duck_tests/pass_to_func.php", + "lineNumber": 9, + "conditions": [ + { + "condType": "ArgToFuncIdx", + "argIdx": "0", + "callerFullName": "somefunc", + "callerName": "somefunc" + }, + { + "condType": "Exact", + "reason": "FuncArg", + "function": "somefunc", + "type": "Duck", + "nodeId": "94489280518" + } + ] + } +] diff --git a/pytests/php-samples/duck_tests/method.php b/pytests/php-samples/duck_tests/method.php new file mode 100644 index 0000000..73d086d --- /dev/null +++ b/pytests/php-samples/duck_tests/method.php @@ -0,0 +1,25 @@ +swim(); + $animal->fly(); +} + +?> diff --git a/pytests/php-samples/duck_tests/pass_to_func.php b/pytests/php-samples/duck_tests/pass_to_func.php new file mode 100644 index 0000000..44890f6 --- /dev/null +++ b/pytests/php-samples/duck_tests/pass_to_func.php @@ -0,0 +1,12 @@ + diff --git a/pytests/php-samples/duck_tests/results.json b/pytests/php-samples/duck_tests/results.json new file mode 100644 index 0000000..2c11785 --- /dev/null +++ b/pytests/php-samples/duck_tests/results.json @@ -0,0 +1 @@ +[{"filename": "/home/neo/quack/pytests/php-samples/duck_tests/field.php", "lineNumber": 11, "allowedTypes": ["Duck", "string"], "allowedClasses": ["Duck"]}, {"filename": "/home/neo/quack/pytests/php-samples/duck_tests/instanceof.php", "lineNumber": 8, "allowedTypes": ["Duck"], "allowedClasses": ["Duck"]}, {"filename": "/home/neo/quack/pytests/php-samples/duck_tests/method.php", "lineNumber": 20, "allowedTypes": ["Duck"], "allowedClasses": ["Duck"]}, {"filename": "/home/neo/quack/pytests/php-samples/duck_tests/pass_to_func.php", "lineNumber": 9, "allowedTypes": ["Duck"], "allowedClasses": ["Duck"]}] \ No newline at end of file diff --git a/pytests/php-samples/duck_tests/runtime_info.json b/pytests/php-samples/duck_tests/runtime_info.json new file mode 100644 index 0000000..1605539 --- /dev/null +++ b/pytests/php-samples/duck_tests/runtime_info.json @@ -0,0 +1,6 @@ +{ + "Joern-Parse (graph creation)": "00:00:04.14", + "Joern (graph analysis)": "00:00:07.14", + "Joern-Script[Analyze]": "00:00:11.03", + "Joern-Script[AvailClasses]": "00:00:10.05" +} \ No newline at end of file diff --git a/pytests/php-samples/nested_test/availclass.json b/pytests/php-samples/nested_test/availclass.json new file mode 100644 index 0000000..e972380 --- /dev/null +++ b/pytests/php-samples/nested_test/availclass.json @@ -0,0 +1 @@ +[{"filename":"/home/neo/quack/pytests/php-samples/nested_test/nested.php","line_numbers":[22],"avail_classes":["Human","Cat","Dog"]}] \ No newline at end of file diff --git a/pytests/php-samples/nested_test/availclass.json.errors b/pytests/php-samples/nested_test/availclass.json.errors new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/pytests/php-samples/nested_test/availclass.json.errors @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pytests/php-samples/nested_test/availclass.json.files_to_classes b/pytests/php-samples/nested_test/availclass.json.files_to_classes new file mode 100644 index 0000000..112fa1a --- /dev/null +++ b/pytests/php-samples/nested_test/availclass.json.files_to_classes @@ -0,0 +1 @@ +/home/neo/quack/pytests/php-samples/nested_test/nested.php -> ListBuffer(Human, Cat, Dog) \ No newline at end of file diff --git a/pytests/php-samples/nested_test/availclass.json.included_files b/pytests/php-samples/nested_test/availclass.json.included_files new file mode 100644 index 0000000..e69de29 diff --git a/pytests/php-samples/nested_test/availclass.json.warnings b/pytests/php-samples/nested_test/availclass.json.warnings new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/pytests/php-samples/nested_test/availclass.json.warnings @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pytests/php-samples/nested_test/availclass_fixed.json b/pytests/php-samples/nested_test/availclass_fixed.json new file mode 100644 index 0000000..5004241 --- /dev/null +++ b/pytests/php-samples/nested_test/availclass_fixed.json @@ -0,0 +1,13 @@ +[ + { + "filename": "/home/neo/quack/pytests/php-samples/nested_test/nested.php", + "line_numbers": [ + 22 + ], + "avail_classes": [ + "Human", + "Cat", + "Dog" + ] + } +] \ No newline at end of file diff --git a/pytests/php-samples/nested_test/joe_analyze.out.warnings b/pytests/php-samples/nested_test/joe_analyze.out.warnings new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/pytests/php-samples/nested_test/joe_analyze.out.warnings @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/pytests/php-samples/nested_test/joe_analyze_pretty.json b/pytests/php-samples/nested_test/joe_analyze_pretty.json new file mode 100644 index 0000000..8990f7f --- /dev/null +++ b/pytests/php-samples/nested_test/joe_analyze_pretty.json @@ -0,0 +1,43 @@ +[ + { + "filename": "/home/neo/quack/pytests/php-samples/nested_test/nested.php", + "lineNumber": 22, + "conditions": [ + { + "condType": "Duck", + "field": "best_friend", + "reason": "HasField", + "type": "Human", + "nodeId": "94489280512" + }, + { + "methodName": "bark", + "condType": "CallsMethod", + "methodFullName": ".bark" + }, + { + "condType": "Duck", + "method": "bark", + "reason": "HasMethod", + "type": "Dog", + "nodeId": "30064771079" + }, + { + "condType": "Duck", + "method": "sing", + "reason": "HasMethod", + "type": "Human", + "nodeId": "94489280512" + }, + { + "condType": "FieldAccess", + "fieldName": "best_friend" + }, + { + "methodName": "sing", + "condType": "CallsMethod", + "methodFullName": "mixed.sing" + } + ] + } +] diff --git a/pytests/php-samples/nested_test/nested.php b/pytests/php-samples/nested_test/nested.php new file mode 100644 index 0000000..4b03d3d --- /dev/null +++ b/pytests/php-samples/nested_test/nested.php @@ -0,0 +1,27 @@ +sing(); +$hacker->best_friend->bark(); + +?> + diff --git a/pytests/php-samples/nested_test/results.json b/pytests/php-samples/nested_test/results.json new file mode 100644 index 0000000..25fe448 --- /dev/null +++ b/pytests/php-samples/nested_test/results.json @@ -0,0 +1 @@ +[{"filename": "/home/neo/quack/pytests/php-samples/nested_test/nested.php", "lineNumber": 22, "allowedTypes": ["Human", "Dog"], "allowedClasses": ["Human", "Dog"]}] \ No newline at end of file diff --git a/pytests/php-samples/nested_test/runtime_info.json b/pytests/php-samples/nested_test/runtime_info.json new file mode 100644 index 0000000..6d6cd67 --- /dev/null +++ b/pytests/php-samples/nested_test/runtime_info.json @@ -0,0 +1,6 @@ +{ + "Joern-Parse (graph creation)": "00:00:03.86", + "Joern (graph analysis)": "00:00:06.61", + "Joern-Script[Analyze]": "00:00:10.16", + "Joern-Script[AvailClasses]": "00:00:09.75" +} \ No newline at end of file diff --git a/pytests/test_class_fragments.py b/pytests/test_class_fragments.py index 0d8e1eb..8cb4259 100644 --- a/pytests/test_class_fragments.py +++ b/pytests/test_class_fragments.py @@ -1,6 +1,7 @@ import pytest + from .conftest import SAMPLES_DIR -from .utils import do_analysis, compare_results +from .utils import compare_results, do_analysis CLASS_TEST_NAME = "class_test.php" @@ -12,11 +13,15 @@ def test_class_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) expected_result = [ - {'filename': 'class_test.php', 'lineNumber': 8, - 'allowedTypes': ['bool'], 'allowedClasses': []} + { + "filename": "class_test.php", + "lineNumber": 8, + "allowedTypes": ["bool"], + "allowedClasses": [], + } ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) CLASS_THIS_TEST_NAME = "class_this_test.php" @@ -28,10 +33,16 @@ def test_class_this_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) - expected_result = [{'filename': 'class_this_test.php', - 'lineNumber': 6, 'allowedTypes': [], 'allowedClasses': ['Person']}] + expected_result = [ + { + "filename": "class_this_test.php", + "lineNumber": 6, + "allowedTypes": [], + "allowedClasses": ["Person"], + } + ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) CLASS_STORAGE_TEST_NAME = "class_storage_test.php" @@ -43,7 +54,19 @@ def test_class_storage_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) - expected_result = [{'filename': 'class_storage_test.php', 'lineNumber': 33, 'allowedTypes': ['string', 'Template', 'InterestingClass', ''], 'allowedClasses': [ - 'InterestingClass']}, {'filename': 'class_storage_test.php', 'lineNumber': 29, 'allowedTypes': ['InterestingClass'], 'allowedClasses': ['InterestingClass']}] + expected_result = [ + { + "filename": "class_storage_test.php", + "lineNumber": 33, + "allowedTypes": ["string", "Template", "InterestingClass"], + "allowedClasses": ["InterestingClass"], + }, + { + "filename": "class_storage_test.php", + "lineNumber": 29, + "allowedTypes": ["InterestingClass"], + "allowedClasses": ["InterestingClass"], + }, + ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_comments_fragments.py b/pytests/test_comments_fragments.py index bd7a4c1..c5ff317 100644 --- a/pytests/test_comments_fragments.py +++ b/pytests/test_comments_fragments.py @@ -15,7 +15,7 @@ def test_comment_types_test(datafiles, tmp_path): expected_result = [{'filename': 'comment_types_test.php', 'lineNumber': 8, 'allowedTypes': ['mixed'], 'allowedClasses': []}] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) COMMENT_TYPES_TEST2_NAME = "comment_types_test2.php" @@ -30,4 +30,4 @@ def test_comment_types_test2(datafiles, tmp_path): expected_result = [{'filename': 'comment_types_test2.php', 'lineNumber': 30, 'allowedTypes': ['float'], 'allowedClasses': []}] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_conditions_fragments.py b/pytests/test_conditions_fragments.py index df888d4..01a08de 100644 --- a/pytests/test_conditions_fragments.py +++ b/pytests/test_conditions_fragments.py @@ -15,10 +15,10 @@ def test_conditions_field_get(datafiles, tmp_path): expected_result = [ {'filename': 'conditions_field_get.php', 'lineNumber': 15, - 'allowedTypes': ['', 'SomeClass'], 'allowedClasses': ['SomeClass']} + 'allowedTypes': ['SomeClass'], 'allowedClasses': ['SomeClass']} ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) CONDITIONS_FIELD_SET_NAME = "conditions_field_set.php" @@ -33,7 +33,7 @@ def test_conditions_field_set(datafiles, tmp_path): expected_result = [{'filename': 'conditions_field_set.php', 'lineNumber': 15, 'allowedTypes': ['SomeClass'], 'allowedClasses': ['SomeClass']}] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) CONDITIONS_TEST_NAME = "conditions_test.php" @@ -46,9 +46,9 @@ def test_conditions_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) expected_result = [{'filename': 'conditions_test.php', 'lineNumber': 8, - 'allowedTypes': ['string', ''], 'allowedClasses': []}] + 'allowedTypes': ['string'], 'allowedClasses': []}] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) CONDITIONS_TWO_CALLS_TEST_NAME = "conditions_two_calls_test.php" @@ -65,7 +65,7 @@ def test_conditions_two_calls_test(datafiles, tmp_path): {'filename': 'conditions_two_calls_test.php', 'lineNumber': 34, 'allowedTypes': ['FirstClass'], 'allowedClasses': ['FirstClass']} ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) CONDITIONALS_TEST_NAME = "conditionals_test.php" @@ -78,18 +78,18 @@ def test_conditionals(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) expected_result = [{'filename': 'conditionals_test.php', 'lineNumber': 6, - 'allowedTypes': ['string', ''], 'allowedClasses': []}, + 'allowedTypes': ['string'], 'allowedClasses': []}, {'filename': 'conditionals_test.php', 'lineNumber': 10, - 'allowedTypes': ['string', ''], 'allowedClasses': []}, + 'allowedTypes': ['string'], 'allowedClasses': []}, {'filename': 'conditionals_test.php', 'lineNumber': 14, 'allowedTypes': [], 'allowedClasses': []}, {'filename': 'conditionals_test.php', 'lineNumber': 20, 'allowedTypes': [], 'allowedClasses': []}, {'filename': 'conditionals_test.php', 'lineNumber': 24, - 'allowedTypes': ['string', ''], 'allowedClasses': []}, + 'allowedTypes': ['string'], 'allowedClasses': []}, {'filename': 'conditionals_test.php', 'lineNumber': 28, - 'allowedTypes': ['string', ''], 'allowedClasses': []}, + 'allowedTypes': ['string'], 'allowedClasses': []}, {'filename': 'conditionals_test.php', 'lineNumber': 32, 'allowedTypes': [], 'allowedClasses': []} ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_dockblocks.py b/pytests/test_dockblocks.py index d5311bc..d43b736 100644 --- a/pytests/test_dockblocks.py +++ b/pytests/test_dockblocks.py @@ -15,7 +15,7 @@ def test_docblock_simple(datafiles, tmp_path): expected_result = [{'filename': 'docblock_simple.php', 'lineNumber': 26, 'allowedTypes': ['mixed'], 'allowedClasses': ['ClassA', 'ClassB']}] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) DOCBLOCK_TIEBREAK_NAME = "docblock_tiebreak.php" @@ -32,7 +32,7 @@ def test_docblock_tiebreak(datafiles, tmp_path): 'ClassA', 'ClassC', 'mixed'], 'allowedClasses': ['ClassC', 'ClassA']} ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) DOCBLOCK_RETURN_INCONSISTENT_TEST_NAME = "docblock_return_inconsistent.php" @@ -47,4 +47,4 @@ def test_docblock_return_inconsistent(datafiles, tmp_path): expected_result = [] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_duck_typing.py b/pytests/test_duck_typing.py new file mode 100644 index 0000000..0936f70 --- /dev/null +++ b/pytests/test_duck_typing.py @@ -0,0 +1,25 @@ +import pytest + +from .conftest import SAMPLES_DIR +from .utils import compare_results, do_analysis + +DUCK_TEST_PROJECT = "duck_test" +DUCK_TEST_NAME = "duck_test.php" + + +@pytest.mark.datafiles(SAMPLES_DIR / DUCK_TEST_PROJECT, keep_top_dir=True) +def test_simple_project(datafiles, tmp_path): + project_path = datafiles / DUCK_TEST_PROJECT + + results = do_analysis(project_path, tmp_path) + + expected_result = [ + { + "filename": DUCK_TEST_NAME, + "lineNumber": 20, + "allowedTypes": ["Duck"], + "allowedClasses": [], + } + ] + + assert compare_results(expected_result, results, project_path) diff --git a/pytests/test_functions.py b/pytests/test_functions.py index 0362aae..1a10fbb 100644 --- a/pytests/test_functions.py +++ b/pytests/test_functions.py @@ -1,9 +1,11 @@ import pytest + from .conftest import SAMPLES_DIR -from .utils import do_analysis, compare_results +from .utils import compare_results, do_analysis LOCAL_ARG_CONFLICT = "local-arg-conflict.php" + # Note this test is only confirmed to work on Joern 2.0.290, and fails on 2.0.156 @pytest.mark.datafiles(SAMPLES_DIR / LOCAL_ARG_CONFLICT) def test_local_arg_conflict(datafiles, tmp_path): @@ -11,11 +13,16 @@ def test_local_arg_conflict(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) - expected_result = [{'filename': 'local-arg-conflict.php', 'lineNumber': 11, - 'allowedTypes': ['string', '', 'mixed'], 'allowedClasses': []}] + expected_result = [ + { + "filename": "local-arg-conflict.php", + "lineNumber": 11, + "allowedTypes": ["string", "mixed"], + "allowedClasses": [], + } + ] - - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) LOCAL_ARG_NOCONFLICT = "local-arg-noconflict.php" @@ -27,8 +34,13 @@ def test_local_arg_noconflict(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) - expected_result = [{'filename': 'local-arg-noconflict.php', 'lineNumber': 11, - 'allowedTypes': ['string', '', 'mixed'], 'allowedClasses': []}] - + expected_result = [ + { + "filename": "local-arg-noconflict.php", + "lineNumber": 11, + "allowedTypes": ["string", "mixed"], + "allowedClasses": [], + } + ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_import_fragments.py b/pytests/test_import_fragments.py index eaf798a..cd5e448 100644 --- a/pytests/test_import_fragments.py +++ b/pytests/test_import_fragments.py @@ -18,4 +18,4 @@ def test_mid_file_import_test(datafiles, tmp_path): expected_result = [{'filename': 'mid_file_import_test.php', 'lineNumber': 6, 'allowedTypes': [], 'allowedClasses': []}, {'filename': 'mid_file_import_test.php', 'lineNumber': 15, 'allowedTypes': [], 'allowedClasses': []}, {'filename': 'mid_file_import_test.php', 'lineNumber': 18, 'allowedTypes': [], 'allowedClasses': []}] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_include_projects.py b/pytests/test_include_projects.py index 09b1ef3..b81461e 100644 --- a/pytests/test_include_projects.py +++ b/pytests/test_include_projects.py @@ -1,8 +1,9 @@ import pytest + from .conftest import SAMPLES_DIR -from .utils import do_analysis, compare_results +from .utils import compare_results, do_analysis -INCLUDE_DIRECTIVE_NAME = 'include_directives' +INCLUDE_DIRECTIVE_NAME = "include_directives" @pytest.mark.datafiles(SAMPLES_DIR / INCLUDE_DIRECTIVE_NAME, keep_top_dir=True) @@ -12,17 +13,36 @@ def test_include_directive_project(datafiles, tmp_path): results = do_analysis(project_path, tmp_path) expected_result = [ - {'filename': 'index.php', 'lineNumber': 9, 'allowedTypes': [], 'allowedClasses': [ - 'Literal', 'Magic', 'Builtin', 'GlobClassThree', 'GlobClassTwo', 'GlobClassOne']}, - {'filename': 'src/BackwardsToo.php', 'lineNumber': 9, - 'allowedTypes': [], 'allowedClasses': ['BackwardsToo', 'Backwards']}, - {'filename': 'src/ConstantIncl.php', 'lineNumber': 7, - 'allowedTypes': [], 'allowedClasses': ['Constant']} + { + "filename": "index.php", + "lineNumber": 9, + "allowedTypes": [], + "allowedClasses": [ + "Literal", + "Magic", + "Builtin", + "GlobClassThree", + "GlobClassTwo", + "GlobClassOne", + ], + }, + { + "filename": "src/BackwardsToo.php", + "lineNumber": 9, + "allowedTypes": [], + "allowedClasses": ["BackwardsToo", "Backwards"], + }, + { + "filename": "src/ConstantIncl.php", + "lineNumber": 7, + "allowedTypes": [], + "allowedClasses": ["Constant"], + }, ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, project_path) -INCLUDE_TEST_NAME = 'include_test' +INCLUDE_TEST_NAME = "include_test" @pytest.mark.datafiles(SAMPLES_DIR / INCLUDE_TEST_NAME, keep_top_dir=True) @@ -32,18 +52,48 @@ def test_include_test_project(datafiles, tmp_path): results = do_analysis(project_path, tmp_path) # TODO make custom check function that doesn't depend on class order - expected_result1 = [{'filename': 'CClass.php', 'lineNumber': 32, 'allowedTypes': [], 'allowedClasses': [ - 'CClass', 'BClass', 'A2Class', 'A1Class', 'D1Class', 'D2Class', 'FClass', 'EClass']}] + expected_result1 = [ + { + "filename": "CClass.php", + "lineNumber": 32, + "allowedTypes": [], + "allowedClasses": [ + "CClass", + "BClass", + "A2Class", + "A1Class", + "D1Class", + "D2Class", + "FClass", + "EClass", + ], + } + ] - expected_result2 = [{'filename': 'CClass.php', 'lineNumber': 32, 'allowedTypes': [], 'allowedClasses': [ - 'CClass', 'BClass', 'A1Class', 'A2Class', 'D1Class', 'D2Class', 'FClass', 'EClass']}] + expected_result2 = [ + { + "filename": "CClass.php", + "lineNumber": 32, + "allowedTypes": [], + "allowedClasses": [ + "CClass", + "BClass", + "A1Class", + "A2Class", + "D1Class", + "D2Class", + "FClass", + "EClass", + ], + } + ] - # assert (check_objects(expected_result1, results) or check_objects(expected_result2, results)) - # assert (results == expected_result1 or results == expected_result2) - assert compare_results(expected_result1, results) + # assert (check_objects(expected_result1, results) or check_objects(expected_result2, results)) + # assert (results == expected_result1 or results == expected_result2) + assert compare_results(expected_result1, results, project_path) -AUTOLOAD_PROJECT_NAME = 'autoload_test' +AUTOLOAD_PROJECT_NAME = "autoload_test" @pytest.mark.datafiles(SAMPLES_DIR / AUTOLOAD_PROJECT_NAME, keep_top_dir=True) @@ -52,6 +102,12 @@ def test_autoload_project(datafiles, tmp_path): results = do_analysis(project_path, tmp_path) - expected_result = [{'filename': 'index.php', 'lineNumber': 19, 'allowedTypes': [ - 'string', '', 'AnotherClass', 'MyClass', 'MyOtherClass'], 'allowedClasses': []}] - assert compare_results(expected_result, results) + expected_result = [ + { + "filename": "index.php", + "lineNumber": 19, + "allowedTypes": ["string", "", "AnotherClass", "MyClass", "MyOtherClass"], + "allowedClasses": [], + } + ] + assert compare_results(expected_result, results, project_path) diff --git a/pytests/test_misc_fragments.py b/pytests/test_misc_fragments.py index 724a852..9fa5e4c 100644 --- a/pytests/test_misc_fragments.py +++ b/pytests/test_misc_fragments.py @@ -1,6 +1,7 @@ import pytest + from .conftest import SAMPLES_DIR -from .utils import do_analysis, compare_results +from .utils import compare_results, do_analysis LIST_TEST_NAME = "list-statement.php" # Note: this test currently fails, see Github Issue #23 @@ -13,17 +14,33 @@ def test_list_construct(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) expected_result = [ - {'filename': 'list-statement.php', 'lineNumber': 20, - 'allowedTypes': ['FooClass'], 'allowedClasses': ['FooClass']}, - {'filename': 'list-statement.php', 'lineNumber': 24, - 'allowedTypes': ['FooClass'], 'allowedClasses': ['FooClass']}, - {'filename': 'list-statement.php', 'lineNumber': 28, - 'allowedTypes': ['BarClass'], 'allowedClasses': ['BarClass']}, - {'filename': 'list-statement.php', 'lineNumber': 33, - 'allowedTypes': ['FooClass', 'BarClass'], 'allowedClasses': ['FooClass', 'BarClass']}, + { + "filename": "list-statement.php", + "lineNumber": 20, + "allowedTypes": ["FooClass"], + "allowedClasses": ["FooClass"], + }, + { + "filename": "list-statement.php", + "lineNumber": 24, + "allowedTypes": ["FooClass"], + "allowedClasses": ["FooClass"], + }, + { + "filename": "list-statement.php", + "lineNumber": 28, + "allowedTypes": ["BarClass"], + "allowedClasses": ["BarClass"], + }, + { + "filename": "list-statement.php", + "lineNumber": 33, + "allowedTypes": ["FooClass", "BarClass"], + "allowedClasses": ["FooClass", "BarClass"], + }, ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) AST_PARENTS_TEST_NAME = "AST_parents_test.php" @@ -35,39 +52,102 @@ def test_AST_parents_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) expected_result = [ - {'filename': 'AST_parents_test.php', 'lineNumber': 12, - 'allowedTypes': [], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 15, - 'allowedTypes': [], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 19, - 'allowedTypes': [], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 22, - 'allowedTypes': [], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 27, - 'allowedTypes': [''], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 32, - 'allowedTypes': ['bool'], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 37, - 'allowedTypes': ['', 'string'], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 41, - 'allowedTypes': ['mixed'], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 46, 'allowedTypes': [ - '.instanceOf..conditional.', '', 'string'], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 54, - 'allowedTypes': ['string'], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 58, - 'allowedTypes': ['string', ''], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 66, - 'allowedTypes': ['string', ''], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 69, - 'allowedTypes': ['array', 'mixed'], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 74, - 'allowedTypes': [], 'allowedClasses': []}, - {'filename': 'AST_parents_test.php', 'lineNumber': 80, - 'allowedTypes': [], 'allowedClasses': []} + { + "filename": "AST_parents_test.php", + "lineNumber": 12, + "allowedTypes": [], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 15, + "allowedTypes": [], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 19, + "allowedTypes": [], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 22, + "allowedTypes": [], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 27, + "allowedTypes": [""], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 32, + "allowedTypes": ["bool"], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 37, + "allowedTypes": ["", "string"], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 41, + "allowedTypes": ["mixed"], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 46, + "allowedTypes": [ + "", + "string", + ], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 54, + "allowedTypes": ["string"], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 58, + "allowedTypes": ["string", ""], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 66, + "allowedTypes": ["string", ""], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 69, + "allowedTypes": ["array", "mixed"], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 74, + "allowedTypes": [], + "allowedClasses": [], + }, + { + "filename": "AST_parents_test.php", + "lineNumber": 80, + "allowedTypes": [], + "allowedClasses": [], + }, ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) FUGIO_INSPIRED_TEST_NAME = "fugio_inspired_test.php" @@ -79,10 +159,16 @@ def test_fugio_inspired_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) - expected_result = [{'filename': 'fugio_inspired_test.php', - 'lineNumber': 8, 'allowedTypes': [''], 'allowedClasses': []}] + expected_result = [ + { + "filename": "fugio_inspired_test.php", + "lineNumber": 8, + "allowedTypes": [""], + "allowedClasses": [], + } + ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) GETTYPE_SWITCH_TEST_NAME = "gettype_switch_test.php" @@ -96,14 +182,27 @@ def test_gettype_switch_test(datafiles, tmp_path): expected_result = [ { - 'filename': 'gettype_switch_test.php', - 'lineNumber': 6, - 'allowedTypes': ['string', 'mixed', 'bool', 'mixed', 'float', 'string', 'int', 'mixed', 'array', 'array', '', 'mixed'], - 'allowedClasses': [] + "filename": "gettype_switch_test.php", + "lineNumber": 6, + "allowedTypes": [ + "string", + "mixed", + "bool", + "mixed", + "float", + "string", + "int", + "mixed", + "array", + "array", + "", + "mixed", + ], + "allowedClasses": [], } ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) MAYBE_UNSERIALIZE_TEST_NAME = "maybe_unserialize_test.php" @@ -115,10 +214,16 @@ def test_maybe_unserialize_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) - expected_result = [{'filename': 'maybe_unserialize_test.php', - 'lineNumber': 6, 'allowedTypes': ['string', ''], 'allowedClasses': []}] + expected_result = [ + { + "filename": "maybe_unserialize_test.php", + "lineNumber": 6, + "allowedTypes": ["string", ""], + "allowedClasses": [], + } + ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) RECURSIVE_VAR_USE_TEST_NAME = "recursive_var_use_test.php" @@ -131,15 +236,27 @@ def test_recursive_var_use_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) expected_result = [ - {'filename': 'recursive_var_use_test.php', 'lineNumber': 4, - 'allowedTypes': ['string', '', 'mixed'], 'allowedClasses': []}, - {'filename': 'recursive_var_use_test.php', 'lineNumber': 17, - 'allowedTypes': ['', 'string'], 'allowedClasses': []}, - {'filename': 'recursive_var_use_test.php', 'lineNumber': 21, - 'allowedTypes': ['numeric'], 'allowedClasses': []} + { + "filename": "recursive_var_use_test.php", + "lineNumber": 4, + "allowedTypes": ["string", "", "mixed"], + "allowedClasses": [], + }, + { + "filename": "recursive_var_use_test.php", + "lineNumber": 17, + "allowedTypes": ["", "string"], + "allowedClasses": [], + }, + { + "filename": "recursive_var_use_test.php", + "lineNumber": 21, + "allowedTypes": ["numeric"], + "allowedClasses": [], + }, ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) TOSTRING_TEST_NAME = "tostring_test.php" @@ -151,7 +268,13 @@ def test_tostring_test(datafiles, tmp_path): results = do_analysis(fragment_path, tmp_path) - expected_result = [{'filename': 'tostring_test.php', 'lineNumber': 24, 'allowedTypes': [ - 'string', 'mixed', 'string', 'HasToString', 'mixed'], 'allowedClasses': []}] + expected_result = [ + { + "filename": "tostring_test.php", + "lineNumber": 24, + "allowedTypes": ["string", "mixed", "string", "HasToString", "mixed"], + "allowedClasses": [], + } + ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_simple_fragments.py b/pytests/test_simple_fragments.py index a8cc5f3..e60bb46 100644 --- a/pytests/test_simple_fragments.py +++ b/pytests/test_simple_fragments.py @@ -17,12 +17,12 @@ def test_simple_print(datafiles, tmp_path): { "filename": SIMPLE_PRINT_NAME, "lineNumber": 4, - "allowedTypes": ['string', ''], + "allowedTypes": ['string'], "allowedClasses": [] } ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) TEST_PRINT_LIST_NAME = "list_print_test.php" @@ -39,12 +39,12 @@ def test_print_list(datafiles, tmp_path): { "filename": TEST_PRINT_LIST_NAME, "lineNumber": 4, - "allowedTypes": ['string', ''], + "allowedTypes": ['string'], "allowedClasses": [] } ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) # Test two simple string cases @@ -61,15 +61,15 @@ def test_two_prints(datafiles, tmp_path): { "filename": TWO_PRINTS_NAME, "lineNumber": 4, - "allowedTypes": ['string', ''], + "allowedTypes": ['string'], "allowedClasses": [] }, { "filename": TWO_PRINTS_NAME, "lineNumber": 8, - "allowedTypes": ['string', ''], + "allowedTypes": ['string'], "allowedClasses": [] } ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, fragment_path) diff --git a/pytests/test_simple_projects.py b/pytests/test_simple_projects.py index 8fdf1b3..498c6c7 100644 --- a/pytests/test_simple_projects.py +++ b/pytests/test_simple_projects.py @@ -18,12 +18,12 @@ def test_simple_project(datafiles, tmp_path): { "filename": SIMPLE_PRINT_NAME, "lineNumber": 4, - "allowedTypes": ['string', ''], + "allowedTypes": ['string'], "allowedClasses": [] } ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, project_path) # Test simple string case inside a project @@ -40,15 +40,15 @@ def test_multi_project(datafiles, tmp_path): { "filename": "simple-print-one.php", "lineNumber": 4, - "allowedTypes": ['string', ''], + "allowedTypes": ['string'], "allowedClasses": [] }, { "filename": "simple-print-two.php", "lineNumber": 4, - "allowedTypes": ['string', ''], + "allowedTypes": ['string'], "allowedClasses": [] } ] - assert compare_results(expected_result, results) + assert compare_results(expected_result, results, project_path) diff --git a/pytests/utils.py b/pytests/utils.py index c39c1a3..8cf4d64 100644 --- a/pytests/utils.py +++ b/pytests/utils.py @@ -1,61 +1,86 @@ -from runner import PHPAnalyzer import json +from pathlib import Path + +from runner import PHPAnalyzer + from .conftest import RESULTS_FNAME def do_analysis(project_path, result_path): - PHPAnalyzer(project_path=project_path, results_path=result_path) + analyzer = PHPAnalyzer(project_path=project_path, results_path=result_path) + analyzer.process_project() with open(result_path / RESULTS_FNAME) as results_file: results = json.loads(results_file.read().strip()) print("Test Results:", results) return results + def print_failure(expected, actual, reason: str): print("Comparison failed:", reason) print("Expected:", expected) print("Actual:", actual) + def sort_by_line_number(results: list[dict]) -> list[dict]: - return sorted(results, key=lambda x: x['lineNumber']) + return sorted(results, key=lambda x: x["lineNumber"]) + def compare_call_result(expected: dict, actual: dict) -> bool: - if expected['filename'] != actual['filename']: + if expected["filename"] != actual["filename"]: print_failure(expected, actual, "Filename mismatch") return False - if expected['lineNumber'] != actual['lineNumber']: + if expected["lineNumber"] != actual["lineNumber"]: print_failure(expected, actual, "Line number mismatch") return False - - all_expected = set(expected['allowedTypes'] + expected['allowedClasses']) - all_actual = set(actual['allowedTypes'] + actual['allowedClasses']) + + all_expected = set(expected["allowedTypes"] + expected["allowedClasses"]) + all_actual = set(actual["allowedTypes"] + actual["allowedClasses"]) # TODO should really prevent empty strings from being added in the first place - all_expected.discard('') - all_actual.discard('') + all_expected.discard("") + all_actual.discard("") - if(all_expected != all_actual): - print_failure(expected, actual, f"Allowed types and classes mismatch, difference: {all_expected.symmetric_difference(all_actual)}") + if all_expected != all_actual: + print_failure( + expected, + actual, + f"Allowed types and classes mismatch, difference: {all_expected.symmetric_difference(all_actual)}", + ) return False # TODO remove these checks, they are mostly for debugging - expected_types = set(expected['allowedTypes']) - actual_types = set(actual['allowedTypes']) - if (expected_types != actual_types): - print_failure(expected, actual, f"(Test still passes) allowed types mismatch, difference: {expected_types.symmetric_difference(actual_types)}") - - expected_classes = set(expected['allowedClasses']) - actual_classes = set(actual['allowedClasses']) - if (expected_classes != actual_classes): - print_failure(expected, actual, f"(Test still passes) Allowed classes mismatch, difference: {expected_classes.symmetric_difference(actual_classes)}") - + expected_types = set(expected["allowedTypes"]) + actual_types = set(actual["allowedTypes"]) + if expected_types != actual_types: + print_failure( + expected, + actual, + f"(Test still passes) allowed types mismatch, difference: {expected_types.symmetric_difference(actual_types)}", + ) + + expected_classes = set(expected["allowedClasses"]) + actual_classes = set(actual["allowedClasses"]) + if expected_classes != actual_classes: + print_failure( + expected, + actual, + f"(Test still passes) Allowed classes mismatch, difference: {expected_classes.symmetric_difference(actual_classes)}", + ) + return True -def compare_results(expected, actual) -> bool: + +def compare_results(expected, actual, base_path) -> bool: if len(expected) != len(actual): print_failure(expected, actual, "Number of results not equal") return False - + + actual = [ + {**item, "filename": str(Path(item["filename"]).relative_to(base_path))} + for item in actual + ] + sorted_expected = sort_by_line_number(expected) sorted_actual = sort_by_line_number(actual) diff --git a/runner.py b/runner.py index 480d99a..81a89df 100755 --- a/runner.py +++ b/runner.py @@ -2,9 +2,10 @@ import dataclasses import json import logging -import sys, os +import os +import sys from pathlib import Path -from subprocess import run, CompletedProcess +from subprocess import CompletedProcess, run from time import time import colorama @@ -14,11 +15,12 @@ logging.root.setLevel(logging.INFO) +# TODO: Move debug.log to output directory fhandler = logging.FileHandler("debug.log") fhandler.setLevel(logging.DEBUG) fhandler.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) handler = logging.StreamHandler(sys.stdout) -handler.setLevel(logging.INFO) +handler.setLevel(logging.DEBUG) handler.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) my_logger = logging.getLogger(__name__) my_logger.setLevel(logging.DEBUG) @@ -49,7 +51,11 @@ def __init__(self, filename, line, startFilePos, endFilePos): def __eq__(self, other): return ( - self.filename == other.filename and self.line == other.line and self.startFilePos == other.startFilePos and self.endFilePos == other.endFilePos) + self.filename == other.filename + and self.line == other.line + and self.startFilePos == other.startFilePos + and self.endFilePos == other.endFilePos + ) def __hash__(self): return hash((self.filename, self.line, self.startFilePos, self.endFilePos)) @@ -67,12 +73,14 @@ class PHPAnalyzer: project_path: Path results_path: Path - def __post_init__(self): - self.process_project() - # "Public" methods - - def process_project(self): + def process_project( + self, + do_create_graph=True, + do_analyze=True, + do_resolve_avail_classes=True, + do_resolve_allowed_classes=True, + ): try: # Make sure the provided project exists if not self.project_path.exists(): @@ -82,14 +90,20 @@ def process_project(self): make_sure_dir_exists(self.results_path) reported_times = {} # For Debug: Put lines here to reduce analysis only to this line + # Format: [(file, line)] + # focus_lines = [("question/type/ddwtos/questiontype.php", 93)] focus_lines = [] # Run the Joern analysis - my_logger.info("Running joern analysis") + # my_logger.info("Running joern analysis") # Path to save the JOERN CPG graph for the project joe_out_graph = self.results_path / "JOEGRAPH" def run_report_time(args, log_prefix, no_exit=False): + my_logger.info(f"{log_prefix} starting") + cmd_strs = [str(c) for c in args] + my_logger.debug(f"executing command: {' '.join(cmd_strs)}") + start_time = time() joe_parse_cmd_ret: CompletedProcess = run(args, capture_output=True) reported_times[log_prefix] = get_elapsed_str(start_time) @@ -97,62 +111,148 @@ def run_report_time(args, log_prefix, no_exit=False): cmd_stdout = joe_parse_cmd_ret.stdout.decode("utf-8") cmd_stderr = joe_parse_cmd_ret.stderr.decode("utf-8") my_logger.debug( - f"[{log_prefix}] CMD=[{args}]\n===:STDOUT:===\n{cmd_stdout}\n===:STDERR:===\n{cmd_stderr}") + f"[{log_prefix}] CMD=[{args}]\n===:STDOUT:===\n{cmd_stdout}\n===:STDERR:===\n{cmd_stderr}" + ) if joe_parse_cmd_ret.returncode != 0: - my_logger.error(f"{log_prefix} failed. Got retcode:{joe_parse_cmd_ret.returncode}. Stopping") + my_logger.error( + f"{log_prefix} failed. Got retcode:{joe_parse_cmd_ret.returncode}. Stopping" + ) if not no_exit: import pdb + pdb.set_trace() exit(joe_parse_cmd_ret.returncode) else: my_logger.info(f"{log_prefix} finished successfully") - run_report_time(["joern-parse", self.project_path, "--language", "php", "--output", joe_out_graph], - "Joern-Prase (graph creation)") + # Joern-Parse (graph creation) phase + # + # - Analyzes the target source code to create a Code Property Graph + # + # - Produces file: + # JOEGRAPH + if do_create_graph: + tmp_graph_path = Path("/tmp", self.project_path.name) + run_report_time( + [ + "joern-parse", + self.project_path, + "--language", + "php", + "--output", + tmp_graph_path, + ], + "Joern-Parse (graph creation)", + ) + + run_report_time( + [ + "joern", + "--script", + Path("tools", "enhance.sc"), + "--param", + f"cpgFile={tmp_graph_path}", + ], + "Joern (graph analysis)", + ) + + # Joern-Script[Analyze] phase + # + # - Analyzes unserialize calls in the target CPG to collect evidence + # of their types, using static duck typing. + # + # Either analyzes all unserialize calls, or restricts analysis to + # only unserialize calls in "focus_lines" + # + # - Produces files: + # joe_analyze.out + # joe_analyze.out.warnings analysis_results_path = self.results_path.joinpath("joe_analyze.out") - joern_analyze_params = ["joern", "--script", Path("tools", "analyze.sc"), "--param", - f"cpgFile={joe_out_graph}", "--param", f"outFile={analysis_results_path}"] - if len(focus_lines) > 0: - focus_lines = ",".join([f"{x[0]}:{x[1]}" for x in focus_lines]) - my_logger.debug(f"Focus lines: {focus_lines}") - joern_analyze_params.extend(["--param", f"focus_lines={focus_lines}"]) - run_report_time(joern_analyze_params, "Joern-Script[Analyze]", True) + if do_analyze: + joern_analyze_params = [ + "joern", + "--script", + Path("tools", "analyze.sc"), + "--param", + f"projectPath={self.project_path}", + "--param", + f"outFile={analysis_results_path}", + ] + if len(focus_lines) > 0: + focus_lines_str = ",".join([f"{x[0]}:{x[1]}" for x in focus_lines]) + my_logger.debug(f"Focus lines: {focus_lines}") + joern_analyze_params.extend( + ["--param", f"focus_lines={focus_lines_str}"] + ) + run_report_time(joern_analyze_params, "Joern-Script[Analyze]", True) + + # Joern-Script[AvailClasses] phase + # + # - TODO + # + # - Produces file: + # availclass.json avail_res_path = self.results_path.joinpath("availclass.json") - psr4_path = Path("tools", "helpers", "get_psr4_mappings.php") - run_report_time( - ["joern", "--script", Path("tools", "resolve_includes.sc"), "--param", f"cpgFile={joe_out_graph}", - "--param", f"outFile={avail_res_path}", "--param", f"psr4Script={psr4_path}"], - "Joern-Script[AvailClasses]") + if do_resolve_avail_classes: + psr4_path = Path("tools", "helpers", "get_psr4_mappings.php") + joern_available_classes_params = [ + "joern", + "--script", + Path("tools", "resolve_includes.sc"), + "--param", + f"projectPath={self.project_path}", + "--param", + f"outFile={avail_res_path}", + "--param", + f"psr4Script={psr4_path}", + ] + if len(focus_lines) > 0: + focus_lines_str = ",".join([f"{x[0]}:{x[1]}" for x in focus_lines]) + my_logger.debug(f"Focus lines: {focus_lines}") + joern_available_classes_params.extend( + ["--param", f"focus_lines={focus_lines_str}"] + ) + run_report_time( + joern_available_classes_params, "Joern-Script[AvailClasses]", True + ) with self.results_path.joinpath("runtime_info.json").open("w") as f: f.write(json.dumps(reported_times, indent=True)) - # Read the results produced from each QUACK sub-component - with analysis_results_path.open() as f: - evidence_entries = json.load(f) - - # TODO: remove this after the fix inside availclass script - with avail_res_path.open() as f: - avail_classes_entries = json.load(f) - for l in avail_classes_entries: - for k, i in l.items(): - if k == "filename": - my_p = Path(i) - l[k] = my_p.relative_to(self.project_path).as_posix() - - with self.results_path.joinpath("availclass_fixed.json").open("w") as f: - f.write(json.dumps(avail_classes_entries, indent=True)) - # END OF TODO - - # Consolidate the available classes with the Psalm analysis results - result_entries = compute_allowed_classes(evidence_entries, avail_classes_entries) # there should only be one project.. - - # Write final results to results JSON file - with self.results_path.joinpath("results.json").open("w") as f: - f.write(json.dumps(result_entries)) - + if ( + do_resolve_allowed_classes + and analysis_results_path.is_file() + and avail_res_path.is_file() + ): + # Read the results produced from each QUACK sub-component + with analysis_results_path.open() as f: + evidence_entries = json.load(f) + + with avail_res_path.open() as f: + avail_classes_entries = json.load(f) + + with self.results_path.joinpath("availclass_fixed.json").open("w") as f: + f.write(json.dumps(avail_classes_entries, indent=True)) + + # END OF TODO + + # Policy generation phase + # + # - Analyzes the evidence produced in the Analyze phase and the + # available classes produced in the AvailClasses phase to + # compute the allowed classes policies for each unserialize call. + # - Produces file: + # TODO + # there should only be one project.. + result_entries = compute_allowed_classes( + evidence_entries, avail_classes_entries + ) + + # Write final results to results JSON file + with self.results_path.joinpath("results.json").open("w") as f: + f.write(json.dumps(result_entries)) except ValueError as e: my_logger.error(f"{e.__class__.__name__}:{e}") @@ -160,9 +260,40 @@ def run_report_time(args, log_prefix, no_exit=False): def main(): arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("project_path", help="Path to project to analyze", type=Path) - arg_parser.add_argument("--output-path", type=Path, default=None, - help="Path for keeping Quack's outputs (defaults to project path)") + arg_parser.add_argument( + "project_path", help="Path to project to analyze", type=Path + ) + arg_parser.add_argument( + "--output-path", + type=Path, + default=None, + help="Path for keeping Quack's outputs (defaults to project path)", + ) + arg_parser.add_argument( + "--no-create-graph", + dest="do_create_graph", + action="store_false", + help="Do not create the Joern graph", + ) + arg_parser.add_argument( + "--no-analyze", + dest="do_analyze", + action="store_false", + help="Do not analyze for class evidence", + ) + arg_parser.add_argument( + "--no-resolve-avail-classes", + dest="do_resolve_avail_classes", + action="store_false", + help="Do not resolve the available classes", + ) + arg_parser.add_argument( + "--no-resolve-allowed-classes", + dest="do_resolve_allowed_classes", + action="store_false", + help="Do not resolve the allowed classes", + ) + # TODO: Add argument for specifying focus lines args = arg_parser.parse_args() # User requested to analyze a specific project @@ -172,7 +303,13 @@ def main(): project_path = args.project_path output_path = args.project_path if args.output_path is None else args.output_path - PHPAnalyzer(project_path, output_path) + analyzer = PHPAnalyzer(project_path, output_path) + analyzer.process_project( + do_create_graph=args.do_create_graph, + do_analyze=args.do_analyze, + do_resolve_avail_classes=args.do_resolve_avail_classes, + do_resolve_allowed_classes=args.do_resolve_allowed_classes, + ) if __name__ == "__main__": diff --git a/tools/analyze.sc b/tools/analyze.sc index 2cb321c..9469b17 100644 --- a/tools/analyze.sc +++ b/tools/analyze.sc @@ -8,81 +8,135 @@ * the evaluated projects, and can be addressed if encountered in future proejcts * * Type matching for rules for unserialized values (corresponds to Table I): - * - Exact -- evidence for exact type - * - type -- the type we deduced - * - reason -- reason for the decision - * - FuncArg -- TypeOf(parameter of function or static method value was passed to) - * - MethodArg -- TypeOf(parameter of method value was passed to) - * - Return -- TypeOf(return value of encapsulating method/function) - * - Cast -- Type value was cast to - * - Ternary -- Type of other value in ternary operator - * - Compared -- compared to value of this type - * - StringOp -- was used in a string operator (e.g., concat) - * - Scalar -- part of a scalar operation - * - Arithmetic -- was used in an arithmetic operator - * - DynamicCall -- used in a dynamic call so we can't say anything about its - * type. When this is encountered, this value is just a placeholder meaning - * all types should be allowed - * - Duck -- evidence for duck typing - * - types -- the types we deduced, separated with '|' - * - reason -- reason for the decision - * - HasMethod -- has method that was called on unserialized object - * - HasField -- has field that was accessed on unserialized object - * - HasToString -- has __toString method and was used in a String operator - * - AssignedToField -- was assigned to a field (property) with known/deduced type + * - Exact -- evidence for exact type + * - type -- the type we deduced + * - reason -- reason for the decision + * - FuncArg -- TypeOf(parameter of function or static method value was passed to) + * - MethodArg -- TypeOf(parameter of method value was passed to) + * - Return -- TypeOf(return value of encapsulating method/function) + * - Cast -- Type value was cast to + * - Ternary -- Type of other value in ternary operator + * - Compared -- compared to value of this type + * - StringOp -- was used in a string operator (e.g., concat) + * - Scalar -- part of a scalar operation + * - Arithmetic -- was used in an arithmetic operator + * - DynamicCall -- used in a dynamic call so we can't say anything about its + * type. When this is encountered, this value is just a placeholder meaning + * all types should be allowed + * - Duck -- evidence for duck typing + * - types -- the types we deduced, separated with '|' + * - reason -- reason for the decision + * - HasMethod -- has method that was called on unserialized object + * - HasField -- has field that was accessed on unserialized object + * - HasToString -- has __toString method and was used in a String operator + * - AssignedToField -- was assigned to a field (property) with known/deduced type * * Collected conditions types for unserialized values (for debugging purposes): - * - ArgToFuncIdx/ArgToMethodIdx -- Passed as argument Idx to function (or static method)/method Arg-To-Func/Method - * - callerFullName: full name of function/method value is being passed to - * - callerName: name of function/method value is being passed to - * - argIdx: argument index value is being passed to - * - Returns -- is returned - * - methodName: name of method that returns the value - * - Conditional -- part of a conditional (e.g., ternary) - * - argIdx: argument index of conditional value is being passed to (e.g., in ternary, 'true ? "a" : unserialize($x)', would be index 1) - * - AssignedToArrayIdx -- is assigned to an array index (e.g., $array[$idx] = unserialize()) - * - array: code of array value was assigned to (e.g., for '$array[$idx] = unserialize()', array is '$array') - * - arrayIdx: code of array index value was assigned to (e.g., for '$array[$idx] = unserialize()', arrayIdx is '$idx') - * - AssignedToField -- is assigned to the field of an object - * - objectName: Name of object (variable) that has the field - * - fieldType: Property or variable - * - fieldName: Name of field (or variable) - * - CallsMethod -- an object and calls a method (e.g., $x = unserialize(); $x->hello()) - * - methodFullName: full name of method being called - * - methodName: name of method being called - * - ArrayRef -- is referenced as an array (a[|]). - * Actually this can also be a string access (str[int-offset]) - * - arrayIdx: similar to AssignedToArrayIdx - * - Scalar -- part of a scalar operation - * - type: type of scalar - * - FieldAccess -- accessing a field/property (read or write) on a tainted var - * - fieldName: Name of field being accessed - * - varFieldName: Name of variable representing field being accessed - * - Iterated -- was used in an iterator - * - Comparison -- was used in a comparison - * - comparisonType: type of comparison (e.g., comparison with literal, comparison with call (i.e., return value), comparison with variable etc.) - * - comparedValue: type of literal in literal comparison, name of function in call comparison, var name in var (identifier) comparison - * - ArithmeticOp -- was used in an arithmetic operation - * - LogicalOp -- was used in an logical operation - * - type: operator name - * - InstanceOf -- was passed to instanceOf - * - class: class being checked against - * - ClassAlloc -- was used as a dynamic class name for a class allocation (new $var()) - * - ByRef -- was passed by reference or assigned by reference + * - ArgToFuncIdx/ArgToMethodIdx -- Passed as argument Idx to function (or static method)/method Arg-To-Func/Method + * - callerFullName: full name of function/method value is being passed to + * - callerName: name of function/method value is being passed to + * - argIdx: argument index value is being passed to + * - Returns -- is returned + * - methodName: name of method that returns the value + * - Conditional -- part of a conditional (e.g., ternary) + * - argIdx: argument index of conditional value is being passed to (e.g., in ternary, 'true ? "a" : unserialize($x)', would be index 1) + * - AssignedToArrayIdx -- is assigned to an array index (e.g., $array[$idx] = unserialize()) + * - array: code of array value was assigned to (e.g., for '$array[$idx] = unserialize()', array is '$array') + * - arrayIdx: code of array index value was assigned to (e.g., for '$array[$idx] = unserialize()', arrayIdx is '$idx') + * - AssignedToField -- is assigned to the field of an object + * - objectName: Name of object (variable) that has the field + * - fieldType: Property or variable + * - fieldName: Name of field (or variable) + * - CallsMethod -- an object and calls a method (e.g., $x = unserialize(); $x->hello()) + * - methodFullName: full name of method being called + * - methodName: name of method being called + * - ArrayRef -- is referenced as an array (a[|]). + * Actually this can also be a string access (str[int-offset]) + * - arrayIdx: similar to AssignedToArrayIdx + * - Scalar -- part of a scalar operation + * - type: type of scalar + * - FieldAccess -- accessing a field/property (read or write) on a tainted var + * - fieldName: Name of field being accessed + * - varFieldName: Name of variable representing field being accessed + * - Iterated -- was used in an iterator + * - Comparison -- was used in a comparison + * - comparisonType: type of comparison (e.g., comparison with literal, comparison with call (i.e., return value), comparison with variable etc.) + * - comparedValue: type of literal in literal comparison, name of function in call comparison, var name in var (identifier) comparison + * - ArithmeticOp -- was used in an arithmetic operation + * - LogicalOp -- was used in an logical operation + * - type: operator name + * - InstanceOf -- was passed to instanceOf + * - class: class being checked against + * - ClassAlloc -- was used as a dynamic class name for a class allocation (new $var()) + * - ByRef -- was passed by reference or assigned by reference */ +import scala.io.Source import io.shiftleft.codepropertygraph.generated.nodes.{ Call => CallNode } import scala.collection.mutable.ListBuffer import scala.collection.immutable.ArraySeq import scala.collection.mutable +import scala.annotation.tailrec + +import java.nio.file.{Path, Paths, Files} import upickle.default.* +def join_paths(p1: String, p2: String) : String = { + return Paths.get(p1, p2).normalize().toString() +} + +def writeFile(path: String, content: String): Unit = { + try { + val parentDir = Paths.get(path).getParent + if (parentDir != null) Files.createDirectories(parentDir) + Files.writeString(Paths.get(path), content) + // logger.info(s"Successfully wrote ${content.length} bytes to $path") + } catch { + case e: Exception => + // logger.error(s"[!] ERROR: Failed to write to file $path.") + // logger.error(s" Reason: ${e.getMessage}") + } +} + var maxDepth: Int = 3 var built_in_log = "/processed/builtins.txt" case class UnserEntry(filename: String, lineNumber: Integer, conditions: Set[Map[String, String]]) derives ReadWriter +def getDefId(use: AstNode): String = { + + if (use.isIdentifier) { + val declarationNodeTraversal = use.asInstanceOf[Identifier].out("REF") + val declarationNodeOption = declarationNodeTraversal.lastOption + + if (declarationNodeOption.isDefined) { + val defNode = declarationNodeOption.get + return defNode.id.toString + } + + } + + return use.id.toString +} + +def getNodeName(n: AstNode): String = { + n match { + case id: Identifier => id.name + case call: CallNode => call.name + case method: Method => method.name + case param: MethodParameterIn => param.name + case ret: MethodReturn => ret.typeFullName // Returns don't have a name, type is descriptive + case ret: Return => ret.code + case local: Local => local.name + case member: Member => member.name + case typeDecl: TypeDecl => typeDecl.name + case literal: Literal => literal.code + case block: Block => "BLOCK" + case _ => "UNKNOWN_NODE" // Default fallback + } +} + // Add new evidence def createCondition(condType: String, extra: mutable.Map[String, String]=mutable.Map()): Map[String, String] = { return extra.addOne(("condType" -> condType)).toMap @@ -91,44 +145,31 @@ def createCondition(condType: String, extra: mutable.Map[String, String]=mutable // Returns the type of the node, or all the dynamic type hints separated with '|' // Returns ANY if type is not known def getNodeType(n: AstNode) : String = { - - // FIXME collect type if node is call - if (n.isInstanceOf[CallNode] || n.isInstanceOf[Block]) { - return "ANY" + val (typeFullName, dynamicTypeHints) = n match { + case id: Identifier => (id.typeFullName, id.dynamicTypeHintFullName) + case mr: MethodReturn => (mr.typeFullName, mr.dynamicTypeHintFullName) + case p: MethodParameterIn => (p.typeFullName, p.dynamicTypeHintFullName) + case l: Literal => (l.typeFullName, l.dynamicTypeHintFullName) + case m: Member => (m.typeFullName, m.dynamicTypeHintFullName) + case _: CallNode | _: Block => ("ANY", Seq.empty[String]) + case _ => + throw new Exception("Got unknown type of node for extracting type: " + n) } - val type_full_name = if (n.isIdentifier) n.asInstanceOf[Identifier].typeFullName - else if (n.isInstanceOf[MethodReturn]) n.asInstanceOf[MethodReturn].typeFullName - else if (n.isInstanceOf[MethodParameterIn]) n.asInstanceOf[MethodParameterIn].typeFullName - else if (n.isInstanceOf[Literal]) n.asInstanceOf[Literal].typeFullName - else if (n.isInstanceOf[Member]) n.asInstanceOf[Member].typeFullName - else throw new Exception("Got unknown type of node for extracting type: " + n) - - val type_full_name_str = type_full_name.asInstanceOf[String] - - val dynamic_type_hints = if (n.isIdentifier) n.asInstanceOf[Identifier].dynamicTypeHintFullName - else if (n.isInstanceOf[MethodReturn]) n.asInstanceOf[MethodReturn].dynamicTypeHintFullName - else if (n.isInstanceOf[MethodParameterIn]) n.asInstanceOf[MethodParameterIn].dynamicTypeHintFullName - else if (n.isInstanceOf[Literal]) n.asInstanceOf[Literal].dynamicTypeHintFullName - else if (n.isInstanceOf[Member]) n.asInstanceOf[Member].dynamicTypeHintFullName - else throw new Exception("Got unknown type of node for extracting type: " + n) - - val dynamic_type_hints_iter = dynamic_type_hints.asInstanceOf[ArraySeq[String]] - - if (type_full_name_str == "ANY") { - if (dynamic_type_hints_iter.length != 0) { - dynamic_type_hints_iter.l.mkString("|") + if (typeFullName == "ANY") { + if (dynamicTypeHints.nonEmpty) { + dynamicTypeHints.mkString("|") } else { - return "ANY" + "ANY" } } else { - return type_full_name_str + typeFullName } } // Get the node that represents the assigned variable def getAssignedVar(assignment: CallNode) : AstNode = { - return assignment.argument.argumentIndex(1).l(0) + return assignment.argument.argumentIndex(1).head } // Check if method is builtin @@ -141,50 +182,45 @@ def isBuiltIn(method: Method) : Boolean = { } // Get the id of the scope of the given node in the CPG +@tailrec def getScopeId(n: AstNode) : Long = { - - if (n._astIn.length == 0) { - return -1 + if (n._astIn.isEmpty) { + -1L + } else { + n.astParent match { + case call: CallNode => call.method.id + case ret: Return => ret.method.id + case method: Method => method.id + case typeDecl: TypeDecl => typeDecl.id + case parent @ (_: ControlStructure | _: Block) => getScopeId(parent) + case parent => getScopeId(parent) // Default case to recurse + } } - - val parent = n.astParent - - return if (parent.isCall) parent.asInstanceOf[CallNode].method.id - else if (parent.isReturn) parent.asInstanceOf[Return].method.id - else if (parent.isMethod) parent.asInstanceOf[Method].id - else if (parent.isTypeDecl) parent.asInstanceOf[TypeDecl].id - else if (parent.isControlStructure || parent.isBlock) getScopeId(parent.astParent) - else throw new Exception("Unknown node in scope id check: " + parent) - } // Add the classes that have a __toString method to the evidence -def addHaveToString(conds: ListBuffer[Map[String, String]]) = { - val have_to_string = cpg.method.name("__toString").typeDecl.name.mkString("|") +def addHaveToString( + conds: mutable.Set[Map[String, String]], + methodCache: Map[String, List[Method]], + nodeId: Long) = { + val have_to_string = methodCache.getOrElse("__toString", List.empty).flatMap(_.typeDecl.name).mkString("|") conds += createCondition("Duck", mutable.Map("reason" -> "HasToString", - "type" -> have_to_string)) + "type" -> have_to_string, + "nodeId" -> nodeId.toString)) } // Follow all uses for the given parameter in the method/function -def collectParameterUses(conds: ListBuffer[Map[String, String]], analyzed: mutable.Set[Long], - parameter: MethodParameterIn, depth: Int, warnings: ListBuffer[String]) : Boolean = { +def collectParameterUses(conds: mutable.Set[Map[String, String]], analyzed: mutable.Set[Long], + parameter: MethodParameterIn, depth: Int, warnings: ListBuffer[String], + methodCache: Map[String, List[Method]], memberCache: Map[String, List[Member]]) : Boolean = { - // Get the refs of the parameter in the method - val refs = cpg.graph.edges.filter(_.isInstanceOf[Ref]).l - val parameter_refs = refs.filter(e => { - (e.inNode.isInstanceOf[AstNode]) && - (e.outNode.isInstanceOf[AstNode]) && - (e.inNode.asInstanceOf[AstNode].id == parameter.id) - }) - - // Get the uses of the parameter - val parameter_uses = parameter_refs.map(n => n.outNode.asInstanceOf[AstNode]) - .filter(n => getScopeId(n) == parameter.method.id) + val parameter_uses = parameter.in("REF").map(_.asInstanceOf[AstNode]).filter(n => getScopeId(n) == parameter.method.id) // Iterate and collect evidence for (use <- parameter_uses) { - extractConditions(conds, use, analyzed, depth, warnings) + // println(use) + extractConditions(conds, use, analyzed, depth, warnings, methodCache, memberCache) } return true @@ -192,8 +228,9 @@ def collectParameterUses(conds: ListBuffer[Map[String, String]], analyzed: mutab } // Collect uses of the parameter in a method -def collectParameterUsesFromMethod(conds: ListBuffer[Map[String, String]], analyzed: mutable.Set[Long], - method: Method, nargs: Int, argIdx: Int, depth: Int, warnings: ListBuffer[String]) : Boolean = { +def collectParameterUsesFromMethod(conds: mutable.Set[Map[String, String]], analyzed: mutable.Set[Long], + method: Method, nargs: Int, argIdx: Int, depth: Int, warnings: ListBuffer[String], + methodCache: Map[String, List[Method]], memberCache: Map[String, List[Member]]) : Boolean = { // We reached max depth, stop here if (depth == maxDepth) { @@ -207,18 +244,19 @@ def collectParameterUsesFromMethod(conds: ListBuffer[Map[String, String]], analy return false } - println("Following parameter use in method " + method.fullName) - // Get the parameter - val parameter = method.parameter.index(argIdx).l(0) + val parameter = method.parameter.index(argIdx).head - collectParameterUses(conds, analyzed, parameter, depth, warnings) + println("Following parameter " + parameter + " use in method " + method.fullName) + + collectParameterUses(conds, analyzed, parameter, depth, warnings, methodCache, memberCache) } // Collect uses of the parameter in a function -def collectParameterUsesFromFunc(conds: ListBuffer[Map[String, String]], analyzed: mutable.Set[Long], - method: Method, nargs: Int, argIdx: Int, depth: Int, warnings: ListBuffer[String]) : Boolean = { +def collectParameterUsesFromFunc(conds: mutable.Set[Map[String, String]], analyzed: mutable.Set[Long], + method: Method, nargs: Int, argIdx: Int, depth: Int, warnings: ListBuffer[String], + methodCache: Map[String, List[Method]], memberCache: Map[String, List[Member]]) : Boolean = { // We reached max depth, stop here if (depth == maxDepth) { @@ -235,9 +273,9 @@ def collectParameterUsesFromFunc(conds: ListBuffer[Map[String, String]], analyze println("Following parameter use in function " + method.fullName) // Get the parameter (+1 cause 0 is $this, doesn't exist in functions) - val parameter = method.parameter.index(argIdx + 1).l(0) + val parameter = method.parameter.index(argIdx + 1).head - collectParameterUses(conds, analyzed, parameter, depth, warnings) + collectParameterUses(conds, analyzed, parameter, depth, warnings, methodCache, memberCache) } @@ -248,27 +286,33 @@ def helpsWithTyping(type_str: String) : Boolean = { // https://stackoverflow.com/questions/5522572/how-to-split-a-string-by-a-string-in-scala val types = type_str.split("\\|") // These don't give us any real type evidence - val types_filtered = types.filter(x => (x != "ANY" && x != "array" && x != "null" && !(x contains "->"))) + val types_filtered = types.filter(x => ( + x != "ANY" && + !x.contains("") && + x != "array" && + x != "null" && + !x.contains("->") + )) return (types_filtered.length > 0) } // Examine uses of the given class field in order to try to infer its type -def collectFieldUses(conds: ListBuffer[Map[String, String]], analyzed: mutable.Set[Long], - the_class: TypeDecl, member: Member, depth: Int, warnings: ListBuffer[String]) : Boolean = { +def collectFieldUses(conds: mutable.Set[Map[String, String]], analyzed: mutable.Set[Long], + the_class: TypeDecl, member: Member, depth: Int, warnings: ListBuffer[String], + methodCache: Map[String, List[Method]], memberCache: Map[String, List[Member]]) : Boolean = { println("Collecting field uses for field '" + member.name + "' of class '" + the_class.name + "'") // Get all field identifiers that are part of a field access operation (should // always be but just sanity check), then get their respective objects, and // filter only the ones that are of type 'the_class' - val field_uses = cpg.all.filter(_.isInstanceOf[FieldIdentifier]) - .map(_.asInstanceOf[FieldIdentifier]).canonicalName(member.name).astParent + val field_uses = cpg.fieldIdentifier.canonicalName(member.name).astParent .filter(x => (x.isInstanceOf[CallNode] && x.asInstanceOf[CallNode].name == ".fieldAccess")) - .map(_.asInstanceOf[CallNode]).map(_.argument.l(0)) + .map(_.asInstanceOf[CallNode]).map(_.argument.head) .filter(x => (x.isIdentifier && x.asInstanceOf[Identifier].typeFullName == the_class.name)).l for (use <- field_uses) { - extractConditions(conds, use.astParent, analyzed, depth, warnings) + extractConditions(conds, use.astParent, analyzed, depth, warnings, methodCache, memberCache) } return true @@ -277,7 +321,7 @@ def collectFieldUses(conds: ListBuffer[Map[String, String]], analyzed: mutable.S // Try to infer the type of an array slice def tryInferSliceType(index_access: CallNode) : Set[String] = { - val arg0 = index_access.argument.l(0) + val arg0 = index_access.argument.head val arg1 = index_access.argument.l(1) if (arg0.isIdentifier && arg1.isLiteral) { @@ -317,35 +361,37 @@ def tryInferSliceType(index_access: CallNode) : Set[String] = { } // Collect evidence from the given assigned deserialized variable -def followAssignedVar(conds: ListBuffer[Map[String, String]], assigned_var: AstNode, - analyzed: mutable.Set[Long], depth: Int, warnings: ListBuffer[String]) : Boolean = { +def followAssignedVar(conds: mutable.Set[Map[String, String]], assigned_var: AstNode, + analyzed: mutable.Set[Long], depth: Int, warnings: ListBuffer[String], + methodCache: Map[String, List[Method]], memberCache: Map[String, List[Member]]) : Boolean = { - println("Following assignment to: " + assigned_var) + println("Following assignment to: " + getNodeName(assigned_var) + " (" + assigned_var + ")") // First check if the assigned variable is actually an index of an array if (assigned_var.isCall) { val call = assigned_var.asInstanceOf[CallNode] - if (call.methodFullName == "list") { - // Assigned to a list of variables, analyze all - // https://www.php.net/manual/en/function.list.php - for (arg <- call.argument) { - val array_index = arg.argumentIndex - 1 - - conds += createCondition("ArrayRef", - mutable.Map("arrayIdx" -> array_index.toString)) - - // TODO: should we really follow each element? - followAssignedVar(conds, arg, analyzed, depth, warnings) - } - return false - } else if (call.methodFullName == ".doubleArrow") { + // if (call.methodFullName == "list") { + // // Assigned to a list of variables, analyze all + // // https://www.php.net/manual/en/function.list.php + // for (arg <- call.argument) { + // val array_index = arg.argumentIndex - 1 + + // conds += createCondition("ArrayRef", + // mutable.Map("arrayIdx" -> array_index.toString)) + + // // TODO: should we really follow each element? + // followAssignedVar(conds, arg, analyzed, depth, warnings, methodCache, memberCache) + // } + // return false + // } else + if (call.methodFullName == ".doubleArrow") { // Get $value from $key => $value - return followAssignedVar(conds, call.argument.argumentIndex(2).l(0), analyzed, depth, warnings) + return followAssignedVar(conds, call.argument.argumentIndex(2).head, analyzed, depth, warnings, methodCache, memberCache) } else if (call.methodFullName == ".fieldAccess") { // Assigned to a field val field_access = assigned_var.asInstanceOf[CallNode] - val fobject = field_access.argument.argumentIndex(1).l(0) + val fobject = field_access.argument.argumentIndex(1).head val fobj_type = getNodeType(fobject) if (!fobject.isIdentifier) { @@ -354,7 +400,7 @@ def followAssignedVar(conds: ListBuffer[Map[String, String]], assigned_var: AstN return false; } val object_name = fobject.asInstanceOf[Identifier].name - val field = field_access.argument.argumentIndex(2).l(0) + val field = field_access.argument.argumentIndex(2).head if (field.isFieldIdentifier) { @@ -367,27 +413,31 @@ def followAssignedVar(conds: ListBuffer[Map[String, String]], assigned_var: AstN ) // Try to check if we know what the type of this property is before tainting it - val field_members_with_name = cpg.member.name(field_ident.canonicalName).l - if (fobj_type != "ANY" && !(fobj_type contains "|")) { + val field_members_with_name = memberCache.getOrElse(field_ident.canonicalName, List.empty) + if (fobj_type != "ANY" && !(fobj_type.contains("|"))) { // Use 'fullName' here to match namespaces as well - val classes_with_field = field_members_with_name.typeDecl.filter(_.fullName == fobj_type).l + val classes_with_field = field_members_with_name.flatMap(_.typeDecl).filter(_.fullName == fobj_type) if (classes_with_field.length == 1) { - val the_class = classes_with_field.l(0) - val member = the_class.member.name(field_ident.canonicalName).l(0) + val the_class = classes_with_field.head + val member = the_class.member.name(field_ident.canonicalName).head val member_type = getNodeType(member) if (helpsWithTyping(member_type)) { conds += createCondition("Duck", - mutable.Map("reason" -> "AssignedToField", - "type" -> member_type, "field" -> member.name)) - - if (member_type contains "string") { - addHaveToString(conds) + mutable.Map( + "reason" -> "AssignedToField", + "type" -> member_type, + "field" -> member.name, + "nodeId" -> getDefId(assigned_var) + )) + + if (member_type.contains("string")) { + addHaveToString(conds, methodCache, call.astParent.id) } // No need to follow it since we deduced the type return false } else { - return collectFieldUses(conds, analyzed, the_class, member, depth, warnings) + return collectFieldUses(conds, analyzed, the_class, member, depth, warnings, methodCache, memberCache) } } } @@ -402,55 +452,41 @@ def followAssignedVar(conds: ListBuffer[Map[String, String]], assigned_var: AstN } else { throw new Exception("Unknown type for field access!") } - return followAssignedVar(conds, fobject, analyzed, depth, warnings) + return followAssignedVar(conds, fobject, analyzed, depth, warnings, methodCache, memberCache) } else if (call.methodFullName == ".indexAccess") { // Assigned to array index val index_access = assigned_var.asInstanceOf[CallNode] - val array = index_access.argument.argumentIndex(1).l(0) + val array = index_access.argument.argumentIndex(1).head // FIXME try to figure out if this is a literal conds += createCondition("AssignedToArrayIdx", - mutable.Map("arrayIdx" -> index_access.argument.argumentIndex(2).l(0).code, + mutable.Map("arrayIdx" -> index_access.argument.argumentIndex(2).head.code, "array" -> array.code)) - return followAssignedVar(conds, array, analyzed, depth, warnings) + return followAssignedVar(conds, array, analyzed, depth, warnings, methodCache, memberCache) } else { throw new Exception("Unknown call in assignment analysis: " + call.methodFullName + " (" + call.code + ")" + call.method.filename) } } - // Get the all the ref edges. Make sure to cast them to lists such that - // functions such as `filter` work as expected - val refs = cpg.graph.edges.filter(_.isInstanceOf[Ref]).l - - // Get the node that represents the local variable - val assigned_var_refs = refs.filter(e => { - (e.inNode.isInstanceOf[AstNode]) && - (e.outNode.isInstanceOf[AstNode]) && - (e.outNode.asInstanceOf[AstNode].id == assigned_var.id) - }) - if (assigned_var_refs.length != 1) { - throw new Exception("Expected 1 assigned_var ref, found " + assigned_var_refs.length) + val local_var_node_l = assigned_var.out("REF").l + if (local_var_node_l.length != 1) { + throw new Exception("Expected 1 declaration for assigned_var, found " + local_var_node_l.length) } - val local_var_node = assigned_var_refs(0).inNode + val local_var_node = local_var_node_l.head - // Get all the other uses of the assigned-to variable and analyze them - val local_var_refs = refs.filter(e => { - (e.inNode.isInstanceOf[AstNode]) && - (e.outNode.isInstanceOf[AstNode]) && - (e.inNode.asInstanceOf[AstNode].id == local_var_node.id) - }).l + val all_uses = local_var_node.in("REF").map(_.asInstanceOf[AstNode]) // Remove the use we started and the ones in different scope and analyze the rest // We check scope by checking if the id of the method containing the variable matches // with the rest of the uses - val var_uses = local_var_refs.map(n => n.outNode.asInstanceOf[AstNode]) + val var_uses = all_uses .filter(_.id != assigned_var.id) .filter(n => getScopeId(n) == assigned_var.astParent.asInstanceOf[CallNode].method.id) .filter(n => {n.lineNumber.getOrElse(-1).asInstanceOf[Int] >= assigned_var.lineNumber.getOrElse(-1).asInstanceOf[Int]}) .l for (use <- var_uses) { - extractConditions(conds, use, analyzed, depth, warnings) + extractConditions(conds, use, analyzed, depth, warnings, methodCache, memberCache) } true @@ -476,11 +512,11 @@ def extractIteratorVariable(iterator_parent: AstNode) : AstNode = { + iterator_parent.asInstanceOf[CallNode].methodFullName) } - var loop_value = val_assignment.asInstanceOf[CallNode].argument.argumentIndex(1).l(0) + var loop_value = val_assignment.asInstanceOf[CallNode].argument.argumentIndex(1).head // Is ($key => $value), just get $value if (loop_value.isInstanceOf[CallNode] && loop_value.asInstanceOf[CallNode].methodFullName == ".doubleArrow") { - loop_value = loop_value.asInstanceOf[CallNode].argument.argumentIndex(2).l(0) + loop_value = loop_value.asInstanceOf[CallNode].argument.argumentIndex(2).head } return loop_value @@ -488,8 +524,9 @@ def extractIteratorVariable(iterator_parent: AstNode) : AstNode = { // The main function that applies the typing rules on a given deserialized node // and collects type information -def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, - analyzed: mutable.Set[Long], depth: Int, warnings: ListBuffer[String]) : Boolean = { +def extractConditions(conds: mutable.Set[Map[String, String]], n: AstNode, + analyzed: mutable.Set[Long], depth: Int, warnings: ListBuffer[String], + methodCache: Map[String, List[Method]], memberCache: Map[String, List[Member]]) : Boolean = { // We reached max depth, stop here if (depth == maxDepth) { @@ -505,7 +542,7 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, } println() - println("Extracting conditions for node: " + n) + println("Extracting conditions for node: " + getNodeName(n) + " (" + n + ")") // Cast the node to its specific class so that we can use certain properties // that don't exist in AstNode @@ -533,22 +570,22 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, // Ignore error suppress prefixes case ".errorSuppress" => { println("Ignoring errorSuppress node") - return extractConditions(conds, parent, analyzed, depth, warnings) + return extractConditions(conds, parent, analyzed, depth, warnings, methodCache, memberCache) } case ".doubleArrow" => { // Part of a foreach, we should have processed it already - return extractConditions(conds, parent, analyzed, depth, warnings) + return extractConditions(conds, parent, analyzed, depth, warnings, methodCache, memberCache) } case "Iterator.next" => { // Part of an iterator, we are extracting the conditions from // Iterator.current - return extractConditions(conds, parent, analyzed, depth, warnings) + return extractConditions(conds, parent, analyzed, depth, warnings, methodCache, memberCache) } // Value is an array and is being indexed case ".indexAccess" => { val index_access = parent.asInstanceOf[CallNode] conds += createCondition("ArrayRef", - mutable.Map("arrayIdx" -> index_access.argument.argumentIndex(2).l(0).code)) + mutable.Map("arrayIdx" -> index_access.argument.argumentIndex(2).head.code)) } // Value was passed by reference (or assigned as a reference) case ".addressOf" => { @@ -564,7 +601,7 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, val loop_value = extractIteratorVariable(loop_assign_call) // We essentially treat the iterated value as a new variable being reassigned // in the loop body - followAssignedVar(conds, loop_value, analyzed, depth, warnings) + followAssignedVar(conds, loop_value, analyzed, depth, warnings, methodCache, memberCache) } // Field of value is being accessed case ".fieldAccess" => { @@ -580,17 +617,21 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, if (arg_idx != 1) { throw new Exception("Field access but argument isn't the one being accessed or the field name! (" + field_access.code + ")") } - val field_identifier = field_access.argument.argumentIndex(2).l(0) + val field_identifier = field_access.argument.argumentIndex(2).head if (field_identifier.isFieldIdentifier) { // FIXME maybe filter based on field type if we have it // Get the classes that have a field with this name val field_name = field_identifier.asInstanceOf[FieldIdentifier].canonicalName - val field_members = cpg.member.name(field_name) - val classes_with_field = field_members.typeDecl.name.mkString("|") + val field_members = memberCache.getOrElse(field_name, List.empty) + val classes_with_field = field_members.map(_.typeDecl.name).mkString("|") conds += createCondition("Duck", - mutable.Map("reason" -> "HasField", - "type" -> classes_with_field, "field" -> field_name)) + mutable.Map( + "reason" -> "HasField", + "type" -> classes_with_field, + "field" -> field_name, + "nodeId" -> getDefId(n) + )) conds += createCondition("FieldAccess", mutable.Map( @@ -604,8 +645,11 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, } else if (field_identifier.isExpression) { // Dynamic field access, so we can't say anything about the type conds += createCondition("Exact", - mutable.Map("reason" -> "DynamicCall", - "type" -> "ANY", "call" -> call.name) + mutable.Map( + "reason" -> "DynamicCall", + "type" -> "ANY", + "call" -> call.name, + "nodeId" -> getDefId(n)) ) return false // No need to collect anything else about this } @@ -617,13 +661,16 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, // Value is being cast to a type case ".cast" => { val casted_type = call.argument(1).asInstanceOf[TypeRef].typeFullName + val casted_obj = call.argument(2) conds += createCondition("Exact", mutable.Map( "type" -> casted_type, - "reason" -> "Cast")) + "reason" -> "Cast", + "nodeId" -> getDefId(n) + )) if (casted_type == "string") { - addHaveToString(conds) + addHaveToString(conds, methodCache, call.astParent.id) } // No need to collect anything else about this since we got the exact @@ -640,7 +687,7 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, //FIXME maybe add different type evidence for identical vs the rest val arg_idx = getArgIdx(call, n_cast) val other_idx = if (arg_idx == 0) 2 else 1 - val other_arg = call.argument.argumentIndex(other_idx).l(0) + val other_arg = call.argument.argumentIndex(other_idx).head if (other_arg.isLiteral) { conds += createCondition("Comparison", mutable.Map("comparisonType" -> "literal", @@ -672,7 +719,11 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, mutable.Map("type" -> parent.asInstanceOf[CallNode].name)) conds += createCondition("Exact", - mutable.Map("reason" -> "Arithmetic", "type" -> "numeric")) + mutable.Map( + "reason" -> "Arithmetic", + "type" -> "numeric", + "nodeId" -> getDefId(n) + )) } // Bitwise operation @@ -684,7 +735,11 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, mutable.Map("type" -> parent.asInstanceOf[CallNode].name)) conds += createCondition("Exact", - mutable.Map("reason" -> "Arithmetic", "type" -> "numeric")) + mutable.Map( + "reason" -> "Arithmetic", + "type" -> "numeric", + "nodId" -> getDefId(n) + )) } // Arithmetic assignment (e.g. +=, -=) case arithmetic_assignment @ (".assignmentPlus" | @@ -696,12 +751,16 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, mutable.Map("type" -> parent.asInstanceOf[CallNode].name)) conds += createCondition("Exact", - mutable.Map("reason" -> "Arithmetic", "type" -> "numeric")) + mutable.Map( + "reason" -> "Arithmetic", + "type" -> "numeric", + "nodeId" -> getDefId(n) + )) var assigned_var = getAssignedVar(parent.asInstanceOf[CallNode]) // Being assigned with another variable if (assigned_var.id != n.id) { - followAssignedVar(conds, assigned_var, analyzed, depth, warnings) + followAssignedVar(conds, assigned_var, analyzed, depth, warnings, methodCache, memberCache) } } // Bitwise assignment (e.g. &=, |=) @@ -713,12 +772,16 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, mutable.Map("type" -> parent.asInstanceOf[CallNode].name)) conds += createCondition("Exact", - mutable.Map("reason" -> "Arithmetic", "type" -> "numeric")) + mutable.Map( + "reason" -> "Arithmetic", + "type" -> "numeric", + "nodeId" -> getDefId(n) + )) var assigned_var = getAssignedVar(parent.asInstanceOf[CallNode]) // Being assigned with another variable if (assigned_var.id != n.id) { - followAssignedVar(conds, assigned_var, analyzed, depth, warnings) + followAssignedVar(conds, assigned_var, analyzed, depth, warnings, methodCache, memberCache) } } // Logical operation @@ -735,8 +798,8 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, val arg_idx = getArgIdx(call, n_cast) // Value being pushed in the array if (arg_idx == 1) { - val array = call.argument.argumentIndex(1).l(0) - followAssignedVar(conds, array, analyzed, depth, warnings) + val array = call.argument.argumentIndex(1).head + followAssignedVar(conds, array, analyzed, depth, warnings, methodCache, memberCache) } } // Part of a conditional (e.g., ternary, elvis) @@ -755,15 +818,19 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, if (arg_idx == 1) 3 else 2 } - val other_arg = call.argument.argumentIndex(other_idx).l(0) + val other_arg = call.argument.argumentIndex(other_idx).head val other_arg_type = getNodeType(other_arg) if (other_arg_type != "ANY" && other_arg_type != "deserialize.") { conds += createCondition("Exact", - mutable.Map("reason" -> "Ternary", "type" -> other_arg_type)) + mutable.Map( + "reason" -> "Ternary", + "type" -> other_arg_type, + "nodeId" -> getDefId(n) + )) if (other_arg_type == "string") { - addHaveToString(conds) + addHaveToString(conds, methodCache, call.astParent.id) } } } @@ -774,16 +841,20 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, // Part of a string operation case string_op @ (".concat" | ".assignmentConcat") => { - addHaveToString(conds) + addHaveToString(conds, methodCache, call.astParent.id) conds += createCondition("Exact", - mutable.Map("type" -> "string", "reason" -> "StringOp")) + mutable.Map( + "type" -> "string", + "reason" -> "StringOp", + "nodeId" -> getDefId(n) + )) if (call.methodFullName == ".assignmentConcat") { var assigned_var = getAssignedVar(parent.asInstanceOf[CallNode]) // Being concatenated to another variable if (assigned_var.id != n.id) { - followAssignedVar(conds, assigned_var, analyzed, depth, warnings) + followAssignedVar(conds, assigned_var, analyzed, depth, warnings, methodCache, memberCache) } } } @@ -794,10 +865,14 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, mutable.Map("type" -> call.typeFullName)) conds += createCondition("Exact", - mutable.Map("type" -> call.typeFullName, "reason" -> "Scalar")) + mutable.Map( + "type" -> call.typeFullName, + "reason" -> "Scalar", + "nodeId" -> getDefId(n) + )) if (call.typeFullName == "string") { - addHaveToString(conds) + addHaveToString(conds, methodCache, call.astParent.id) } } // Used in 'new' as a dynamic class name (new $var()) @@ -810,8 +885,30 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, /* Unserialized value is assigned to a variable, follow it to collect * more conditions */ var assigned_var = getAssignedVar(parent.asInstanceOf[CallNode]) - if (assigned_var.id != n.id) { - followAssignedVar(conds, assigned_var, analyzed, depth, warnings) + // New representation of list() + if (assigned_var.isIdentifier && assigned_var.asInstanceOf[Identifier].name.contains("@tmp-")) { + val temp_identifier = assigned_var.asInstanceOf[Identifier] + val varName = temp_identifier.name + val scopeId = getScopeId(temp_identifier) + + // Find all usages of this temporary variable directly, without relying on a Local declaration. + val all_temp_uses = cpg.identifier + .name(varName) + .filter(i => getScopeId(i) == scopeId) + .l + + // Filter out the current node (the assignment target) and only process subsequent uses. + val subsequent_uses = all_temp_uses + .filter(_.id != temp_identifier.id) + .filter(n => {n.lineNumber.getOrElse(-1) >= temp_identifier.lineNumber.getOrElse(-1)}) + + for (use <- subsequent_uses) { + // By extracting conditions from the USES of the temp var, we will find the + // subsequent assignments (e.g., `$a = @tmp-0[0]`) and trace the real variables. + extractConditions(conds, use, analyzed, depth, warnings, methodCache, memberCache) + } + } else if (assigned_var.id != n.id) { + followAssignedVar(conds, assigned_var, analyzed, depth, warnings, methodCache, memberCache) } else { // Don't follow re-assignments println("Node " + n + " is being reassigned, ignoring") @@ -822,7 +919,7 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, case callbacks @ ("array_map" | "call_user_func_array") => { val fcall = parent.asInstanceOf[CallNode] // FIXME for now we assume callback is the first argument - val callback_name_node = fcall.argument.argumentIndex(1).l(0) + val callback_name_node = fcall.argument.argumentIndex(1).head val arg_idx = getArgIdx(fcall, n_cast) // Treat the callback as a function call if we know its name @@ -833,25 +930,28 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, if (getNodeType(callback_name_literal) == "string") { val callback_name = callback_name_literal.code.replace("\"", "") - val callback_func_l = cpg.method.name(callback_name).l + val callback_func_l = methodCache.getOrElse(callback_name, List.empty) if (callback_func_l.length > 0) { - val callback_func = callback_func_l.l(0).asInstanceOf[Method] + val callback_func = callback_func_l.head.asInstanceOf[Method] // Normally we would have to -1 the arg index, but we would have // add it again because joern indexing, so just keep what we got, // it's correct val callback_arg_l = callback_func.parameter.index(arg_idx).l if (callback_arg_l.length > 0) { - val callback_arg = callback_arg_l.l(0) + val callback_arg = callback_arg_l.head val arg_type = getNodeType(callback_arg) if (helpsWithTyping(arg_type)) { - conds += createCondition("Exact", mutable.Map("reason" -> - "FuncArg", "type" -> arg_type, "function" -> - callback_func.name)) + conds += createCondition("Exact", mutable.Map( + "reason" -> "FuncArg", + "type" -> arg_type, + "function" -> callback_func.name, + "nodeId" -> getDefId(n) + )) if (arg_type contains "string") { - addHaveToString(conds) + addHaveToString(conds, methodCache, call.astParent.id) } } } @@ -861,16 +961,24 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, } else { // We can't resolve what the callback is, we need to allow everything conds += createCondition("Exact", - mutable.Map("reason" -> "DynamicCall", - "type" -> "ANY", "call" -> call.name)) + mutable.Map( + "reason" -> "DynamicCall", + "type" -> "ANY", + "call" -> call.name, + "nodeId" -> getDefId(n) + )) } } // Passed to instanceOf case ".instanceOf" => { - val class_name = call.argument.argumentIndex(2).l(0).asInstanceOf[Identifier].name - conds += createCondition("InstanceOf", - mutable.Map("type" -> class_name)) + val class_name = call.argument.argumentIndex(2).head.asInstanceOf[Identifier].name + conds += createCondition("Duck", + mutable.Map( + "reason" -> "InstanceOf", + "type" -> class_name, + "nodeId" -> getDefId(n) + )) } // The node is used as an argument to a call, record the condition case _ => { @@ -881,7 +989,7 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, val fcall = parent.asInstanceOf[CallNode] val nargs = fcall.argument.l.length // Functions with this name - var methods = cpg.method.name(fcall.name).l + var methods = methodCache.getOrElse(fcall.name, List.empty) val arg_idx = getArgIdx(fcall, n_cast) if (fcall.dispatchType == "STATIC_DISPATCH") { @@ -898,21 +1006,26 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, val param_node = method.parameter.index(arg_idx + 1).l var param_type = "ANY" if (param_node.length > 0) { - val param = param_node.l(0) + val param = param_node.head param_type = getNodeType(param) } // Check if we know the parameter type before collecting evidence from within the function if (helpsWithTyping(param_type)) { conds += createCondition("Exact", - mutable.Map("reason" -> "FuncArg", "type" -> param_type, "function" -> fcall.name)) - - if (param_type contains "string") { - addHaveToString(conds) + mutable.Map( + "reason" -> "FuncArg", + "type" -> param_type, + "function" -> fcall.name, + "nodeId" -> getDefId(n) + )) + + if (param_type.contains("string")) { + addHaveToString(conds, methodCache, call.astParent.id) } } else if (!isBuiltIn(method)) { - collectParameterUsesFromFunc(conds, analyzed, method, nargs, arg_idx, depth + 1, warnings) + collectParameterUsesFromFunc(conds, analyzed, method, nargs, arg_idx, depth + 1, warnings, methodCache, memberCache) } } else { println("More arguments than parameters in function " + method.name + " " + method.astParentFullName) @@ -926,12 +1039,16 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, mutable.Map("methodName" -> fcall.name, "methodFullName" -> fcall.methodFullName)) // FIXME maybe filter based on number of arguments - val methods = cpg.method.name(fcall.name).filter(_.astParentFullName != "") - val types = methods.typeDecl.name.mkString("|") + val methods = methodCache.getOrElse(fcall.name, List.empty).filter(_.astParentFullName != "") + val types = methods.flatMap(_.typeDecl.name).mkString("|") conds += createCondition("Duck", - mutable.Map("reason" -> "HasMethod", - "type" -> types, "method" -> fcall.name)) + mutable.Map( + "reason" -> "HasMethod", + "type" -> types, + "method" -> fcall.name, + "nodeId" -> getDefId(n) + )) } else { // Try to identify the object first and filter, else just try everything @@ -940,14 +1057,17 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, if (fobject.isIdentifier) { val fobj_type = fobject.asInstanceOf[Identifier].typeFullName if (fobj_type != "ANY") { - methods = methods.filter(x => {(x.typeDecl.l.length != 0) && (x.typeDecl.name.l(0) == fobj_type)}) + methods = methods.filter(x => {(x.typeDecl.l.length != 0) && (x.typeDecl.name.head == fobj_type)}) } if (fobj_type.startsWith("$") && fcall.name == "__construct") { // Flows into a dynamic constructor which we can't follow, // we have to allow all classes conds += createCondition("Exact", mutable.Map("reason" -> "DynamicCall", - "type" -> "ANY", "call" -> (fobj_type + "->__construct"))) + "type" -> "ANY", + "call" -> (fobj_type + "->__construct"), + "nodeId" -> getDefId(n) + )) return false } @@ -955,25 +1075,25 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, // Check if it's a field with known type too val call = fobject.asInstanceOf[CallNode] if (call.methodFullName == ".fieldAccess") { - val fobject = call.argument.argumentIndex(1).l(0) + val fobject = call.argument.argumentIndex(1).head val fobj_type = getNodeType(fobject) if (fobject.isIdentifier) { - val field = call.argument.argumentIndex(2).l(0) + val field = call.argument.argumentIndex(2).head if (field.isFieldIdentifier) { val field_ident = field.asInstanceOf[FieldIdentifier] - val field_members_with_name = cpg.member.name(field_ident.canonicalName).l + val field_members_with_name = memberCache.getOrElse(field_ident.canonicalName, List.empty) if (fobj_type != "ANY" && !(fobj_type contains "|")) { // Use 'fullName' here to match namespaces as well - val classes_with_field = field_members_with_name.typeDecl.filter(_.fullName == fobj_type).l + val classes_with_field = field_members_with_name.flatMap(_.typeDecl).filter(_.fullName == fobj_type) if (classes_with_field.length == 1) { - val the_class = classes_with_field.l(0) - val member = the_class.member.name(field_ident.canonicalName).l(0) + val the_class = classes_with_field.head + val member = the_class.member.name(field_ident.canonicalName).head val member_type = getNodeType(member) if (member_type != "ANY") { - methods = methods.filter(x => {(x.typeDecl.l.length != 0) && (x.typeDecl.name.l(0) == member_type)}) + methods = methods.filter(x => {(x.typeDecl.l.length != 0) && (x.typeDecl.name.head == member_type)}) // println("Filtered methods: "+ methods) } @@ -984,7 +1104,7 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, } else if (call.methodFullName == ".indexAccess") { val inferred_types = tryInferSliceType(call) - methods = methods.filter(x => {(x.typeDecl.l.length != 0) && (inferred_types.contains(x.typeDecl.name.l(0)))}) + methods = methods.filter(x => {(x.typeDecl.l.length != 0) && (inferred_types.contains(x.typeDecl.name.head))}) } } // The value is the argument to the method call @@ -996,29 +1116,32 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, // Recursively collect evidence by following parameter use in the // called method for (method <- methods) { - // println("Checking " + method.name + " nargs: " + nargs + " param length: " + method.parameter.l.length) + println("Checking " + method.name + " nargs: " + nargs + " param length: " + method.parameter.l.length) // Only do this if this method has enough parameters if (method.parameter.l.length >= nargs) { - val param = method.parameter.index(arg_idx).l(0) + val param = method.parameter.index(arg_idx).head val param_type = getNodeType(param) // Check if we know the parameter type before collecting evidence from within the method if (helpsWithTyping(param_type)) { conds += createCondition("Exact", - mutable.Map("reason" -> "MethodArg", + mutable.Map( + "reason" -> "MethodArg", "type" -> param_type, - "method" -> fcall.name)) + "method" -> fcall.name, + "nodeId" -> getDefId(n) + )) - if (param_type contains "string") { - addHaveToString(conds) + if (param_type.contains("string")) { + addHaveToString(conds, methodCache, call.astParent.id) } } else if (!isBuiltIn(method)) { // FIXME we should always know the types for built-ins, so this check // might not be needed // argument length - 1 to account for object - collectParameterUsesFromMethod(conds, analyzed, method, nargs, arg_idx, depth + 1, warnings) + collectParameterUsesFromMethod(conds, analyzed, method, nargs, arg_idx, depth + 1, warnings, methodCache, memberCache) } } else { println("More arguments than parameters in method " + method.name + " " + method.typeDecl.name) @@ -1040,30 +1163,46 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, ) // Check if we know the return type before collecting evidence from the call sites - if (return_type != "ANY") { + if (helpsWithTyping(return_type)) { conds += createCondition("Exact", - mutable.Map("reason" -> "Return", "type" -> return_type, "methodName" -> method.name)) + mutable.Map( + "reason" -> "Return", + "type" -> return_type, + "methodName" -> method.name, + "nodeId" -> getDefId(n) + )) } else if (!isBuiltIn(method)) { // Recurse backwards to collect more evidence from the call sites of // the parent method val calls_to_parent_method = cpg.call.name(method.name.replace("\\", "\\\\")) for (call <- calls_to_parent_method) { - extractConditions(conds, call, analyzed, depth + 1, warnings) + extractConditions(conds, call, analyzed, depth + 1, warnings, methodCache, memberCache) } } } else { throw new Exception("Unknown node: " + parent) } - return extractConditions(conds, parent, analyzed, depth, warnings) + return extractConditions(conds, parent, analyzed, depth, warnings, methodCache, memberCache) } -@main def exec(cpgFile: String, outFile: String, kBound: Int = 3, focus_lines: String = "") = { +@main def exec(projectPath: String, outFile: String, kBound: Int = 3, focus_lines: String = "") = { + + val projectName = Paths.get(projectPath).getFileName().toString() + open(projectName) val outFileWarnings = outFile + ".warnings" maxDepth = kBound - importCpg(cpgFile) + + println("Building CPG query caches for performance...") + // Create a map from a method's name to all Method nodes with that name + val methodCache: Map[String, List[Method]] = cpg.method.toList.groupBy(_.name) + // Create a map from a member's name to all Member nodes with that name + val memberCache: Map[String, List[Member]] = cpg.member.toList.groupBy(_.name) + println("Caches built.") + + var project_root = cpg.metaData.l.head.root // Names of deserialization APIs to look for var calls = cpg.call.name("unserialize") ++ cpg.call.name("maybe_unserialize") ++ cpg.call.name("deserialize") @@ -1074,7 +1213,7 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, } // Remove calls in the dependencies - calls = calls.filter(!_.file.name.l(0).startsWith("vendor/")) + calls = calls.filter(!_.file.name.head.startsWith("vendor/")) // We save all the collected evidence in here var collected_conditions = new ListBuffer[String]() @@ -1085,13 +1224,13 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, // Iterate through each deserialization call and collect evidence for (call <- calls) { - println(call.file.name.l(0) + ":" + call.lineNumber.getOrElse(-1)) - var conditions = new ListBuffer[Map[String, String]]() + println(call.file.name.head + ":" + call.lineNumber.getOrElse(-1)) + var conditions = mutable.Set[Map[String, String]]() // Call the main function that implements the type inference algorithm - extractConditions(conditions, call, analyzed_node_ids, 0, warnings) + extractConditions(conditions, call, analyzed_node_ids, 0, warnings, methodCache, memberCache) // Create and store an entry containing the inferred types - val entry = UnserEntry(call.file.name.l(0), call.lineNumber.getOrElse(-1), conditions.toSet) + val entry = UnserEntry(join_paths(project_root, call.file.name.head), call.lineNumber.getOrElse(-1), conditions.toSet) val conds_json: String = write(entry) collected_conditions += conds_json @@ -1103,12 +1242,12 @@ def extractConditions(conds: ListBuffer[Map[String, String]], n: AstNode, } } - ("[" + collected_conditions.mkString(",") + "]") #> outFile + writeFile(outFile, "[" + collected_conditions.mkString(",") + "]") println("Warnings:") for (warn <- warnings.toSet) { println("\t" + warn) } - ("[" + warnings.mkString(",") + "]") #> outFileWarnings + writeFile(outFileWarnings, "[" + warnings.mkString(",") + "]") } diff --git a/tools/enhance.sc b/tools/enhance.sc new file mode 100644 index 0000000..71b9c06 --- /dev/null +++ b/tools/enhance.sc @@ -0,0 +1,5 @@ +@main def exec(cpgFile: String) = { + + importCpg(cpgFile) + +} diff --git a/tools/resolve_includes.sc b/tools/resolve_includes.sc index 634c848..804b3f1 100644 --- a/tools/resolve_includes.sc +++ b/tools/resolve_includes.sc @@ -10,10 +10,12 @@ // [ ] - Optimize when paths are fully resolved import io.shiftleft.codepropertygraph.generated.nodes.{ Call => CallNode } +import io.shiftleft.semanticcpg.language._ import scala.collection.mutable import scala.sys.process._ import java.nio.file.{Path, Paths, Files} import scala.Console.{RED, BLUE, YELLOW, WHITE, RESET} +import scala.util.matching.Regex import upickle.default.* @@ -42,76 +44,30 @@ trait Logger { } } -class RegexPath(path: Path) { - - private var _path: Path = path.normalize(); - - def this(path_str: String) = this(Paths.get(path_str)) - - def +(that: RegexPath) : RegexPath = - RegexPath(this._path.toString() + that._path.toString()) - - def +(that: String) : RegexPath = - RegexPath(this._path.toString() + that) - - def isFullyResolved() : Boolean = { - return !(this._path.toString() contains UNKNOWN_NODE) - } - - def getParent() : RegexPath = { - return RegexPath(this._path.getParent) +def writeFile(path: String, content: String): Unit = { + try { + val parentDir = Paths.get(path).getParent + if (parentDir != null) Files.createDirectories(parentDir) + Files.writeString(Paths.get(path), content) + logger.info(s"Successfully wrote ${content.length} bytes to $path") + } catch { + case e: Exception => + logger.error(s"[!] ERROR: Failed to write to file $path.") + logger.error(s" Reason: ${e.getMessage}") } - - def asPath() : Path = { - return this._path - } - - // Either starts with the given string or the UNKNOWN_NODE - def startsWith(s: String) : Boolean = { - return this._path.toString().startsWith(s) || this._path.toString().startsWith(UNKNOWN_NODE) - } - - def startsWith(s: RegexPath) : Boolean = { - return this._path.toString().startsWith(s.toString()) || this._path.toString().startsWith(UNKNOWN_NODE) - } - - // Either ends with the given string or the UNKNOWN_NODE - def endsWith(s: String) : Boolean = { - return this._path.toString().endsWith(s) || this._path.toString().endsWith(UNKNOWN_NODE) - } - - def endsWith(s: RegexPath) : Boolean = { - return this._path.toString().endsWith(s.toString()) || this._path.toString().endsWith(UNKNOWN_NODE) - } - - override def equals(other: Any) : Boolean = { - // print("Checking equality between " + this + " and " + other) - other match { - case that: RegexPath => { - // Check both directions, since either can contain a regular expression - val equal = this._path.toString() == that._path.toString() || - this._path.toString().r.matches(that._path.toString()) || - that._path.toString().r.matches(this._path.toString()) - return equal - } - case _ => false - } - } - - override def hashCode() : Int = { - this._path.hashCode() - } - - override def toString = this._path.toString() - } -var project_root : RegexPath = RegexPath("") + +var project_root : String = "" var psr4_script : java.nio.file.Path = Paths.get("") var warnings = mutable.ListBuffer[String]() var errors = mutable.ListBuffer[String]() -var unhandled_autoloader_files = List[RegexPath]() -var logger : Logger = new Logger{ logLevel = Info }; +var unhandled_autoloader_files = List[String]() +var logger : Logger = new Logger{ logLevel = Debug }; + +// Caches for performance +val includedFilesCache = mutable.Map[String, List[String]]() +val regexCache = mutable.Map[String, Regex]() val MAGIC_CONSTS : List[String] = List.apply("__DIR__", "__FILE__") val BUILTINS : List[String] = List.apply("dirname") @@ -150,7 +106,7 @@ def try_resolve_const(n: CallNode) : String = { return UNKNOWN_NODE } else if (definitions.length == 1) { logger.info("Found definition for constant " + n.code) - val const_val = get_include_string(definitions.l(0).argument.argumentIndex(2).l(0).asInstanceOf[AstNode]) + val const_val = get_include_string(definitions.head.argument.argumentIndex(2).head.asInstanceOf[AstNode]) return const_val } else { logger.warning("Multiple definitions found for " + n.code) @@ -159,29 +115,30 @@ def try_resolve_const(n: CallNode) : String = { } // Returns the normalized absolute path by prepending the project root as a string -def join_paths(p1: RegexPath, p2: RegexPath) : RegexPath = { - return RegexPath(p1.toString() + "/" + p2.toString()) +def join_paths(p1: String, p2: String) : String = { + return Paths.get(p1, p2).normalize().toString() } + // Check if the provided includes for a file contain a file with an unhandled autoloader -def includes_unhandled_autoloader(includes: mutable.ListBuffer[RegexPath]) : Boolean = { - for (autoload_file <- unhandled_autoloader_files) { - if (includes.filter(_ == autoload_file).l.length > 0) { - return true +def includes_unhandled_autoloader(includes: mutable.ListBuffer[String]) : Boolean = { + // Check if any concrete autoloader file path is matched by any of the include patterns. + unhandled_autoloader_files.exists { autoload_file => + includes.exists { pattern => + val regex = regexCache.getOrElseUpdate(pattern, pattern.r) + regex.matches(autoload_file) } } - return false } // Resolve a magic const def resolve_magic_const(n: CallNode) : String = { logger.debug("Resolving magic const " + n.code) - val filename = RegexPath(n.file.name.l(0)) + val filename = n.file.name.head n.code match { case "__DIR__" => { val file_path = join_paths(project_root, filename) - val dir_path = file_path.getParent() - return dir_path.toString() + return Paths.get(file_path).getParent.toString } case "__FILE__" => { return join_paths(project_root, filename).toString() @@ -196,7 +153,7 @@ def resolve_builtin(n: CallNode) : String = { case "dirname" => { val args = n.argument.l val path = get_include_string(args(0)) - if (!(RegexPath(path).isFullyResolved())) { + if (path.contains(UNKNOWN_NODE)) { return UNKNOWN_NODE } val levels = if (args.length > 1) args(1).asInstanceOf[Int] else 1 @@ -219,9 +176,10 @@ def get_include_string(n: AstNode) : String = { val call = n.asInstanceOf[CallNode] call.methodFullName match { case ".concat" => { - val arg1 = get_include_string(call.argument.argumentIndex(1).l(0).asInstanceOf[AstNode]) - val arg2 = get_include_string(call.argument.argumentIndex(2).l(0).asInstanceOf[AstNode]) - return Paths.get(arg1 + arg2).normalize().toString() + val arg1 = get_include_string(call.argument.argumentIndex(1).head.asInstanceOf[AstNode]) + val arg2 = get_include_string(call.argument.argumentIndex(2).head.asInstanceOf[AstNode]) + // return Paths.get(arg1, arg2).normalize().toString() + return arg1 + arg2 } case ".fieldAccess" => { if (is_magic_const(call)) { @@ -238,15 +196,34 @@ def get_include_string(n: AstNode) : String = { if (is_builtin(call)) { return resolve_builtin(call) } else { - logger.warning("Unknown call " + call.methodFullName + " at " + call.file.name.l(0) + ":" + call.lineNumber.getOrElse(-1)) + logger.warning("Unknown call " + call.methodFullName + " at " + call.file.name.head + ":" + call.lineNumber.getOrElse(-1)) return UNKNOWN_NODE } } } } else if (n.isIdentifier) { - // XXX: Maybe try to resolve its value first if it's in the same scope - logger.debug("Unknown node: " + n) - return UNKNOWN_NODE + // XXX: Maybe try to resolve its value first if it's in the same scope + logger.debug("Unknown node: " + n) + logger.debug("trying to resolve value by finding where it is defined") + val identifier = n.asInstanceOf[Identifier] + /* The argument is a variable identifier. In this case, we find where the + * variable is assigned by searching the containing method for + * assignment statements where the lhs matches the variable name. + */ + val assignments_rhs = identifier + .method + .call.where(_.name(".assignment")) + .where(_.argument.argumentIndex(1).isIdentifier.name(identifier.name)) + .argument.argumentIndex(2) + .l + + assignments_rhs.length match { + case 0 => return UNKNOWN_NODE + case 1 => return get_include_string(assignments_rhs.head.asInstanceOf[AstNode]) + /* This case ignores if there are multiple assignments to the variable in this function. + * TODO: try to take the assignment immediately prior to the variable use. */ + case _ => return get_include_string(assignments_rhs.head.asInstanceOf[AstNode]) + } } else { throw new Exception("Unknown type for include argument: " + n) } @@ -254,8 +231,8 @@ def get_include_string(n: AstNode) : String = { // Returns a string representing the path of included file. In cases where the // path can't fully be resolved, the part of the path that can't be resolved -// is replaced with a wildcard (*) -def get_include_path(including_file: RegexPath, n: AstNode) : RegexPath = { +// is replaced with a wildcard (.*) +def get_include_path(including_file: String, n: AstNode) : String = { logger.debug("Resolving include path for " + n) @@ -265,24 +242,24 @@ def get_include_path(including_file: RegexPath, n: AstNode) : RegexPath = { // else make it absolute first if (incl_string.startsWith("/") || incl_string.startsWith(UNKNOWN_NODE)) { logger.debug("Resolved include string may be absolute: " + incl_string) - return RegexPath(incl_string) + return incl_string } else { logger.debug("Resolved include string is relative: " + incl_string) - val incl_dir = including_file.getParent() - return join_paths(incl_dir, RegexPath(incl_string)) + val incl_dir = Paths.get(including_file).getParent().toString() + return join_paths(incl_dir, incl_string) } } // Gets the autloaded files for a Composer-generated, PSR-4 compliant // autoloader, as described in https://www.php-fig.org/psr/psr-4/ -def get_composer_autoloaded_files(composer_psr4_mappings_path: RegexPath, - files_to_classes: mutable.Map[RegexPath, mutable.ListBuffer[String]]) - : mutable.ListBuffer[RegexPath] = { - var autoloaded_files = mutable.ListBuffer[RegexPath]() +def get_composer_autoloaded_files(composer_psr4_mappings_path: String, + files_to_classes: mutable.Map[String, mutable.ListBuffer[String]]) + : List[String] = { + var autoloaded_files = mutable.ListBuffer[String]() // If there is not autoloader, return an empty set of files - if (!Files.exists(composer_psr4_mappings_path.asPath())) { - return autoloaded_files + if (!Files.exists(Paths.get(composer_psr4_mappings_path))) { + return List() } // Load the namespace-to-path mappings @@ -300,7 +277,7 @@ def get_composer_autoloaded_files(composer_psr4_mappings_path: RegexPath, if (class_fqn.startsWith(namespace)) { val remaining_namespace = class_fqn.substring(namespace.length) val remaining_path = remaining_namespace.replace("\\", "/") + ".php" - val class_file_path = RegexPath(path.str + "/" + remaining_path) + val class_file_path = Paths.get(path.str, remaining_path).normalize().toString() if (class_file_path == file) { autoloaded_files += file } @@ -310,209 +287,214 @@ def get_composer_autoloaded_files(composer_psr4_mappings_path: RegexPath, } } - return autoloaded_files + return autoloaded_files.distinct.toList } // Get files that filename includes // We need to provide a list of all the project files here in order to resolve // wildcards (includes.get would just return a string with a wildcard if we don't // compare it to an actual list of files to force it to use its 'equals' method) -def get_included_files(filename: RegexPath, - includes: mutable.Map[RegexPath, mutable.ListBuffer[RegexPath]], - project_files: List[RegexPath]) : - mutable.ListBuffer[RegexPath] = { - val included = mutable.ListBuffer[RegexPath]() - val to_match = includes.get(filename).getOrElse(mutable.ListBuffer[RegexPath]()) - for (file <- to_match) { - included ++= project_files.filter(x => {x == file}); +def get_included_files(filename: String, + includes: mutable.Map[String, mutable.ListBuffer[String]], + project_files: List[String]) : + List[String] = { + if (includedFilesCache.contains(filename)) { + return includedFilesCache(filename) } - return included + + val included = mutable.ListBuffer[String]() + val patterns_to_match = includes.getOrElse(filename, mutable.ListBuffer[String]()) + for (pattern <- patterns_to_match) { + val patternRegex = pattern.r + included ++= project_files.filter(project_file => + patternRegex.matches(project_file) || project_file.r.matches(pattern) + ) + } + + val result = included.distinct.toList + includedFilesCache.put(filename, result) + result } -// Get files that include filename -def get_files_that_include(filename: RegexPath, - includes: mutable.Map[RegexPath, mutable.ListBuffer[RegexPath]]) : - List[RegexPath] = { - return includes.filter(_._2.contains(filename)).keys.l +def get_files_that_include(filename: String, + includes: mutable.Map[String, mutable.ListBuffer[String]]) : + List[String] = { + includes.filter { case (_, patterns) => + patterns.exists { p => + p.r.matches(filename) || filename.r.matches(p) + } + }.keys.toList } // Add all the files including the target file to the list of files to add, and continue // recursively until we get all the files in the dependency chain -def add_includes_backwards(filename: RegexPath, - includes: mutable.Map[RegexPath, mutable.ListBuffer[RegexPath]], - files_to_add: mutable.ListBuffer[RegexPath]) : Boolean = { +def add_includes_backwards(filename: String, + includes: mutable.Map[String, mutable.ListBuffer[String]], + files_to_add: mutable.Set[String]) : Boolean = { // Use Set for performance - var all_including_files = get_files_that_include(filename, includes) + val all_including_files = get_files_that_include(filename, includes) for (including <- all_including_files) { - // Don't re-add to avoid infinite loops - if (!files_to_add.contains(including)) { - files_to_add += including + // Don't re-add to avoid infinite loops. Set `add` returns false if item already exists. + if (files_to_add.add(including)) { + logger.info("Backwards add: " + including) add_includes_backwards(including, includes, files_to_add) } } - true } // Creates the list of available classes at each deserialization call def resolve_avail_classes( - project_files: List[RegexPath], - included_files: mutable.Map[RegexPath, mutable.ListBuffer[RegexPath]], - files_to_classes: mutable.Map[RegexPath, mutable.ListBuffer[String]], + project_files: List[String], + included_files: mutable.Map[String, mutable.ListBuffer[String]], + files_to_classes: mutable.Map[String, mutable.ListBuffer[String]], + focus_lines: String = "" ) : List[AvailClassesEntry] = { - logger.debug("Resolving available classes") + logger.info("Resolving available classes") - var unser_calls_all = cpg.call.name("unserialize") ++ cpg.call.name("maybe_unserialize") ++ cpg.call.name("deserialize") ++ cpg.call.name("dunserialize") - // Group calls by filename - // XXX: might have to change this to account for mid-file includes - var unser_calls_grouped = unser_calls_all.groupBy(_.file.name.l(0)) + var unser_calls_traversal = cpg.call.name("unserialize") ++ cpg.call.name("maybe_unserialize") ++ cpg.call.name("deserialize") ++ cpg.call.name("dunserialize") + if (focus_lines != "") { + val focus_entries = focus_lines.split(",") + unser_calls_traversal = unser_calls_traversal.filter(x => + focus_entries.contains(x.method.filename + ":" + x.lineNumber.getOrElse(-1).toString)) + } + // Group calls by filename. Materialize here as groupBy needs a collection. + val unser_calls_grouped = unser_calls_traversal.l.groupBy(_.file.name.head) var avail_classes_entries = mutable.ListBuffer[AvailClassesEntry]() for ((filename, unser_calls) <- unser_calls_grouped) { - val full_filename = join_paths(project_root, RegexPath(filename)) - // All the files that are on a dependency path passing from filename - var files_to_add = mutable.ListBuffer[RegexPath]() - files_to_add += full_filename + val full_filename = join_paths(project_root, filename) + // Use a Set for files_to_add for performance + val files_to_add = mutable.Set[String](full_filename) - logger.debug("Adding includes backwards for " + full_filename) + logger.info("Adding includes backwards for " + full_filename) add_includes_backwards(full_filename, included_files, files_to_add) - var avail_classes = mutable.ListBuffer[String]() - var checked_files = mutable.ListBuffer[RegexPath]() - - while (!files_to_add.isEmpty) { - - - // Get all the files that match the included file name (might be more than - // one cause filename might contain wildwards) - val incl_filename = files_to_add.remove(0) + var avail_classes = mutable.Set[String]() // Use Set to handle duplicates efficiently + var checked_files = mutable.Set[String]() + val queue = mutable.Queue[String](files_to_add.toSeq: _*) - logger.debug("Adding classes from " + incl_filename) + while (queue.nonEmpty) { + val incl_filename = queue.dequeue() - val incl_classes = mutable.ListBuffer[String]() - for ((filename, classes) <- files_to_classes) { - if (filename == incl_filename) { - incl_classes ++= classes - } - } + if (checked_files.add(incl_filename)) { + logger.info("Adding classes from " + incl_filename) + avail_classes ++= files_to_classes.getOrElse(incl_filename, mutable.ListBuffer.empty) - avail_classes ++= incl_classes - checked_files += incl_filename - - // Add all the files included by the current file to the list of files to add - for (included <- get_included_files(incl_filename, included_files, project_files)) { - if (!checked_files.contains(included) && !files_to_add.contains(included)) { - logger.debug("Adding " + included + " to files to check") - files_to_add += included + // Add all the files included by the current file to the list of files to add + for (included <- get_included_files(incl_filename, included_files, project_files)) { + if (!checked_files.contains(included) && !queue.contains(included)) { + logger.info("Adding " + included + " to files to check") + queue.enqueue(included) + } } } } - avail_classes_entries += AvailClassesEntry(full_filename.toString(), unser_calls.lineNumber.l, avail_classes.toList) - + avail_classes_entries += AvailClassesEntry(full_filename, unser_calls.map(_.lineNumber.getOrElse(-1)), avail_classes.toList) } - return avail_classes_entries.toList - + avail_classes_entries.toList } -@main def exec(cpgFile: String, outFile: String, psr4Script: String) = { +@main def exec(projectPath: String, outFile: String, psr4Script: String, focus_lines: String = "") = { - importCpg(cpgFile) + val projectName = Paths.get(projectPath).getFileName().toString() + open(projectName) val outFileWarnings = outFile + ".warnings" val outFileErrors = outFile + ".errors" psr4_script = Paths.get(psr4Script) // Root directory of analyzed project - project_root = RegexPath(cpg.metaData.l(0).root) - // All include directives in the project - val include_directives = (cpg.call.methodFullName("include") ++ cpg.call.methodFullName("include_once") ++ cpg.call.methodFullName("require") ++ cpg.call.methodFullName("require_once")).l + project_root = cpg.metaData.l.head.root + // Keep queries as Traversals to materialize as late as possible + val include_directives_traversal = (cpg.call.methodFullName("include") ++ cpg.call.methodFullName("include_once") ++ cpg.call.methodFullName("require") ++ cpg.call.methodFullName("require_once")) + // val all_classes_traversal = cpg.typeDecl.filter(_.code.startsWith("class ")).filter(_.code != "class ") + val all_classes_traversal = cpg.typeDecl.filterNot(_.name == "").filterNot(_.fullName.endsWith("")) - val project_files = cpg.file.l.filter(_.name != "").map(x => {join_paths(project_root, RegexPath(x.name))}) + val project_files = cpg.file.l.filter(_.name != "").map(x => join_paths(project_root, x.name)) // Check if the project uses Composer. If it does, figure out its dependency // directory (usually just vendor/) - var vendor_dir = join_paths(project_root, RegexPath("vendor/")) - val composer_json_path = join_paths(project_root, RegexPath("composer.json")) - if (Files.exists(composer_json_path.asPath())) { - val composer_contents = os.read(os.Path(composer_json_path.toString())) + var vendor_dir = join_paths(project_root, "vendor/") + val composer_json_path = join_paths(project_root, "composer.json") + if (Files.exists(Paths.get(composer_json_path))) { + val composer_contents = os.read(os.Path(composer_json_path)) val data = ujson.read(composer_contents) if (data.obj.get("config").nonEmpty) { val config = data("config") if (config.obj.get("vendor-dir").nonEmpty) { - vendor_dir = RegexPath(config("vendor-dir").str) + vendor_dir = config("vendor-dir").str } } } - val composer_vendor_dir = join_paths(vendor_dir, RegexPath("composer")) - val autoload_file_path = join_paths(vendor_dir, RegexPath("autoload.php")) - val composer_psr4_mappings_path = join_paths(composer_vendor_dir, RegexPath("autoload_psr4.php")) + val composer_vendor_dir = join_paths(vendor_dir, "composer") + val autoload_file_path = join_paths(vendor_dir, "autoload.php") + val composer_psr4_mappings_path = join_paths(composer_vendor_dir, "autoload_psr4.php") // Map from filename to its included files - var included_files_map = mutable.Map[RegexPath, mutable.ListBuffer[RegexPath]]() + val included_files_map = mutable.Map[String, mutable.ListBuffer[String]]() // Map from filename to the classes it defines - var files_to_classes_map = mutable.Map[RegexPath, mutable.ListBuffer[String]]() - // All classes declared in project - // FIXME: make sure this is the right way to filter the classes - val all_classes = cpg.typeDecl.filter(_.code.startsWith("class ")).filter(_.code != "class ").l + val files_to_classes_map = mutable.Map[String, mutable.ListBuffer[String]]() + + // Materialize all_classes here as we need to group them by filename + val all_classes = all_classes_traversal.l // Check if there are any autoloaders registered other than the composer one val autoload_registrations = cpg.call.methodFullName("spl_autoload_register").l - val non_composer_autoloaders = autoload_registrations.filter(x => {!join_paths(project_root, RegexPath(x.method.filename)).startsWith(composer_vendor_dir)}).l - unhandled_autoloader_files = non_composer_autoloaders.map(x => {RegexPath(x.file.name.l(0))}) + val non_composer_autoloaders = autoload_registrations.filter(x => !join_paths(project_root, x.method.filename).startsWith(composer_vendor_dir)) + unhandled_autoloader_files = non_composer_autoloaders.map(x => join_paths(project_root, x.file.name.l.head)) - if (non_composer_autoloaders.length > 0) { + if (non_composer_autoloaders.nonEmpty) { for (autoloader <- non_composer_autoloaders) { - logger.error("Unhandled autoloader registered at " + autoloader.file.name.l(0)) + logger.error("Unhandled autoloader registered at " + autoloader.file.name.l.head) } } // Create the file to class map by matching each class definition with its containing filename - for (file <- cpg.file) { + for (file <- cpg.file.l) { val filename = file.name - if (filename != "") { - val full_path = join_paths(project_root, RegexPath(filename)) - var contained_classes = all_classes.filter(_.filename == filename) - files_to_classes_map.getOrElseUpdate(full_path, mutable.ListBuffer[String]()) ++= contained_classes.fullName.l + val full_path = join_paths(project_root, filename) + val contained_classes = all_classes.filter(_.filename == filename) + if (contained_classes.nonEmpty) { + files_to_classes_map.getOrElseUpdate(full_path, mutable.ListBuffer[String]()) ++= contained_classes.map(_.fullName) + } } - } - for (include_directive <- include_directives) { - - // println("Include directive: " + include_directive) - - val including_filename = join_paths(project_root, RegexPath(include_directive.file.name.l(0))) - // println("Including filename: " + including_filename) + // Iterate on the traversal directly + for (include_directive <- include_directives_traversal) { + val including_filename = join_paths(project_root, include_directive.file.name.l.head) val line = include_directive.lineNumber.getOrElse(-1) logger.info("Analyzing include directive at " + including_filename + ":" + line) - if (include_directive.argument.l.length > 1) { + if (include_directive.argument.size > 1) { throw new Exception("More than one arguments in include directive: " + including_filename + ":" + line) } - val included_arg = include_directive.argument.l(0) + val included_arg = include_directive.argument.head val included_path = get_include_path(including_filename, included_arg) - logger.debug("Resolved include path for " + including_filename + ":" + line + ": " + included_path) + logger.info("Resolved include path for " + including_filename + ":" + line + ": " + included_path) if (included_path.endsWith(".php")) { - included_files_map.getOrElseUpdate(including_filename, mutable.ListBuffer[RegexPath]()) += included_path + included_files_map.getOrElseUpdate(including_filename, mutable.ListBuffer[String]()) += included_path } } + logger.info("Moving on") // If the project has a Composer-generated autoloader, add the autoloaded // files in the results as well - if (Files.exists(autoload_file_path.asPath())) { + if (Files.exists(Paths.get(autoload_file_path))) { val composer_autoloaded_files = get_composer_autoloaded_files(composer_psr4_mappings_path, files_to_classes_map) // Check which files include the autoload.php file, and add the autoloaded classes // in their list of includes @@ -526,24 +508,27 @@ def resolve_avail_classes( // Note: this has to be done last, to make sure we resolved all other includes first if (includes_unhandled_autoloader(includes)){ logger.debug(filename.toString() + " includes unhandled autoloader") - included_files_map.update(filename, mutable.ListBuffer(files_to_classes_map.keys.l: _*)) + included_files_map.update(filename, mutable.ListBuffer(files_to_classes_map.keys.toList: _*)) } } } - files_to_classes_map.mkString("\n") #> (outFile + ".files_to_classes") - included_files_map.mkString("\n") #> (outFile + ".included_files") + logger.info("Finalizing") + + writeFile(outFile + ".files_to_classes", files_to_classes_map.mkString("\n")) + writeFile(outFile + ".included_files", included_files_map.mkString("\n")) - val avail_classes = resolve_avail_classes(project_files, included_files_map, files_to_classes_map) + val avail_classes = resolve_avail_classes(project_files, included_files_map, files_to_classes_map, focus_lines) val avail_classes_json: String = write(avail_classes) - avail_classes_json #> outFile - println(avail_classes_json) + writeFile(outFile, avail_classes_json) + // println(avail_classes_json) + + writeFile(outFileWarnings, "[" + warnings.mkString(",") + "]") + writeFile(outFileErrors, "[" + errors.mkString(",") + "]") - ("[" + warnings.mkString(",") + "]") #> outFileWarnings - ("[" + errors.mkString(",") + "]") #> outFileErrors // println(warnings) - if (errors.length > 0) { + if (errors.nonEmpty) { println("Analysis finished with the following errors: ") println(errors) } else {