From fcefe35847a582ffd37a6efb24e6c34c057b36b7 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Mon, 22 Apr 2024 16:22:06 -0700 Subject: [PATCH 01/38] Add `exclusives` and `ignored_mods` to filter out irrelevant modules --- pycg/processing/postprocessor.py | 6 ++++++ pycg/processing/preprocessor.py | 14 ++++++++++++++ pycg/pycg.py | 8 +++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index f114e43..82ba2aa 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -30,6 +30,8 @@ def __init__( self, input_file, modname, + exclusives, + ignored_mods, import_manager, scope_manager, def_manager, @@ -38,6 +40,8 @@ def __init__( modules_analyzed=None, ): super().__init__(input_file, modname, modules_analyzed) + self.exclusives = exclusives + self.ignored_mods = ignored_mods self.import_manager = import_manager self.scope_manager = scope_manager self.def_manager = def_manager @@ -329,6 +333,8 @@ def update_parent_classes(self, defi): def analyze_submodules(self): super().analyze_submodules( PostProcessor, + self.exclusives, + self.ignored_mods, self.import_manager, self.scope_manager, self.def_manager, diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 5d5c7db..3be3e89 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -30,6 +30,8 @@ def __init__( self, filename, modname, + exclusives, + ignored_mods, import_manager, scope_manager, def_manager, @@ -40,6 +42,8 @@ def __init__( super().__init__(filename, modname, modules_analyzed) self.modname = modname + self.exclusives = exclusives + self.ignored_mods = ignored_mods self.mod_dir = "/".join(self.filename.split("/")[:-1]) self.import_manager = import_manager @@ -71,6 +75,8 @@ def analyze_submodule(self, modname): super().analyze_submodule( PreProcessor, modname, + self.exclusives, + self.ignored_mods, self.import_manager, self.scope_manager, self.def_manager, @@ -204,6 +210,14 @@ def add_external_def(name, target): src_name = handle_src_name(import_item.name) tgt_name = import_item.asname if import_item.asname else import_item.name imported_name = self.import_manager.handle_import(src_name, level) + + # Limit to exclusive module + if src_name.split(".")[0] not in self.exclusives: + continue + + # Skip ignored modules + if tgt_name in self.ignored_mods: + continue if not imported_name: add_external_def(src_name, tgt_name) diff --git a/pycg/pycg.py b/pycg/pycg.py index 2f77017..2e8f657 100644 --- a/pycg/pycg.py +++ b/pycg/pycg.py @@ -35,9 +35,11 @@ class CallGraphGenerator(object): - def __init__(self, entry_points, package, max_iter, operation): + def __init__(self, entry_points, package, exclusives, ignored_mods, max_iter, operation): self.entry_points = entry_points self.package = package + self.exclusives = exclusives + self.ignored_mods = ignored_mods self.state = None self.max_iter = max_iter self.operation = operation @@ -162,6 +164,8 @@ def analyze(self): self.do_pass( PreProcessor, True, + self.exclusives, + self.ignored_mods, self.import_manager, self.scope_manager, self.def_manager, @@ -179,6 +183,8 @@ def analyze(self): self.do_pass( PostProcessor, False, + self.exclusives, + self.ignored_mods, self.import_manager, self.scope_manager, self.def_manager, From bc8ec6c147589662cdb792619e717a8b64cdfd19 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Mon, 22 Apr 2024 22:32:45 -0700 Subject: [PATCH 02/38] Linting --- pycg/processing/preprocessor.py | 6 +++--- pycg/pycg.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 3be3e89..56c6490 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -210,12 +210,12 @@ def add_external_def(name, target): src_name = handle_src_name(import_item.name) tgt_name = import_item.asname if import_item.asname else import_item.name imported_name = self.import_manager.handle_import(src_name, level) - - # Limit to exclusive module + + # Limit to exclusive module if src_name.split(".")[0] not in self.exclusives: continue - # Skip ignored modules + # Skip ignored modules if tgt_name in self.ignored_mods: continue diff --git a/pycg/pycg.py b/pycg/pycg.py index 2e8f657..bc673b9 100644 --- a/pycg/pycg.py +++ b/pycg/pycg.py @@ -35,7 +35,9 @@ class CallGraphGenerator(object): - def __init__(self, entry_points, package, exclusives, ignored_mods, max_iter, operation): + def __init__( + self, entry_points, package, exclusives, ignored_mods, max_iter, operation + ): self.entry_points = entry_points self.package = package self.exclusives = exclusives From 4484b9694bdfe0d94e7c78779e14489020e5b1cf Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 00:23:10 -0700 Subject: [PATCH 03/38] Return even earlier --- pycg/processing/preprocessor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 56c6490..88ad98e 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -206,15 +206,15 @@ def add_external_def(name, target): tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) + # Limit to exclusive module + if node.module not in self.exclusives: + return + for import_item in node.names: src_name = handle_src_name(import_item.name) tgt_name = import_item.asname if import_item.asname else import_item.name imported_name = self.import_manager.handle_import(src_name, level) - # Limit to exclusive module - if src_name.split(".")[0] not in self.exclusives: - continue - # Skip ignored modules if tgt_name in self.ignored_mods: continue From 332ed7e3174ef365d0c43ca65bd97ea4d26f3b21 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 00:29:45 -0700 Subject: [PATCH 04/38] Hm --- pycg/processing/preprocessor.py | 53 ++++++++++++++++----------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 88ad98e..ecc7fe9 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -207,33 +207,32 @@ def add_external_def(name, target): scope.add_def(target, tgt_defi) # Limit to exclusive module - if node.module not in self.exclusives: - return - - for import_item in node.names: - src_name = handle_src_name(import_item.name) - tgt_name = import_item.asname if import_item.asname else import_item.name - imported_name = self.import_manager.handle_import(src_name, level) - - # Skip ignored modules - if tgt_name in self.ignored_mods: - continue - - if not imported_name: - add_external_def(src_name, tgt_name) - continue - - fname = self.import_manager.get_filepath(imported_name) - if not fname: - add_external_def(src_name, tgt_name) - continue - # only analyze modules under the current directory - if self.import_manager.get_mod_dir() in fname: - if imported_name not in self.modules_analyzed: - self.analyze_submodule(imported_name) - handle_scopes(import_item.name, tgt_name, imported_name) - else: - add_external_def(src_name, tgt_name) + print() + if node.module in self.exclusives: + for import_item in node.names: + src_name = handle_src_name(import_item.name) + tgt_name = import_item.asname if import_item.asname else import_item.name + imported_name = self.import_manager.handle_import(src_name, level) + + # Skip ignored modules + if tgt_name in self.ignored_mods: + continue + + if not imported_name: + add_external_def(src_name, tgt_name) + continue + + fname = self.import_manager.get_filepath(imported_name) + if not fname: + add_external_def(src_name, tgt_name) + continue + # only analyze modules under the current directory + if self.import_manager.get_mod_dir() in fname: + if imported_name not in self.modules_analyzed: + self.analyze_submodule(imported_name) + handle_scopes(import_item.name, tgt_name, imported_name) + else: + add_external_def(src_name, tgt_name) # handle all modules that were not analyzed for modname in self.import_manager.get_imports(self.modname): From 91450797e464f18ad39966b72c7afe4b153abfc6 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 00:36:08 -0700 Subject: [PATCH 05/38] Hmm --- pycg/processing/preprocessor.py | 57 +++++++++++++++++---------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index ecc7fe9..76d99ba 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -206,36 +206,39 @@ def add_external_def(name, target): tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) - # Limit to exclusive module - print() - if node.module in self.exclusives: - for import_item in node.names: - src_name = handle_src_name(import_item.name) - tgt_name = import_item.asname if import_item.asname else import_item.name - imported_name = self.import_manager.handle_import(src_name, level) - - # Skip ignored modules - if tgt_name in self.ignored_mods: - continue - - if not imported_name: - add_external_def(src_name, tgt_name) - continue - - fname = self.import_manager.get_filepath(imported_name) - if not fname: - add_external_def(src_name, tgt_name) - continue - # only analyze modules under the current directory - if self.import_manager.get_mod_dir() in fname: - if imported_name not in self.modules_analyzed: - self.analyze_submodule(imported_name) - handle_scopes(import_item.name, tgt_name, imported_name) - else: - add_external_def(src_name, tgt_name) + print('node', node) + for import_item in node.names: + src_name = handle_src_name(import_item.name) + tgt_name = import_item.asname if import_item.asname else import_item.name + imported_name = self.import_manager.handle_import(src_name, level) + + # Limit to exclusive module + if src_name.split(".")[0] not in self.exclusives: + continue + + # Skip ignored modules + if tgt_name in self.ignored_mods: + continue + + if not imported_name: + add_external_def(src_name, tgt_name) + continue + + fname = self.import_manager.get_filepath(imported_name) + if not fname: + add_external_def(src_name, tgt_name) + continue + # only analyze modules under the current directory + if self.import_manager.get_mod_dir() in fname: + if imported_name not in self.modules_analyzed: + self.analyze_submodule(imported_name) + handle_scopes(import_item.name, tgt_name, imported_name) + else: + add_external_def(src_name, tgt_name) # handle all modules that were not analyzed for modname in self.import_manager.get_imports(self.modname): + print('mod', mod) fname = self.import_manager.get_filepath(modname) if not fname: From 3fbb6172d4bdd225715fb11c1d9ec87ae537b060 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 00:39:11 -0700 Subject: [PATCH 06/38] Fix --- pycg/processing/preprocessor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 76d99ba..a77023b 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -237,8 +237,9 @@ def add_external_def(name, target): add_external_def(src_name, tgt_name) # handle all modules that were not analyzed + print('mod full', self.modname) for modname in self.import_manager.get_imports(self.modname): - print('mod', mod) + print('mod', modname) fname = self.import_manager.get_filepath(modname) if not fname: From 09ace52af23ef840560e73409138730a8cfab9cc Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 00:42:59 -0700 Subject: [PATCH 07/38] Hm --- pycg/processing/preprocessor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index a77023b..c473317 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -237,8 +237,10 @@ def add_external_def(name, target): add_external_def(src_name, tgt_name) # handle all modules that were not analyzed - print('mod full', self.modname) for modname in self.import_manager.get_imports(self.modname): + if modname.split(".")[0] not in self.exclusives: + continue + print('mod', modname) fname = self.import_manager.get_filepath(modname) From 8f2107b7506c1c1b933e2cb99790fbd8ecb9ac8e Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 00:45:14 -0700 Subject: [PATCH 08/38] Remove print --- pycg/processing/preprocessor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index c473317..d630e24 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -206,7 +206,6 @@ def add_external_def(name, target): tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) - print('node', node) for import_item in node.names: src_name = handle_src_name(import_item.name) tgt_name = import_item.asname if import_item.asname else import_item.name @@ -240,8 +239,7 @@ def add_external_def(name, target): for modname in self.import_manager.get_imports(self.modname): if modname.split(".")[0] not in self.exclusives: continue - - print('mod', modname) + fname = self.import_manager.get_filepath(modname) if not fname: From 227325d634c4938d76a0572ef0ccf6f9aa1cd914 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 01:07:04 -0700 Subject: [PATCH 09/38] Log --- pycg/processing/postprocessor.py | 2 ++ pycg/processing/preprocessor.py | 1 + 2 files changed, 3 insertions(+) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index 82ba2aa..3da2302 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -187,7 +187,9 @@ def visit_ClassDef(self, node): cls = self.class_manager.create(cls_def.get_ns(), self.modname) cls.clear_mro() + print('base node', node) for base in node.bases: + print('base', base) # all bases are of the type ast.Name self.visit(base) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index d630e24..269418f 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -113,6 +113,7 @@ def iterate_mod_items(items, const): items = self.scope_manager.handle_module( self.modname, self.filename, self.contents ) + print('pre items', items) root_sc = self.scope_manager.get_scope(self.modname) root_defi = self.def_manager.get(self.modname) From 46681514b5185c1fe7cab0b0f3cc58a3a5cc6809 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 01:20:54 -0700 Subject: [PATCH 10/38] Hm --- pycg/processing/postprocessor.py | 2 +- pycg/processing/preprocessor.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index 3da2302..ee1f9f2 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -189,11 +189,11 @@ def visit_ClassDef(self, node): cls.clear_mro() print('base node', node) for base in node.bases: - print('base', base) # all bases are of the type ast.Name self.visit(base) bases = self.decode_node(base) + print('bases', bases) for base_def in bases: if not isinstance(base_def, Definition): continue diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 269418f..c7c0f31 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -113,6 +113,10 @@ def iterate_mod_items(items, const): items = self.scope_manager.handle_module( self.modname, self.filename, self.contents ) + items = [ + i for i in items if not \ + any(ignored_mod in i for ignored_mod in ignored_mods_set) + ] print('pre items', items) root_sc = self.scope_manager.get_scope(self.modname) From 0c7f1ec184b34ee00bbccd9b8325976185ad7dd6 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 01:22:48 -0700 Subject: [PATCH 11/38] Fix --- pycg/processing/preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index c7c0f31..3b59973 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -115,7 +115,7 @@ def iterate_mod_items(items, const): ) items = [ i for i in items if not \ - any(ignored_mod in i for ignored_mod in ignored_mods_set) + any(ignored_mod in i for ignored_mod in self.ignored_mods) ] print('pre items', items) From 165a26d38e7c6bc88443063eff5cfc2967845845 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 01:24:30 -0700 Subject: [PATCH 12/38] Fix --- pycg/processing/preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 3b59973..9a3540c 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -113,11 +113,11 @@ def iterate_mod_items(items, const): items = self.scope_manager.handle_module( self.modname, self.filename, self.contents ) + print('pre items', items) items = [ i for i in items if not \ any(ignored_mod in i for ignored_mod in self.ignored_mods) ] - print('pre items', items) root_sc = self.scope_manager.get_scope(self.modname) root_defi = self.def_manager.get(self.modname) From 9d70e92e0488471375bab4a891161e8c058da53c Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 01:25:44 -0700 Subject: [PATCH 13/38] Fix --- pycg/processing/preprocessor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 9a3540c..ecd8469 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -87,6 +87,11 @@ def analyze_submodule(self, modname): def visit_Module(self, node): def iterate_mod_items(items, const): + items = [ + i for i in items if not \ + any(ignored_mod in i for ignored_mod in self.ignored_mods) + ] + for item in items: defi = self.def_manager.get(item) if not defi: @@ -113,11 +118,6 @@ def iterate_mod_items(items, const): items = self.scope_manager.handle_module( self.modname, self.filename, self.contents ) - print('pre items', items) - items = [ - i for i in items if not \ - any(ignored_mod in i for ignored_mod in self.ignored_mods) - ] root_sc = self.scope_manager.get_scope(self.modname) root_defi = self.def_manager.get(self.modname) From 0f498e2a189580a182b2843ce2b8aaf0accba7a7 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 01:26:50 -0700 Subject: [PATCH 14/38] Test --- pycg/processing/preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index ecd8469..39fd754 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -91,7 +91,7 @@ def iterate_mod_items(items, const): i for i in items if not \ any(ignored_mod in i for ignored_mod in self.ignored_mods) ] - + print('items',items) for item in items: defi = self.def_manager.get(item) if not defi: From d150cb55a7f95ecdc8bf946a92ef71e3d156ddba Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Wed, 24 Apr 2024 01:33:07 -0700 Subject: [PATCH 15/38] Remove --- pycg/processing/postprocessor.py | 2 -- pycg/processing/preprocessor.py | 1 - 2 files changed, 3 deletions(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index ee1f9f2..82ba2aa 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -187,13 +187,11 @@ def visit_ClassDef(self, node): cls = self.class_manager.create(cls_def.get_ns(), self.modname) cls.clear_mro() - print('base node', node) for base in node.bases: # all bases are of the type ast.Name self.visit(base) bases = self.decode_node(base) - print('bases', bases) for base_def in bases: if not isinstance(base_def, Definition): continue diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 39fd754..b41dac6 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -91,7 +91,6 @@ def iterate_mod_items(items, const): i for i in items if not \ any(ignored_mod in i for ignored_mod in self.ignored_mods) ] - print('items',items) for item in items: defi = self.def_manager.get(item) if not defi: From 712ebb17fceaa5fa0639bb28d3ed03c2954dbd0e Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 11:41:25 -0700 Subject: [PATCH 16/38] Add default --- pycg/__main__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pycg/__main__.py b/pycg/__main__.py index fb254c3..8de8f58 100644 --- a/pycg/__main__.py +++ b/pycg/__main__.py @@ -12,6 +12,10 @@ def main(): parser.add_argument( "--package", help="Package containing the code to be analyzed", default=None ) + parser.add_argument( + "--exclusives", help="Exclusive package to analyze", default=[]) + parser.add_argument( + "--ignored-mods", help="Modules to ignore during traversal", default=[]) parser.add_argument( "--fasten", help="Produce call graph using the FASTEN format", @@ -57,7 +61,9 @@ def main(): args = parser.parse_args() cg = CallGraphGenerator( - args.entry_point, args.package, args.max_iter, args.operation + args.entry_point, args.package, + args.exclusives, args.ignored_mods, + args.max_iter, args.operation ) cg.analyze() From 0f25a10a942d4a4d7264105c6af5f3d2c1f2a8ea Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 15:46:43 -0700 Subject: [PATCH 17/38] Print --- pycg/pycg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/pycg.py b/pycg/pycg.py index bc673b9..9c9be70 100644 --- a/pycg/pycg.py +++ b/pycg/pycg.py @@ -141,7 +141,7 @@ def do_pass(self, cls, install_hooks=False, *args, **kwargs): if not input_pkg: input_pkg = os.path.dirname(input_file) - + print(input_pkg) if input_mod not in modules_analyzed: if install_hooks: self.import_manager.set_pkg(input_pkg) From cdf8c30323ba13eefdc0b4094b5962d2ede8a310 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 15:50:58 -0700 Subject: [PATCH 18/38] Edit --- pycg/processing/preprocessor.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index b41dac6..79f33c5 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -87,6 +87,10 @@ def analyze_submodule(self, modname): def visit_Module(self, node): def iterate_mod_items(items, const): + print('iter items', items = [ + i for i in items if \ + any(ignored_mod in i for ignored_mod in self.ignored_mods) + ]) items = [ i for i in items if not \ any(ignored_mod in i for ignored_mod in self.ignored_mods) @@ -117,6 +121,7 @@ def iterate_mod_items(items, const): items = self.scope_manager.handle_module( self.modname, self.filename, self.contents ) + print('items', items) root_sc = self.scope_manager.get_scope(self.modname) root_defi = self.def_manager.get(self.modname) From a478da3fb1920287f0c858c496fa46976bb9cb4d Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 15:51:29 -0700 Subject: [PATCH 19/38] Fix --- pycg/processing/preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 79f33c5..d07509b 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -87,7 +87,7 @@ def analyze_submodule(self, modname): def visit_Module(self, node): def iterate_mod_items(items, const): - print('iter items', items = [ + print('iter items', [ i for i in items if \ any(ignored_mod in i for ignored_mod in self.ignored_mods) ]) From 7f944eb8976bec44dad26329e89f12a87f7fd902 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 16:28:45 -0700 Subject: [PATCH 20/38] Edit --- pycg/processing/preprocessor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index d07509b..d38f82c 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -105,7 +105,9 @@ def iterate_mod_items(items, const): parentns = ".".join(splitted[:-1]) self.scope_manager.get_scope(parentns).add_def(name, defi) + print('modname', self.modname, self.filename) self.import_manager.set_current_mod(self.modname, self.filename) + print('modname', self.modname, self.filename) mod = self.module_manager.create(self.modname, self.filename) From 7270fe7900f37537f394e38b89f3262a6313d2a9 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 16:35:18 -0700 Subject: [PATCH 21/38] Edit --- pycg/processing/preprocessor.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index d38f82c..59fa4ad 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -87,14 +87,10 @@ def analyze_submodule(self, modname): def visit_Module(self, node): def iterate_mod_items(items, const): - print('iter items', [ + print('bad iter items', [ i for i in items if \ any(ignored_mod in i for ignored_mod in self.ignored_mods) ]) - items = [ - i for i in items if not \ - any(ignored_mod in i for ignored_mod in self.ignored_mods) - ] for item in items: defi = self.def_manager.get(item) if not defi: @@ -105,9 +101,12 @@ def iterate_mod_items(items, const): parentns = ".".join(splitted[:-1]) self.scope_manager.get_scope(parentns).add_def(name, defi) + if "test_" in self.filename: + print('skip', self.filename) + return + print('modname', self.modname, self.filename) self.import_manager.set_current_mod(self.modname, self.filename) - print('modname', self.modname, self.filename) mod = self.module_manager.create(self.modname, self.filename) From 948250616868381d2c78d879a067ec16fd4cdf16 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 16:37:59 -0700 Subject: [PATCH 22/38] Edit --- pycg/pycg.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pycg/pycg.py b/pycg/pycg.py index 9c9be70..b691457 100644 --- a/pycg/pycg.py +++ b/pycg/pycg.py @@ -139,9 +139,14 @@ def do_pass(self, cls, install_hooks=False, *args, **kwargs): if not input_mod: continue + print(input_pkg, input_file) + + if "test_" in input_file: + print('skip!') + continue + if not input_pkg: input_pkg = os.path.dirname(input_file) - print(input_pkg) if input_mod not in modules_analyzed: if install_hooks: self.import_manager.set_pkg(input_pkg) From 3c36d6f6ac320c532e3fb5039d8b7db4f97a7fae Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 16:48:09 -0700 Subject: [PATCH 23/38] Edit --- pycg/processing/preprocessor.py | 12 +++++------- pycg/pycg.py | 6 ------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 59fa4ad..734d332 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -87,10 +87,13 @@ def analyze_submodule(self, modname): def visit_Module(self, node): def iterate_mod_items(items, const): - print('bad iter items', [ + bad_items = [ i for i in items if \ any(ignored_mod in i for ignored_mod in self.ignored_mods) - ]) + ] + if len(bad_items) > 0: + print('bad iter items', bad_items) + for item in items: defi = self.def_manager.get(item) if not defi: @@ -101,11 +104,6 @@ def iterate_mod_items(items, const): parentns = ".".join(splitted[:-1]) self.scope_manager.get_scope(parentns).add_def(name, defi) - if "test_" in self.filename: - print('skip', self.filename) - return - - print('modname', self.modname, self.filename) self.import_manager.set_current_mod(self.modname, self.filename) mod = self.module_manager.create(self.modname, self.filename) diff --git a/pycg/pycg.py b/pycg/pycg.py index b691457..713311a 100644 --- a/pycg/pycg.py +++ b/pycg/pycg.py @@ -139,12 +139,6 @@ def do_pass(self, cls, install_hooks=False, *args, **kwargs): if not input_mod: continue - print(input_pkg, input_file) - - if "test_" in input_file: - print('skip!') - continue - if not input_pkg: input_pkg = os.path.dirname(input_file) if input_mod not in modules_analyzed: From 0036baf1a553399f65a99afb700512bb6e1a9625 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 16:54:02 -0700 Subject: [PATCH 24/38] Fix --- pycg/processing/preprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 734d332..3786790 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -219,8 +219,8 @@ def add_external_def(name, target): tgt_name = import_item.asname if import_item.asname else import_item.name imported_name = self.import_manager.handle_import(src_name, level) - # Limit to exclusive module - if src_name.split(".")[0] not in self.exclusives: + # Limit to exclusive module if exclusives exist + if len(self.exclusives) > 0 and src_name.split(".")[0] not in self.exclusives: continue # Skip ignored modules From 9a5ce9209e97f14e0360cb7703b6c0fbd985e1cd Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 16:58:26 -0700 Subject: [PATCH 25/38] Fix --- pycg/processing/preprocessor.py | 6 ++++-- pycg/pycg.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 3786790..c8a334f 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -214,13 +214,15 @@ def add_external_def(name, target): tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) + print(node.names, self.modname) + for import_item in node.names: src_name = handle_src_name(import_item.name) tgt_name = import_item.asname if import_item.asname else import_item.name imported_name = self.import_manager.handle_import(src_name, level) # Limit to exclusive module if exclusives exist - if len(self.exclusives) > 0 and src_name.split(".")[0] not in self.exclusives: + if self.exclusives and src_name.split(".")[0] not in self.exclusives: continue # Skip ignored modules @@ -245,7 +247,7 @@ def add_external_def(name, target): # handle all modules that were not analyzed for modname in self.import_manager.get_imports(self.modname): - if modname.split(".")[0] not in self.exclusives: + if self.exclusives and modname.split(".")[0] not in self.exclusives: continue fname = self.import_manager.get_filepath(modname) diff --git a/pycg/pycg.py b/pycg/pycg.py index 713311a..bc673b9 100644 --- a/pycg/pycg.py +++ b/pycg/pycg.py @@ -141,6 +141,7 @@ def do_pass(self, cls, install_hooks=False, *args, **kwargs): if not input_pkg: input_pkg = os.path.dirname(input_file) + if input_mod not in modules_analyzed: if install_hooks: self.import_manager.set_pkg(input_pkg) From c4822437eba1a9250108bd48c6f00e270e6c50d3 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:02:20 -0700 Subject: [PATCH 26/38] Edit --- pycg/processing/preprocessor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index c8a334f..d4b27ab 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -120,7 +120,12 @@ def iterate_mod_items(items, const): items = self.scope_manager.handle_module( self.modname, self.filename, self.contents ) - print('items', items) + bad_items = [ + i for i in items if \ + any(ignored_mod in i for ignored_mod in self.ignored_mods) + ] + if len(bad_items)>0: + print('bad_items', bad_items) root_sc = self.scope_manager.get_scope(self.modname) root_defi = self.def_manager.get(self.modname) @@ -214,7 +219,7 @@ def add_external_def(name, target): tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) - print(node.names, self.modname) + print([*node.names], self.modname) for import_item in node.names: src_name = handle_src_name(import_item.name) From 57137191c019eabcf3f61663a385e80fc7e49382 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:09:44 -0700 Subject: [PATCH 27/38] Fix --- pycg/processing/preprocessor.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index d4b27ab..6e8406a 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -87,13 +87,11 @@ def analyze_submodule(self, modname): def visit_Module(self, node): def iterate_mod_items(items, const): - bad_items = [ + items = [ i for i in items if \ - any(ignored_mod in i for ignored_mod in self.ignored_mods) + not any(ignored_mod in i for ignored_mod in self.ignored_mods) ] - if len(bad_items) > 0: - print('bad iter items', bad_items) - + for item in items: defi = self.def_manager.get(item) if not defi: @@ -120,12 +118,6 @@ def iterate_mod_items(items, const): items = self.scope_manager.handle_module( self.modname, self.filename, self.contents ) - bad_items = [ - i for i in items if \ - any(ignored_mod in i for ignored_mod in self.ignored_mods) - ] - if len(bad_items)>0: - print('bad_items', bad_items) root_sc = self.scope_manager.get_scope(self.modname) root_defi = self.def_manager.get(self.modname) From 5d950da6bb67547067edf7f0b2b6af3a8cf73ac1 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:10:00 -0700 Subject: [PATCH 28/38] Fix --- pycg/processing/preprocessor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 6e8406a..98b7640 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -211,8 +211,6 @@ def add_external_def(name, target): tgt_defi.get_name_pointer().add(defi.get_ns()) scope.add_def(target, tgt_defi) - print([*node.names], self.modname) - for import_item in node.names: src_name = handle_src_name(import_item.name) tgt_name = import_item.asname if import_item.asname else import_item.name From a159b35af93f1789bfa14f4ec7ac71340ec0ede6 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:28:22 -0700 Subject: [PATCH 29/38] Fast re search --- pycg/processing/preprocessor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 98b7640..07bbbd2 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -19,6 +19,7 @@ # under the License. # import ast +import re from pycg import utils from pycg.machinery.definitions import Definition @@ -52,6 +53,8 @@ def __init__( self.class_manager = class_manager self.module_manager = module_manager + self.ignored_mod_pattern = re.compile('|'.join(map(re.escape, self.ignored_mods))) + def _get_fun_defaults(self, node): defaults = {} start = len(node.args.args) - len(node.args.defaults) @@ -88,8 +91,8 @@ def analyze_submodule(self, modname): def visit_Module(self, node): def iterate_mod_items(items, const): items = [ - i for i in items if \ - not any(ignored_mod in i for ignored_mod in self.ignored_mods) + item for item in items + if not self.ignored_mod_pattern.search(item) ] for item in items: @@ -224,6 +227,8 @@ def add_external_def(name, target): if tgt_name in self.ignored_mods: continue + print(tgt_name) + if not imported_name: add_external_def(src_name, tgt_name) continue @@ -245,6 +250,8 @@ def add_external_def(name, target): if self.exclusives and modname.split(".")[0] not in self.exclusives: continue + print(modname) + fname = self.import_manager.get_filepath(modname) if not fname: From dc182c73fb36b3ac2cfbcfe6dd107bc10284208c Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:34:01 -0700 Subject: [PATCH 30/38] Edit --- pycg/processing/postprocessor.py | 2 +- pycg/processing/preprocessor.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index 82ba2aa..0c74f05 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -185,7 +185,7 @@ def visit_ClassDef(self, node): cls = self.class_manager.get(cls_def.get_ns()) if not cls: cls = self.class_manager.create(cls_def.get_ns(), self.modname) - + print(cls.__name__) cls.clear_mro() for base in node.bases: # all bases are of the type ast.Name diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 07bbbd2..d5c483e 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -223,11 +223,7 @@ def add_external_def(name, target): if self.exclusives and src_name.split(".")[0] not in self.exclusives: continue - # Skip ignored modules - if tgt_name in self.ignored_mods: - continue - - print(tgt_name) + print('target',tgt_name) if not imported_name: add_external_def(src_name, tgt_name) @@ -250,8 +246,6 @@ def add_external_def(name, target): if self.exclusives and modname.split(".")[0] not in self.exclusives: continue - print(modname) - fname = self.import_manager.get_filepath(modname) if not fname: From c5319c5acdd700b95be229038cd11d70258acfdf Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:34:50 -0700 Subject: [PATCH 31/38] Opps --- pycg/processing/preprocessor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index d5c483e..b97fe06 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -223,7 +223,10 @@ def add_external_def(name, target): if self.exclusives and src_name.split(".")[0] not in self.exclusives: continue - print('target',tgt_name) + # Skip ignored modules + if tgt_name in self.ignored_mods: + print('skip',tgt_name) + continue if not imported_name: add_external_def(src_name, tgt_name) @@ -246,6 +249,8 @@ def add_external_def(name, target): if self.exclusives and modname.split(".")[0] not in self.exclusives: continue + print(modname) + fname = self.import_manager.get_filepath(modname) if not fname: From 48f573180fc6eed2da16d0518b4dcdb8a71f4689 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:42:26 -0700 Subject: [PATCH 32/38] Move imported_name down --- pycg/processing/preprocessor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index b97fe06..42313f4 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -109,6 +109,8 @@ def iterate_mod_items(items, const): mod = self.module_manager.create(self.modname, self.filename) + print('visit_mod', self.modname,self.filename) + first = 1 last = len(self.contents.splitlines()) if last == 0: @@ -217,8 +219,7 @@ def add_external_def(name, target): for import_item in node.names: src_name = handle_src_name(import_item.name) tgt_name = import_item.asname if import_item.asname else import_item.name - imported_name = self.import_manager.handle_import(src_name, level) - + # Limit to exclusive module if exclusives exist if self.exclusives and src_name.split(".")[0] not in self.exclusives: continue @@ -228,6 +229,8 @@ def add_external_def(name, target): print('skip',tgt_name) continue + imported_name = self.import_manager.handle_import(src_name, level) + if not imported_name: add_external_def(src_name, tgt_name) continue From b57b4c3b4a19df102352438255ff2a3e3f1ebfe2 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:45:13 -0700 Subject: [PATCH 33/38] Fix --- pycg/processing/postprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index 0c74f05..a3c0b73 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -185,7 +185,7 @@ def visit_ClassDef(self, node): cls = self.class_manager.get(cls_def.get_ns()) if not cls: cls = self.class_manager.create(cls_def.get_ns(), self.modname) - print(cls.__name__) + print(cls) cls.clear_mro() for base in node.bases: # all bases are of the type ast.Name From 89465773d385e1c8797c14fa813bccd20ff07dee Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:48:03 -0700 Subject: [PATCH 34/38] Tet --- pycg/processing/postprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index a3c0b73..1091596 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -185,7 +185,7 @@ def visit_ClassDef(self, node): cls = self.class_manager.get(cls_def.get_ns()) if not cls: cls = self.class_manager.create(cls_def.get_ns(), self.modname) - print(cls) + print(cls_def, node.name, node.bases) cls.clear_mro() for base in node.bases: # all bases are of the type ast.Name From 5cb4728b8a4765f382c4bc2f6fcf920c138b38d1 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 17:57:35 -0700 Subject: [PATCH 35/38] Skip --- pycg/processing/postprocessor.py | 5 ++++- pycg/processing/preprocessor.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index 1091596..b8b80c7 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -178,6 +178,10 @@ def visit_FunctionDef(self, node): super().visit_FunctionDef(node) def visit_ClassDef(self, node): + print('post visit', node.name) + if node.name in self.ignored_mods: + print('skip') + return # create a definition for the class (node.name) cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) @@ -185,7 +189,6 @@ def visit_ClassDef(self, node): cls = self.class_manager.get(cls_def.get_ns()) if not cls: cls = self.class_manager.create(cls_def.get_ns(), self.modname) - print(cls_def, node.name, node.bases) cls.clear_mro() for base in node.bases: # all bases are of the type ast.Name diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 42313f4..2fa47d2 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -433,6 +433,11 @@ def visit_Lambda(self, node): def visit_ClassDef(self, node): # create a definition for the class (node.name) + print('pre class', node.name) + if node.name in self.ignored_mods: + print('skip') + return + cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) mod = self.module_manager.get(self.modname) From 7daf94e8b64a213000eb6570258915731ab876f8 Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 18:07:59 -0700 Subject: [PATCH 36/38] Remove prints --- pycg/processing/postprocessor.py | 2 -- pycg/processing/preprocessor.py | 7 ------- 2 files changed, 9 deletions(-) diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index b8b80c7..5e6eb01 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -178,9 +178,7 @@ def visit_FunctionDef(self, node): super().visit_FunctionDef(node) def visit_ClassDef(self, node): - print('post visit', node.name) if node.name in self.ignored_mods: - print('skip') return # create a definition for the class (node.name) cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 2fa47d2..55a18c2 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -109,8 +109,6 @@ def iterate_mod_items(items, const): mod = self.module_manager.create(self.modname, self.filename) - print('visit_mod', self.modname,self.filename) - first = 1 last = len(self.contents.splitlines()) if last == 0: @@ -226,7 +224,6 @@ def add_external_def(name, target): # Skip ignored modules if tgt_name in self.ignored_mods: - print('skip',tgt_name) continue imported_name = self.import_manager.handle_import(src_name, level) @@ -252,8 +249,6 @@ def add_external_def(name, target): if self.exclusives and modname.split(".")[0] not in self.exclusives: continue - print(modname) - fname = self.import_manager.get_filepath(modname) if not fname: @@ -433,9 +428,7 @@ def visit_Lambda(self, node): def visit_ClassDef(self, node): # create a definition for the class (node.name) - print('pre class', node.name) if node.name in self.ignored_mods: - print('skip') return cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) From 697c4ad554c1c092805164a8517c6e7420019f6e Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 18:16:09 -0700 Subject: [PATCH 37/38] Cleanup --- pycg/__main__.py | 2 +- pycg/processing/postprocessor.py | 11 +++++++---- pycg/processing/preprocessor.py | 16 ++++++++-------- pycg/pycg.py | 8 ++++---- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pycg/__main__.py b/pycg/__main__.py index 8de8f58..0228971 100644 --- a/pycg/__main__.py +++ b/pycg/__main__.py @@ -62,7 +62,7 @@ def main(): cg = CallGraphGenerator( args.entry_point, args.package, - args.exclusives, args.ignored_mods, + args.exclusives, args.skip_classes, args.max_iter, args.operation ) cg.analyze() diff --git a/pycg/processing/postprocessor.py b/pycg/processing/postprocessor.py index 5e6eb01..757bee7 100644 --- a/pycg/processing/postprocessor.py +++ b/pycg/processing/postprocessor.py @@ -31,7 +31,7 @@ def __init__( input_file, modname, exclusives, - ignored_mods, + skip_classes, import_manager, scope_manager, def_manager, @@ -41,7 +41,7 @@ def __init__( ): super().__init__(input_file, modname, modules_analyzed) self.exclusives = exclusives - self.ignored_mods = ignored_mods + self.skip_classes = skip_classes self.import_manager = import_manager self.scope_manager = scope_manager self.def_manager = def_manager @@ -178,8 +178,10 @@ def visit_FunctionDef(self, node): super().visit_FunctionDef(node) def visit_ClassDef(self, node): - if node.name in self.ignored_mods: + # Return if visiting class should be skipped + if node.name in self.skip_classes: return + # create a definition for the class (node.name) cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) @@ -187,6 +189,7 @@ def visit_ClassDef(self, node): cls = self.class_manager.get(cls_def.get_ns()) if not cls: cls = self.class_manager.create(cls_def.get_ns(), self.modname) + cls.clear_mro() for base in node.bases: # all bases are of the type ast.Name @@ -335,7 +338,7 @@ def analyze_submodules(self): super().analyze_submodules( PostProcessor, self.exclusives, - self.ignored_mods, + self.skip_classes, self.import_manager, self.scope_manager, self.def_manager, diff --git a/pycg/processing/preprocessor.py b/pycg/processing/preprocessor.py index 55a18c2..7edfb8f 100644 --- a/pycg/processing/preprocessor.py +++ b/pycg/processing/preprocessor.py @@ -32,7 +32,7 @@ def __init__( filename, modname, exclusives, - ignored_mods, + skip_classes, import_manager, scope_manager, def_manager, @@ -44,7 +44,7 @@ def __init__( self.modname = modname self.exclusives = exclusives - self.ignored_mods = ignored_mods + self.skip_classes = skip_classes self.mod_dir = "/".join(self.filename.split("/")[:-1]) self.import_manager = import_manager @@ -53,7 +53,7 @@ def __init__( self.class_manager = class_manager self.module_manager = module_manager - self.ignored_mod_pattern = re.compile('|'.join(map(re.escape, self.ignored_mods))) + self.skip_classes_pattern = re.compile('|'.join(map(re.escape, self.skip_classes))) def _get_fun_defaults(self, node): defaults = {} @@ -79,7 +79,7 @@ def analyze_submodule(self, modname): PreProcessor, modname, self.exclusives, - self.ignored_mods, + self.skip_classes, self.import_manager, self.scope_manager, self.def_manager, @@ -92,7 +92,7 @@ def visit_Module(self, node): def iterate_mod_items(items, const): items = [ item for item in items - if not self.ignored_mod_pattern.search(item) + if not self.skip_classes_pattern.search(item) ] for item in items: @@ -222,8 +222,8 @@ def add_external_def(name, target): if self.exclusives and src_name.split(".")[0] not in self.exclusives: continue - # Skip ignored modules - if tgt_name in self.ignored_mods: + # Skip specified classes + if tgt_name in self.skip_classes: continue imported_name = self.import_manager.handle_import(src_name, level) @@ -428,7 +428,7 @@ def visit_Lambda(self, node): def visit_ClassDef(self, node): # create a definition for the class (node.name) - if node.name in self.ignored_mods: + if node.name in self.skip_classes: return cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) diff --git a/pycg/pycg.py b/pycg/pycg.py index bc673b9..7a354f9 100644 --- a/pycg/pycg.py +++ b/pycg/pycg.py @@ -36,12 +36,12 @@ class CallGraphGenerator(object): def __init__( - self, entry_points, package, exclusives, ignored_mods, max_iter, operation + self, entry_points, package, exclusives, skip_classes, max_iter, operation ): self.entry_points = entry_points self.package = package self.exclusives = exclusives - self.ignored_mods = ignored_mods + self.skip_classes = skip_classes self.state = None self.max_iter = max_iter self.operation = operation @@ -167,7 +167,7 @@ def analyze(self): PreProcessor, True, self.exclusives, - self.ignored_mods, + self.skip_classes, self.import_manager, self.scope_manager, self.def_manager, @@ -186,7 +186,7 @@ def analyze(self): PostProcessor, False, self.exclusives, - self.ignored_mods, + self.skip_classes, self.import_manager, self.scope_manager, self.def_manager, From b2a5d15466c79bf211043c333e1e04ae937affde Mon Sep 17 00:00:00 2001 From: Jackson Goode Date: Tue, 30 Apr 2024 23:03:40 -0700 Subject: [PATCH 38/38] Fix --- pycg/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycg/__main__.py b/pycg/__main__.py index 0228971..a5beffe 100644 --- a/pycg/__main__.py +++ b/pycg/__main__.py @@ -15,7 +15,7 @@ def main(): parser.add_argument( "--exclusives", help="Exclusive package to analyze", default=[]) parser.add_argument( - "--ignored-mods", help="Modules to ignore during traversal", default=[]) + "--skip-classes", help="Classes to skip during traversal", default=[]) parser.add_argument( "--fasten", help="Produce call graph using the FASTEN format",