Fix bug finding implicit namespace packages

AUTHORS.txt

@@ -62,6 +62,7 @@ Code Contributors
 - Andrii Kolomoiets (@muffinmad)
 - Leo Ryu (@Leo-Ryu)
 - Joseph Birkner (@josephbirkner)
+- Bryan Bugyi (@bbugyi200) <[email protected]>
 
 And a few more "anonymous" contributors.
 

CHANGELOG.rst

@@ -7,6 +7,7 @@ Unreleased
 ++++++++++
 
 - Implict namespaces are now a separate types in ``Name().type``
+- Fix bug finding implicit namespace packages (`Issue:#1759 <https://github.com/davidhalter/jedi/issues/1759>`_, `PR:#1784 <https://github.com/davidhalter/jedi/pull/1784>`_)
 
 0.18.0 (2020-12-25)
 +++++++++++++++++++

jedi/inference/compiled/subprocess/functions.py

@@ -6,14 +6,29 @@
 from pathlib import Path
 from zipfile import ZipFile
 from zipimport import zipimporter, ZipImportError
-from importlib.machinery import all_suffixes
+from io import FileIO
+from typing import (
+    Any,
+    Optional,
+    Sequence,
+    Tuple,
+    TYPE_CHECKING,
+    Union,
+)
 
 from jedi.inference.compiled import access
 from jedi import debug
 from jedi import parser_utils
 from jedi.file_io import KnownContentFileIO, ZipFileIO
 
 
+if TYPE_CHECKING:
+    from jedi.inference import InferenceState
+
+
+ModuleInfoResult = Tuple[Union[Any, FileIO, None], Optional[bool]]
+
+
 def get_sys_path():
     return sys.path
 
@@ -31,14 +46,29 @@ def create_simple_object(inference_state, obj):
     return access.create_access_path(inference_state, obj)
 
 
-def get_module_info(inference_state, sys_path=None, full_name=None, **kwargs):
+def get_module_info(
+    inference_state: "InferenceState",
+    *,
+    string: str,
+    sys_path: Sequence[str] = None,
+    full_name: str = None,
+    path: Sequence[str] = None,
+    is_global_search: bool = True,
+) -> ModuleInfoResult:
     """
     Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
     """
+    del inference_state
+
     if sys_path is not None:
-        sys.path, temp = sys_path, sys.path
+        sys.path, temp = list(sys_path), sys.path
     try:
-        return _find_module(full_name=full_name, **kwargs)
+        return _find_module(
+            string=string,
+            full_name=full_name,
+            path=path,
+            is_global_search=is_global_search,
+        )
     except ImportError:
         return None, None
     finally:
@@ -69,18 +99,6 @@ def _test_print(inference_state, stderr=None, stdout=None):
         sys.stdout.flush()
 
 
-def _get_init_path(directory_path):
-    """
-    The __init__ file can be searched in a directory. If found return it, else
-    None.
-    """
-    for suffix in all_suffixes():
-        path = os.path.join(directory_path, '__init__' + suffix)
-        if os.path.exists(path):
-            return path
-    return None
-
-
 def safe_literal_eval(inference_state, value):
     return parser_utils.safe_literal_eval(value)
 
@@ -124,7 +142,12 @@ def _iter_module_names(inference_state, paths):
                         yield modname
 
 
-def _find_module(string, path=None, full_name=None, is_global_search=True):
+def _find_module(
+    string: str,
+    path: Sequence[str] = None,
+    full_name: str = None,
+    is_global_search: bool = True,
+) -> ModuleInfoResult:
     """
     Provides information about a module.
 
@@ -138,10 +161,11 @@ def _find_module(string, path=None, full_name=None, is_global_search=True):
     loader = None
 
     for finder in sys.meta_path:
-        if is_global_search and finder != importlib.machinery.PathFinder:
+        if is_global_search and finder != importlib.machinery.PathFinder:  # type: ignore
             p = None
         else:
             p = path
+
         try:
             find_spec = finder.find_spec
         except AttributeError:
@@ -155,14 +179,22 @@ def _find_module(string, path=None, full_name=None, is_global_search=True):
             if loader is None and not spec.has_location:
                 # This is a namespace package.
                 full_name = string if not path else full_name
-                implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
+                implicit_ns_info = ImplicitNSInfo(
+                    full_name,
+                    spec.submodule_search_locations._path,  # type: ignore
+                )
                 return implicit_ns_info, True
+
             break
 
     return _find_module_py33(string, path, loader)
 
 
-def _find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
+def _find_module_py33(
+    string: str,
+    path: Sequence[str] = None,
+    loader: Any = None,
+) -> ModuleInfoResult:
     loader = loader or importlib.machinery.PathFinder.find_module(string, path)
 
     if loader is None and path is None:  # Fallback to find builtins
@@ -185,7 +217,7 @@ def _find_module_py33(string, path=None, loader=None, full_name=None, is_global_
     return _from_loader(loader, string)
 
 
-def _from_loader(loader, string):
+def _from_loader(loader: Any, string: str) -> ModuleInfoResult:
     try:
         is_package_method = loader.is_package
     except AttributeError:

jedi/inference/imports.py

@@ -10,6 +10,7 @@
 """
 import os
 from pathlib import Path
+from typing import Any, Sequence, TYPE_CHECKING
 
 from parso.python import tree
 from parso.tree import search_ancestor
@@ -32,6 +33,10 @@
 from jedi.plugins import plugin_manager
 
 
+if TYPE_CHECKING:
+    from jedi.inference import InferenceState
+
+
 class ModuleCache:
     def __init__(self):
         self._name_cache = {}
@@ -394,7 +399,12 @@ def import_module_by_names(inference_state, import_names, sys_path=None,
 
 @plugin_manager.decorate()
 @import_module_decorator
-def import_module(inference_state, import_names, parent_module_value, sys_path):
+def import_module(
+    inference_state: "InferenceState",
+    import_names: Sequence[str],
+    parent_module_value: Any,
+    sys_path: Sequence[str],
+) -> ValueSet:
     """
     This method is very similar to importlib's `_gcd_import`.
     """
@@ -422,20 +432,13 @@ def import_module(inference_state, import_names, parent_module_value, sys_path):
             # The module might not be a package.
             return NO_VALUES
 
-        for path in paths:
-            # At the moment we are only using one path. So this is
-            # not important to be correct.
-            if not isinstance(path, list):
-                path = [path]
-            file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
-                string=import_names[-1],
-                path=path,
-                full_name=module_name,
-                is_global_search=False,
-            )
-            if is_pkg is not None:
-                break
-        else:
+        file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
+            string=import_names[-1],
+            path=paths,
+            full_name=module_name,
+            is_global_search=False,
+        )
+        if is_pkg is None:
             return NO_VALUES
 
     if isinstance(file_io_or_ns, ImplicitNSInfo):

test/test_api/test_project.py

@@ -98,7 +98,8 @@ def test_load_save_project(tmpdir):
         ('examples.implicit_namespace_package.ns1.pkg.ns1_file',
          ['examples.implicit_namespace_package.ns1.pkg.ns1_file'], {}),
         ('implicit_namespace_package.ns1.pkg.',
-         ['examples.implicit_namespace_package.ns1.pkg.ns1_file'],
+         ['examples.implicit_namespace_package.ns1.pkg.ns1_file',
+          'examples.implicit_namespace_package.ns1.pkg.subpkg'],
          dict(complete=True)),
         ('implicit_namespace_package.',
          ['examples.implicit_namespace_package.ns1',

test/test_inference/test_implicit_namespace_package.py

@@ -1,4 +1,8 @@
-from test.helpers import get_example_dir, example_dir
+from test.helpers import example_dir, get_example_dir
+from typing import Any, Iterable
+
+import pytest
+
 from jedi import Project
 
 
@@ -28,7 +32,7 @@ def script_with_path(*args, **kwargs):
     # completion
     completions = script_with_path('from pkg import ').complete()
     names = [c.name for c in completions]
-    compare = ['ns1_file', 'ns2_file']
+    compare = ['ns1_file', 'ns2_file', 'subpkg']
     # must at least contain these items, other items are not important
     assert set(compare) == set(names)
 
@@ -68,19 +72,40 @@ def test_implicit_namespace_package_import_autocomplete(Script):
     assert [c.name for c in compl] == ['implicit_namespace_package']
 
 
-def test_namespace_package_in_multiple_directories_autocompletion(Script):
-    code = 'from pkg.'
+@pytest.mark.parametrize(
+    'code,expected',
+    [
+        ('from pkg.', ['ns1_file', 'ns2_file', 'subpkg']),
+        ('from pkg.subpkg.', ['ns3_file', 'ns4_file']),
+    ]
+)
+def test_namespace_package_in_multiple_directories_autocompletion(
+    code: str,
+    expected: Iterable[str],
+    Script: Any,
+) -> None:
     sys_path = [get_example_dir('implicit_namespace_package', 'ns1'),
                 get_example_dir('implicit_namespace_package', 'ns2')]
 
     project = Project('.', sys_path=sys_path)
     script = Script(code, project=project)
     compl = script.complete()
-    assert set(c.name for c in compl) == set(['ns1_file', 'ns2_file'])
-
-
-def test_namespace_package_in_multiple_directories_goto_definition(Script):
-    code = 'from pkg import ns1_file'
+    assert set(c.name for c in compl) == set(expected)
+
+
+@pytest.mark.parametrize(
+    'code',
+    [
+        'from pkg import ns1_file',
+        'from pkg import ns2_file',
+        'from pkg.subpkg import ns3_file',
+        'from pkg.subpkg import ns4_file',
+    ]
+)
+def test_namespace_package_in_multiple_directories_goto_definition(
+    code: str,
+    Script: Any,
+) -> None:
     sys_path = [get_example_dir('implicit_namespace_package', 'ns1'),
                 get_example_dir('implicit_namespace_package', 'ns2')]
     project = Project('.', sys_path=sys_path)

test/test_inference/test_imports.py

@@ -118,7 +118,7 @@ def test_find_module_not_package_zipped(Script, inference_state, environment):
     assert len(script.complete()) == 1
 
     file_io, is_package = inference_state.compiled_subprocess.get_module_info(
-        sys_path=map(str, sys_path),
+        sys_path=list(map(str, sys_path)),
         string='not_pkg',
         full_name='not_pkg'
     )