From: Alex Willmer <alex@moreati.org.uk>
Date: Sun, 17 Mar 2024 14:55:15 +0000
Subject: mitogen: Support PEP 451 ModuleSpec API, required for Python 3.12

importlib.machinery.ModuleSpec and find_spec() were introduced in Python 3.4
under PEP 451. They replace the find_module() API of PEP 302, which was
deprecated from Python 3.4. They were removed in Python 3.12 along with the
imp module.

This change adds support for the PEP 451 APIs. Mitogen should no longer import
imp on Python versions that support ModuleSpec. Tests have been added to cover
the new APIs.

CI jobs have been added to cover Python 3.x on macOS.

Refs #1033
Co-authored-by: Witold Baryluk <witold.baryluk@gmail.com>
Bug-Debain: https://bugs.debian.org/1111363
---
 ansible_mitogen/module_finder.py                   | 122 +++++++++++++++-
 ansible_mitogen/runner.py                          |  91 +++++++++++-
 docs/internals.rst                                 |   2 +-
 mitogen/core.py                                    | 153 +++++++++++++++++++--
 mitogen/master.py                                  | 122 +++++++++++++++-
 .../connection_delegation/delegate_to_template.yml |   4 +-
 .../interpreter_discovery/ansible_2_8_tests.yml    |   7 -
 tests/ansible/lib/modules/module_finder_test.py    |  12 ++
 tests/ansible/tests/module_finder_test.py          |  80 +++++++++++
 tests/importer_test.py                             | 126 +++++++++++++++--
 tests/module_finder_test.py                        |  14 +-
 tests/testlib.py                                   |   9 +-
 12 files changed, 697 insertions(+), 45 deletions(-)
 create mode 100644 tests/ansible/lib/modules/module_finder_test.py
 create mode 100644 tests/ansible/tests/module_finder_test.py

diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py
index cec465c..f227c7b 100644
--- a/ansible_mitogen/module_finder.py
+++ b/ansible_mitogen/module_finder.py
@@ -31,12 +31,23 @@ from __future__ import unicode_literals
 __metaclass__ = type
 
 import collections
-import imp
+import logging
 import os
+import re
+import sys
+
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
 
 import mitogen.master
 
 
+LOG = logging.getLogger(__name__)
 PREFIX = 'ansible.module_utils.'
 
 
@@ -119,14 +130,121 @@ def find_relative(parent, name, path=()):
 
 
 def scan_fromlist(code):
+    """Return an iterator of (level, name) for explicit imports in a code
+    object.
+
+    Not all names identify a module. `from os import name, path` generates
+    `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
+
+    >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
+    >>> code = compile(src, '<str>', 'exec')
+    >>> list(scan_fromlist(code))
+    [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
+    """
     for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
         for name in fromlist:
-            yield level, '%s.%s' % (modname_s, name)
+            yield level, str('%s.%s' % (modname_s, name))
         if not fromlist:
             yield level, modname_s
 
 
+def walk_imports(code, prefix=None):
+    """Return an iterator of names for implicit parent imports & explicit
+    imports in a code object.
+
+    If a prefix is provided, then only children of that prefix are included.
+    Not all names identify a module. `from os import name, path` generates
+    `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
+
+    >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
+    >>> code = compile(source, '<str>', 'exec')
+    >>> list(walk_imports(code))
+    ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
+    >>> list(walk_imports(code, prefix='b'))
+    ['b.c', 'b.d', 'b.d.e', 'b.d.f']
+    """
+    if prefix is None:
+        prefix = ''
+    pattern = re.compile(r'(^|\.)(\w+)')
+    start = len(prefix)
+    for _, name, fromlist in mitogen.master.scan_code_imports(code):
+        if not name.startswith(prefix):
+            continue
+        for match in pattern.finditer(name, start):
+            yield name[:match.end()]
+        for leaf in fromlist:
+            yield str('%s.%s' % (name, leaf))
+
+
 def scan(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    """Return a list of (name, path, is_package) for ansible.module_utils
+    imports used by an Ansible module.
+    """
+    log = LOG.getChild('scan')
+    log.debug('%r, %r, %r', module_name, module_path, search_path)
+
+    if sys.version_info >= (3, 4):
+        result = _scan_importlib_find_spec(
+            module_name, module_path, search_path,
+        )
+        log.debug('_scan_importlib_find_spec %r', result)
+    else:
+        result = _scan_imp_find_module(module_name, module_path, search_path)
+        log.debug('_scan_imp_find_module %r', result)
+    return result
+
+
+def _scan_importlib_find_spec(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    module = importlib.machinery.ModuleSpec(
+        module_name, loader=None, origin=module_path,
+    )
+    prefix = importlib.machinery.ModuleSpec(
+        PREFIX.rstrip('.'), loader=None,
+    )
+    prefix.submodule_search_locations = search_path
+    queue = collections.deque([module])
+    specs = {prefix.name: prefix}
+    while queue:
+        spec = queue.popleft()
+        if spec.origin is None:
+            continue
+        try:
+            with open(spec.origin, 'rb') as f:
+                code = compile(f.read(), spec.name, 'exec')
+        except Exception as exc:
+            raise ValueError((exc, module, spec, specs))
+
+        for name in walk_imports(code, prefix.name):
+            if name in specs:
+                continue
+
+            parent_name = name.rpartition('.')[0]
+            parent = specs[parent_name]
+            if parent is None or not parent.submodule_search_locations:
+                specs[name] = None
+                continue
+
+            child = importlib.util._find_spec(
+                name, parent.submodule_search_locations,
+            )
+            if child is None or child.origin is None:
+                specs[name] = None
+                continue
+
+            specs[name] = child
+            queue.append(child)
+
+    del specs[prefix.name]
+    return sorted(
+        (spec.name, spec.origin, spec.submodule_search_locations is not None)
+        for spec in specs.values() if spec is not None
+    )
+
+
+def _scan_imp_find_module(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
     module = Module(module_name, module_path, imp.PY_SOURCE, None)
     stack = [module]
     seen = set()
diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py
index 31ccf1c..4306b0b 100644
--- a/ansible_mitogen/runner.py
+++ b/ansible_mitogen/runner.py
@@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import atexit
-import imp
 import os
 import re
 import shlex
@@ -63,6 +62,14 @@ except ImportError:
     # Python 2.4
     ctypes = None
 
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+
 try:
     import json
 except ImportError:
@@ -519,10 +526,71 @@ class ModuleUtilsImporter(object):
             sys.modules.pop(fullname, None)
 
     def find_module(self, fullname, path=None):
+        """
+        Return a loader for the module with fullname, if we will load it.
+
+        Implements importlib.abc.MetaPathFinder.find_module().
+        Deprecrated in Python 3.4+, replaced by find_spec().
+        Raises ImportWarning in Python 3.10+. Removed in Python 3.12.
+        """
         if fullname in self._by_fullname:
             return self
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`.
+
+        Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+.
+        """
+        if fullname.endswith('.'):
+            return None
+
+        try:
+            module_path, is_package = self._by_fullname[fullname]
+        except KeyError:
+            LOG.debug('Skipping %s: not present', fullname)
+            return None
+
+        LOG.debug('Handling %s', fullname)
+        origin = 'master:%s' % (module_path,)
+        return importlib.machinery.ModuleSpec(
+            fullname, loader=self, origin=origin, is_package=is_package,
+        )
+
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        spec = module.__spec__
+        path, _ = self._by_fullname[spec.name]
+        source = ansible_mitogen.target.get_small_file(self._context, path)
+        code = compile(source, path, 'exec', 0, 1)
+        exec(code, module.__dict__)
+        self._loaded.add(spec.name)
+
     def load_module(self, fullname):
+        """
+        Return the loaded module specified by fullname.
+
+        Implements PEP 302 importlib.abc.Loader.load_module().
+        Deprecated in Python 3.4+, replaced by create_module() & exec_module().
+        """
         path, is_pkg = self._by_fullname[fullname]
         source = ansible_mitogen.target.get_small_file(self._context, path)
         code = compile(source, path, 'exec', 0, 1)
@@ -823,12 +891,17 @@ class NewStyleRunner(ScriptRunner):
         synchronization mechanism by importing everything the module will need
         prior to detaching.
         """
+        # I think "custom" means "found in custom module_utils search path",
+        # e.g. playbook relative dir, ~/.ansible/..., Ansible collection.
         for fullname, _, _ in self.module_map['custom']:
             mitogen.core.import_module(fullname)
+
+        # I think "builtin" means "part of ansible/ansible-base/ansible-core",
+        # as opposed to Python builtin modules such as sys.
         for fullname in self.module_map['builtin']:
             try:
                 mitogen.core.import_module(fullname)
-            except ImportError:
+            except ImportError as exc:
                 # #590: Ansible 2.8 module_utils.distro is a package that
                 # replaces itself in sys.modules with a non-package during
                 # import. Prior to replacement, it is a real package containing
@@ -839,8 +912,18 @@ class NewStyleRunner(ScriptRunner):
                 # loop progresses to the next entry and attempts to preload
                 # 'distro._distro', the import mechanism will fail. So here we
                 # silently ignore any failure for it.
-                if fullname != 'ansible.module_utils.distro._distro':
-                    raise
+                if fullname == 'ansible.module_utils.distro._distro':
+                    continue
+
+                # ansible.module_utils.compat.selinux raises ImportError if it
+                # can't load libselinux.so. The importer would usually catch
+                # this & skip selinux operations. We don't care about selinux,
+                # we're using import to get a copy of the module.
+                if (fullname == 'ansible.module_utils.compat.selinux'
+                    and exc.msg == 'unable to load libselinux.so'):
+                    continue
+
+                raise
 
     def _setup_excepthook(self):
         """
diff --git a/docs/internals.rst b/docs/internals.rst
index 7f44d7b..434a74f 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -174,7 +174,7 @@ Module Finders
    :members:
 
 .. currentmodule:: mitogen.master
-.. autoclass:: ParentEnumerationMethod
+.. autoclass:: ParentImpEnumerationMethod
    :members:
 
 
diff --git a/mitogen/core.py b/mitogen/core.py
index bee722e..253fb07 100644
--- a/mitogen/core.py
+++ b/mitogen/core.py
@@ -54,13 +54,18 @@ import syslog
 import threading
 import time
 import traceback
+import types
 import warnings
 import weakref
 import zlib
 
-# Python >3.7 deprecated the imp module.
-warnings.filterwarnings('ignore', message='the imp module is deprecated')
-import imp
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
 
 # Absolute imports for <2.5.
 select = __import__('select')
@@ -1353,6 +1358,19 @@ class Importer(object):
     def __repr__(self):
         return 'Importer'
 
+    @staticmethod
+    def _loader_from_module(module, default=None):
+        """Return the loader for a module object."""
+        try:
+            return module.__spec__.loader
+        except AttributeError:
+            pass
+        try:
+            return module.__loader__
+        except AttributeError:
+            pass
+        return default
+
     def builtin_find_module(self, fullname):
         # imp.find_module() will always succeed for __main__, because it is a
         # built-in module. That means it exists on a special linked list deep
@@ -1388,14 +1406,13 @@ class Importer(object):
         try:
             #_v and self._log.debug('Python requested %r', fullname)
             fullname = to_text(fullname)
-            pkgname, dot, _ = str_rpartition(fullname, '.')
+            pkgname, _, suffix = str_rpartition(fullname, '.')
             pkg = sys.modules.get(pkgname)
             if pkgname and getattr(pkg, '__loader__', None) is not self:
                 self._log.debug('%s is submodule of a locally loaded package',
                                 fullname)
                 return None
 
-            suffix = fullname[len(pkgname+dot):]
             if pkgname and suffix not in self._present.get(pkgname, ()):
                 self._log.debug('%s has no submodule %s', pkgname, suffix)
                 return None
@@ -1415,6 +1432,66 @@ class Importer(object):
         finally:
             del _tls.running
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`, allowing other finders to try.
+
+        fullname    Fully qualified name of the module (e.g. foo.bar.baz)
+        path        Path entries to search. None for a top-level module.
+        target      Existing module to be reloaded (if any).
+
+        Implements importlib.abc.MetaPathFinder.find_spec()
+        Python 3.4+.
+        """
+        # Presence of _tls.running indicates we've re-invoked importlib.
+        # Abort early to prevent infinite recursion. See below.
+        if hasattr(_tls, 'running'):
+            return None
+
+        log = self._log.getChild('find_spec')
+
+        if fullname.endswith('.'):
+            return None
+
+        pkgname, _, modname = fullname.rpartition('.')
+        if pkgname and modname not in self._present.get(pkgname, ()):
+            log.debug('Skipping %s. Parent %s has no submodule %s',
+                      fullname, pkgname, modname)
+            return None
+
+        pkg = sys.modules.get(pkgname)
+        pkg_loader = self._loader_from_module(pkg)
+        if pkgname and pkg_loader is not self:
+            log.debug('Skipping %s. Parent %s was loaded by %r',
+                      fullname, pkgname, pkg_loader)
+            return None
+
+        # #114: whitelisted prefixes override any system-installed package.
+        if self.whitelist != ['']:
+            if any(s and fullname.startswith(s) for s in self.whitelist):
+                log.debug('Handling %s. It is whitelisted', fullname)
+                return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+        if fullname == '__main__':
+            log.debug('Handling %s. A special case', fullname)
+            return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+        # Re-invoke the import machinery to allow other finders to try.
+        # Set a guard, so we don't infinitely recurse. See top of this method.
+        _tls.running = True
+        try:
+            spec = importlib.util._find_spec(fullname, path, target)
+        finally:
+            del _tls.running
+
+        if spec:
+            log.debug('Skipping %s. Available as %r', fullname, spec)
+            return spec
+
+        log.debug('Handling %s. Unavailable locally', fullname)
+        return importlib.machinery.ModuleSpec(fullname, loader=self)
+
     blacklisted_msg = (
         '%r is present in the Mitogen importer blacklist, therefore this '
         'context will not attempt to request it from the master, as the '
@@ -1501,6 +1578,64 @@ class Importer(object):
         if present:
             callback()
 
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        self._log.debug('Creating module for %r', spec)
+
+        # FIXME Should this be done in find_spec()? Can it?
+        self._refuse_imports(spec.name)
+
+        # FIXME "create_module() should properly handle the case where it is
+        #       called more than once for the same spec/module." -- PEP-451
+        event = threading.Event()
+        self._request_module(spec.name, callback=event.set)
+        event.wait()
+
+        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+        _, pkg_present, path, _, _ = self._cache[spec.name]
+
+        if path is None:
+            raise ImportError(self.absent_msg % (spec.name))
+
+        spec.origin = self.get_filename(spec.name)
+        if pkg_present is not None:
+            # TODO Namespace packages
+            spec.submodule_search_locations = []
+            self._present[spec.name] = pkg_present
+
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        name = module.__spec__.name
+        origin = module.__spec__.origin
+        self._log.debug('Executing %s from %s', name, origin)
+        source = self.get_source(name)
+        try:
+            # Compile the source into a code object. Don't add any __future__
+            # flags and don't inherit any from this module.
+            # FIXME Should probably be exposed as get_code()
+            code = compile(source, origin, 'exec', flags=0, dont_inherit=True)
+        except SyntaxError:
+            # FIXME Why is this LOG, rather than self._log?
+            LOG.exception('while importing %r', name)
+            raise
+
+        exec(code, module.__dict__)
+
     def load_module(self, fullname):
         """
         Return the loaded module specified by fullname.
@@ -1516,11 +1651,11 @@ class Importer(object):
         self._request_module(fullname, event.set)
         event.wait()
 
-        ret = self._cache[fullname]
-        if ret[2] is None:
+        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+        _, pkg_present, path, _, _ = self._cache[fullname]
+        if path is None:
             raise ModuleNotFoundError(self.absent_msg % (fullname,))
 
-        pkg_present = ret[1]
         mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
         mod.__file__ = self.get_filename(fullname)
         mod.__loader__ = self
@@ -3921,7 +4056,7 @@ class ExternalContext(object):
 
     def _setup_package(self):
         global mitogen
-        mitogen = imp.new_module('mitogen')
+        mitogen = types.ModuleType('mitogen')
         mitogen.__package__ = 'mitogen'
         mitogen.__path__ = []
         mitogen.__loader__ = self.importer
diff --git a/mitogen/master.py b/mitogen/master.py
index 4fb535f..9b5c2ff 100644
--- a/mitogen/master.py
+++ b/mitogen/master.py
@@ -37,7 +37,6 @@ contexts.
 
 import dis
 import errno
-import imp
 import inspect
 import itertools
 import logging
@@ -50,6 +49,16 @@ import threading
 import types
 import zlib
 
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+    from _imp import is_builtin as _is_builtin
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+    from imp import is_builtin as _is_builtin
+
 try:
     import sysconfig
 except ImportError:
@@ -122,14 +131,14 @@ def is_stdlib_name(modname):
     """
     Return :data:`True` if `modname` appears to come from the standard library.
     """
-    # `imp.is_builtin()` isn't a documented as part of Python's stdlib API.
+    # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib.
     #
     # """
     # Main is a little special - imp.is_builtin("__main__") will return False,
     # but BuiltinImporter is still the most appropriate initial setting for
     # its __loader__ attribute.
     # """ -- comment in CPython pylifecycle.c:add_main_module()
-    if imp.is_builtin(modname) != 0:
+    if _is_builtin(modname) != 0:
         return True
 
     module = sys.modules.get(modname)
@@ -460,6 +469,9 @@ class FinderMethod(object):
     name according to the running Python interpreter. You'd think this was a
     simple task, right? Naive young fellow, welcome to the real world.
     """
+    def __init__(self):
+        self.log = LOG.getChild(self.__class__.__name__)
+
     def __repr__(self):
         return '%s()' % (type(self).__name__,)
 
@@ -641,7 +653,7 @@ class SysModulesMethod(FinderMethod):
         return path, source, is_pkg
 
 
-class ParentEnumerationMethod(FinderMethod):
+class ParentImpEnumerationMethod(FinderMethod):
     """
     Attempt to fetch source code by examining the module's (hopefully less
     insane) parent package, and if no insane parents exist, simply use
@@ -668,9 +680,9 @@ class ParentEnumerationMethod(FinderMethod):
     @staticmethod
     def _iter_parents(fullname):
         """
-        >>> list(ParentEnumerationMethod._iter_parents('a'))
+        >>> list(ParentImpEnumerationMethod._iter_parents('a'))
         [('', 'a')]
-        >>> list(ParentEnumerationMethod._iter_parents('a.b.c'))
+        >>> list(ParentImpEnumerationMethod._iter_parents('a.b.c'))
         [('a.b', 'c'), ('a', 'b'), ('', 'a')]
         """
         while fullname:
@@ -770,6 +782,9 @@ class ParentEnumerationMethod(FinderMethod):
         """
         See implementation for a description of how this works.
         """
+        if sys.version_info >= (3, 4):
+            return None
+
         #if fullname not in sys.modules:
             # Don't attempt this unless a module really exists in sys.modules,
             # else we could return junk.
@@ -798,6 +813,98 @@ class ParentEnumerationMethod(FinderMethod):
                 return self._found_module(fullname, path, fp)
 
 
+class ParentSpecEnumerationMethod(ParentImpEnumerationMethod):
+    def _find_parent_spec(self, fullname):
+        #history = []
+        debug = self.log.debug
+        children = []
+        for parent_name, child_name in self._iter_parents(fullname):
+            children.insert(0, child_name)
+            if not parent_name:
+                debug('abandoning %r, reached top-level', fullname)
+                return None, children
+
+            try:
+                parent = sys.modules[parent_name]
+            except KeyError:
+                debug('skipping %r, not in sys.modules', parent_name)
+                continue
+
+            try:
+                spec = parent.__spec__
+            except AttributeError:
+                debug('skipping %r: %r.__spec__ is absent',
+                      parent_name, parent)
+                continue
+
+            if not spec:
+                debug('skipping %r: %r.__spec__=%r',
+                      parent_name, parent, spec)
+                continue
+
+            if spec.name != parent_name:
+                debug('skipping %r: %r.__spec__.name=%r does not match',
+                      parent_name, parent, spec.name)
+                continue
+
+            if not spec.submodule_search_locations:
+                debug('skipping %r: %r.__spec__.submodule_search_locations=%r',
+                      parent_name, parent, spec.submodule_search_locations)
+                continue
+
+            return spec, children
+
+        raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
+    def find(self, fullname):
+        # Returns absolute path, ParentImpEnumerationMethod returns relative
+        # >>> spec_pem.find('six_brokenpkg._six')[::2]
+        # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False)
+
+        if sys.version_info < (3, 4):
+            return None
+
+        fullname = to_text(fullname)
+        spec, children = self._find_parent_spec(fullname)
+        for child_name in children:
+            if spec:
+                name = '%s.%s' % (spec.name, child_name)
+                submodule_search_locations = spec.submodule_search_locations
+            else:
+                name = child_name
+                submodule_search_locations = None
+            spec = importlib.util._find_spec(name, submodule_search_locations)
+            if spec is None:
+                self.log.debug('%r spec unavailable from %s', fullname, spec)
+                return None
+
+            is_package = spec.submodule_search_locations is not None
+            if name != fullname:
+                if not is_package:
+                    self.log.debug('%r appears to be child of non-package %r',
+                                   fullname, spec)
+                    return None
+                continue
+
+            if not spec.has_location:
+                self.log.debug('%r.origin cannot be read as a file', spec)
+                return None
+
+            if os.path.splitext(spec.origin)[1] != '.py':
+                self.log.debug('%r.origin does not contain Python source code',
+                               spec)
+                return None
+
+            # FIXME This should use loader.get_source()
+            with open(spec.origin, 'rb') as f:
+                source = f.read()
+
+            return spec.origin, source, is_package
+
+        raise ValueError('%s.find(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
 class ModuleFinder(object):
     """
     Given the name of a loaded module, make a best-effort attempt at finding
@@ -838,7 +945,8 @@ class ModuleFinder(object):
         DefectivePython3xMainMethod(),
         PkgutilMethod(),
         SysModulesMethod(),
-        ParentEnumerationMethod(),
+        ParentSpecEnumerationMethod(),
+        ParentImpEnumerationMethod(),
     ]
 
     def get_module_source(self, fullname):
diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml
index be083ff..357c485 100644
--- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml
+++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml
@@ -42,7 +42,7 @@
               'keepalive_count': 10,
               'password': null,
               'port': null,
-              'python_path': ["/usr/bin/python"],
+              'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
               'remote_name': null,
               'ssh_args': [
                 '-o',
@@ -72,7 +72,7 @@
               'keepalive_count': 10,
               'password': null,
               'port': null,
-              'python_path': ["/usr/bin/python"],
+              'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
               'remote_name': null,
               'ssh_args': [
                 '-o',
diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
index 201ef8b..fa9baba 100644
--- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
+++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
@@ -190,13 +190,6 @@
               - distro == 'ubuntu'
               - distro_version is version('16.04', '>=', strict=True)
 
-          - name: mac assertions
-            assert:
-              that:
-              - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
-              fail_msg: auto_out={{auto_out}}
-            when: os_family == 'Darwin'
-
           always:
           - meta: clear_facts
       when:
diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py
new file mode 100644
index 0000000..41cf1c1
--- /dev/null
+++ b/tests/ansible/lib/modules/module_finder_test.py
@@ -0,0 +1,12 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+import ansible.module_utils.external1
+
+from ansible.module_utils.externalpkg.extmod import path as epem_path
+
+def main():
+    pass
diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py
new file mode 100644
index 0000000..79e8fdb
--- /dev/null
+++ b/tests/ansible/tests/module_finder_test.py
@@ -0,0 +1,80 @@
+import os.path
+import sys
+import textwrap
+import unittest
+
+import ansible_mitogen.module_finder
+
+import testlib
+
+
+class ScanFromListTest(testlib.TestCase):
+    def test_absolute_imports(self):
+        source = textwrap.dedent('''\
+            from __future__ import absolute_import
+            import a; import b.c; from d.e import f; from g import h, i
+        ''')
+        code = compile(source, '<str>', 'exec')
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.scan_fromlist(code)),
+            [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')],
+        )
+
+
+class WalkImportsTest(testlib.TestCase):
+    def test_absolute_imports(self):
+        source = textwrap.dedent('''\
+            from __future__ import absolute_import
+            import a; import b; import b.c; from b.d import e, f
+        ''')
+        code = compile(source, '<str>', 'exec')
+
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.walk_imports(code)),
+            ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'],
+        )
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')),
+            ['b.c', 'b.d', 'b.d.e', 'b.d.f'],
+        )
+
+
+class ScanTest(testlib.TestCase):
+    module_name = 'ansible_module_module_finder_test__this_should_not_matter'
+    module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py')
+    search_path = (
+        'does_not_exist/module_utils',
+        testlib.ANSIBLE_MODULE_UTILS_DIR,
+    )
+
+    @staticmethod
+    def relpath(path):
+        return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable')
+    def test_importlib_find_spec(self):
+        scan = ansible_mitogen.module_finder._scan_importlib_find_spec
+        actual = scan(self.module_name, self.module_path, self.search_path)
+        self.assertEqual(
+            [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+            [
+                ('ansible.module_utils.external1', 'external1.py', False),
+                ('ansible.module_utils.external2', 'external2.py', False),
+                ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+                ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+            ],
+        )
+
+    @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred')
+    def test_imp_find_module(self):
+        scan = ansible_mitogen.module_finder._scan_imp_find_module
+        actual = scan(self.module_name, self.module_path, self.search_path)
+        self.assertEqual(
+            [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+            [
+                ('ansible.module_utils.external1', 'external1.py', False),
+                ('ansible.module_utils.external2', 'external2.py', False),
+                ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+                ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+            ],
+        )
diff --git a/tests/importer_test.py b/tests/importer_test.py
index e48c02a..e86af8a 100644
--- a/tests/importer_test.py
+++ b/tests/importer_test.py
@@ -2,6 +2,7 @@ import sys
 import threading
 import types
 import zlib
+import unittest
 
 import mock
 
@@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin):
         super(ImporterMixin, self).tearDown()
 
 
+class InvalidNameTest(ImporterMixin, testlib.TestCase):
+    modname = 'trailingdot.'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, None, None, None)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_find_spec_invalid(self):
+        self.set_get_module_response(self.response)
+        self.assertEqual(self.importer.find_spec(self.modname, path=None), None)
+
+
+class MissingModuleTest(ImporterMixin, testlib.TestCase):
+    modname = 'missing'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, None, None, None)
+
+    @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
+    def test_load_module_missing(self):
+        self.set_get_module_response(self.response)
+        self.assertRaises(ImportError, self.importer.load_module, self.modname)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_find_spec_missing(self):
+        """
+        Importer should optimistically offer itself as a module loader
+        when there are no disqualifying criteria.
+        """
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = self.importer.find_spec(self.modname, path=None)
+        self.assertIsInstance(spec, importlib.machinery.ModuleSpec)
+        self.assertEqual(spec.name, self.modname)
+        self.assertEqual(spec.loader, self.importer)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_create_module_missing(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        self.assertRaises(ImportError, self.importer.create_module, spec)
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadModuleTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("data = 1\n\n"))
     path = 'fake_module.py'
@@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
     # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
     response = (modname, None, path, data, [])
 
-    def test_no_such_module(self):
-        self.set_get_module_response(
-            # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-            (self.modname, None, None, None, None)
-        )
-        self.assertRaises(ImportError,
-            lambda: self.importer.load_module(self.modname))
-
     def test_module_added_to_sys_modules(self):
         self.set_get_module_response(self.response)
         mod = self.importer.load_module(self.modname)
@@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
         self.assertIsNone(mod.__package__)
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ModuleSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("data = 1\n\n"))
+    path = 'fake_module.py'
+    modname = 'fake_module'
+
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'fake_module')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("data = 1\n\n"))
     path = 'fake_module.py'
@@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
         self.assertEqual(mod.__package__, 'mypkg')
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class SubmoduleSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("data = 1\n\n"))
+    path = 'fake_module.py'
+    modname = 'mypkg.fake_module'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'mypkg.fake_module')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("func = lambda: 1\n\n"))
     path = 'fake_pkg/__init__.py'
@@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
         self.assertEqual(mod.func.__module__, self.modname)
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class PackageSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("func = lambda: 1\n\n"))
+    path = 'fake_pkg/__init__.py'
+    modname = 'fake_pkg'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, [], path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'fake_pkg')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+    def test_get_filename(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        _ = self.importer.create_module(spec)
+        filename = self.importer.get_filename(self.modname)
+        self.assertEqual('master:fake_pkg/__init__.py', filename)
+
+    def test_get_source(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        _ = self.importer.create_module(spec)
+        source = self.importer.get_source(self.modname)
+        self.assertEqual(source,
+            mitogen.core.to_text(zlib.decompress(self.data)))
+
+
 class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase):
     def initdir(self, caplog):
         self.caplog = caplog
diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index 401a607..8ccbd88 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -139,9 +139,7 @@ class SysModulesMethodTest(testlib.TestCase):
         self.assertIsNone(tup)
 
 
-class GetModuleViaParentEnumerationTest(testlib.TestCase):
-    klass = mitogen.master.ParentEnumerationMethod
-
+class ParentEnumerationMixin(object):
     def call(self, fullname):
         return self.klass().find(fullname)
 
@@ -231,6 +229,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
         self.assertEqual(is_pkg, False)
 
 
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4')
+class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+    klass = mitogen.master.ParentImpEnumerationMethod
+
+
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+    klass = mitogen.master.ParentSpecEnumerationMethod
+
+
 class ResolveRelPathTest(testlib.TestCase):
     klass = mitogen.master.ModuleFinder
 
diff --git a/tests/testlib.py b/tests/testlib.py
index 8ab895c..1146a92 100644
--- a/tests/testlib.py
+++ b/tests/testlib.py
@@ -41,8 +41,13 @@ except NameError:
 
 
 LOG = logging.getLogger(__name__)
-DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
-MODS_DIR = os.path.join(DATA_DIR, 'importer')
+
+TESTS_DIR =                     os.path.join(os.path.dirname(__file__))
+ANSIBLE_LIB_DIR =               os.path.join(TESTS_DIR, 'ansible', 'lib')
+ANSIBLE_MODULE_UTILS_DIR =      os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
+ANSIBLE_MODULES_DIR =           os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules')
+DATA_DIR =                      os.path.join(TESTS_DIR, 'data')
+MODS_DIR =                      os.path.join(TESTS_DIR, 'data', 'importer')
 
 sys.path.append(DATA_DIR)
 sys.path.append(MODS_DIR)