[Import-SIG] PEP 489: Multi-phase extension module initialization; version 5

[Petr Viktorin]

On 05/19/2015 01:06 PM, Petr Viktorin wrote:
> On 05/19/2015 05:51 AM, Nick Coghlan wrote:
>> On 19 May 2015 at 10:07, Eric Snow <ericsnowcurrently at gmail.com> wrote:
>>> On Mon, May 18, 2015 at 8:02 AM, Petr Viktorin <encukou at gmail.com> wrote:
[snip]
>>>>
>>>> The proposal
>>>> ============
>>>
>>> This section should include an indication of how the loader (and
>>> perhaps finder) will change for builtin, frozen, and extension
>>> modules.  It may help to describe the proposal up front by how the
>>> loader implementation would look if it were somehow implemented in
>>> Python code.  The subsequent sections sometimes indicate where
>>> different things take place, but an explicit outline (as Python code)
>>> would make the entire flow really obvious.  Putting that toward the
>>> beginning of this section would help clearly set the stage for the
>>> rest of the proposal.
>>
>> +1 for a pseudo-code overview of the loader implementation.

Here is an overview of how the modified importers will operate.
Details such as logging or handling of errors and invalid states
are left out, and C code is presented with a concise Python-like syntax.

The framework that calls the importers is explained in PEP 451
[#pep-0451-loading]_.

importlib/_bootstrap.py:

    class BuiltinImporter:
        def create_module(self, spec):
            module = _imp.create_builtin(spec)

        def exec_module(self, module):
            _imp.exec_dynamic(module)

        def load_module(self, name):
            # use a backwards compatibility shim
            _load_module_shim(self, name)

importlib/_bootstrap_external.py:

    class ExtensionFileLoader:
        def create_module(self, spec):
            module = _imp.create_dynamic(spec)

        def exec_module(self, module):
            _imp.exec_dynamic(module)

        def load_module(self, name):
            # use a backwards compatibility shim
            _load_module_shim(self, name)

Python/import.c (the _imp module):

    def create_dynamic(spec):
        name = spec.name
        path = spec.origin

        # Find an already loaded module that used single-phase init.
        # For multi-phase initialization, mod is NULL, so a new module
        # is always created.
        mod = _PyImport_FindExtensionObject(name, name)
        if mod:
            return mod

        return _PyImport_LoadDynamicModuleWithSpec(spec)

    def exec_dynamic(module):
        def = PyModule_GetDef(module)
        state = PyModule_GetState(module)
        if state is NULL:
            PyModule_ExecDef(module, def)

    def create_builtin(spec):
        name = spec.name

        # Find an already loaded module that used single-phase init.
        # For multi-phase initialization, mod is NULL, so a new module
        # is always created.
        mod = _PyImport_FindExtensionObject(name, name)
        if mod:
            return mod

        for initname, initfunc in PyImport_Inittab:
            if name == initname:
                m = initfunc()
                if isinstance(m, PyModuleDef):
                    def = m
                    return PyModule_FromDefAndSpec(def, spec)
                else:
                    # fall back to single-phase initialization
                    module = m
                    _PyImport_FixupExtensionObject(module, name, name)
                    return module

Python/importdl.c:

    def _PyImport_LoadDynamicModuleWithSpec(spec):
        path = spec.origin
        package, dot, name = spec.name.rpartition('.')

        # see the "Non-ASCII module names" section for export_hook_name
        hook_name = export_hook_name(name)

        # call platform-specific function for loading exported function
        # from shared library
        exportfunc = _find_shared_funcptr(hook_name, path)

        m = exportfunc()
        if isinstance(m, PyModuleDef):
            def = m
            return PyModule_FromDefAndSpec(def, spec)

        module = m

        # fall back to single-phase initialization
        ....

Objects/moduleobject.c:

    def PyModule_FromDefAndSpec(def, spec):
        name = spec.name
        create = None
        for slot, value in def.m_slots:
            if slot == Py_mod_create:
                create = value
        if create:
            m = create(spec, def)
        else:
            m = PyModule_New(name)

        if isinstance(m, types.ModuleType):
            m.md_state = None
            m.md_def = def

        if def.m_methods:
            PyModule_AddFunctions(m, def.m_methods)
        if def.m_doc:
            PyModule_SetDocString(m, def.m_doc)

    def PyModule_ExecDef(module, def):
        if isinstance(module, types.module_type):
            if module.md_state is NULL:
                # allocate a block of zeroed-out memory
                module.md_state = _alloc(module.md_size)

        if def.m_slots is NULL:
            return

        for slot, value in def.m_slots:
            if slot == Py_mod_exec:
                value(module)