[pypy-svn] r69947 - in pypy/trunk/pypy: interpreter module/__builtin__ module/__builtin__/test module/_demo module/_demo/test module/oracle

afa at codespeak.net afa at codespeak.net
Mon Dec 7 15:46:29 CET 2009


Author: afa
Date: Mon Dec  7 15:46:29 2009
New Revision: 69947

Added:
   pypy/trunk/pypy/module/_demo/test/   (props changed)
      - copied from r69901, pypy/trunk/pypy/module/_demo/test/
   pypy/trunk/pypy/module/_demo/test/test_import.py
      - copied unchanged from r69901, pypy/trunk/pypy/module/_demo/test/test_import.py
Modified:
   pypy/trunk/pypy/interpreter/baseobjspace.py
   pypy/trunk/pypy/interpreter/mixedmodule.py
   pypy/trunk/pypy/interpreter/module.py
   pypy/trunk/pypy/module/__builtin__/importing.py
   pypy/trunk/pypy/module/__builtin__/test/test_import.py
   pypy/trunk/pypy/module/_demo/__init__.py
   pypy/trunk/pypy/module/oracle/__init__.py
   pypy/trunk/pypy/module/oracle/interp_error.py
Log:
Change the way built-in modules are initialized and imported:
(2nd attempt)

- All built-in modules are inserted into space.builtin_modules
  when they are discovered, but not in sys.modules.

- The end of space initialization calls setup_builtin_modules(),
  which calls mod.setup_after_space_initialization() for every module.
  At this stage, sys.modules contains the "sys" module, and other
  modules imported by mod.setup_after_space_initialization().

- The space and modules are frozen in this state, and translated.

- When pypy-c starts, it calls mod.startup() on built-in modules
  already in sys.modules.

- when importing another builtin module, pypy fetches it from
  space.builtin_modules, adds it to sys.modules, and calls
  its setup() method.

space.getbuiltinmodule(name) is still the main interface to
import-or-get a built-in module.

This speeds up pypy startup, but more importantly, solves bootstraping
issues where for example a higher-level module would "import posix"
before it is fully initialized; when this happens, os.environ is empty!


Modified: pypy/trunk/pypy/interpreter/baseobjspace.py
==============================================================================
--- pypy/trunk/pypy/interpreter/baseobjspace.py	(original)
+++ pypy/trunk/pypy/interpreter/baseobjspace.py	Mon Dec  7 15:46:29 2009
@@ -239,6 +239,8 @@
             config = get_pypy_config(translating=False)
         self.config = config
 
+        self.builtin_modules = {}
+
         # import extra modules for side-effects
         import pypy.interpreter.nestedscope     # register *_DEREF bytecodes
 
@@ -266,28 +268,34 @@
     def startup(self):
         # To be called before using the space
 
-        # Initialize all builtin modules
+        # Initialize already imported builtin modules
         from pypy.interpreter.module import Module
+        w_modules = self.sys.get('modules')
         for w_modname in self.unpackiterable(
                                 self.sys.get('builtin_module_names')):
+            try:
+                w_mod = self.getitem(w_modules, w_modname)
+            except OperationError, e:
+                if e.match(self, self.w_KeyError):
+                    continue
+                raise
             modname = self.str_w(w_modname)
-            mod = self.interpclass_w(self.getbuiltinmodule(modname))
-            if isinstance(mod, Module):
-                import time
+            mod = self.interpclass_w(w_mod)
+            if isinstance(mod, Module) and not mod.startup_called:
                 self.timer.start("startup " + modname)
                 mod.startup(self)
+                mod.startup_called = True
                 self.timer.stop("startup " + modname)
 
+
     def finish(self):
         w_exitfunc = self.sys.getdictvalue(self, 'exitfunc')
         if w_exitfunc is not None:
             self.call_function(w_exitfunc)
         from pypy.interpreter.module import Module
-        for w_modname in self.unpackiterable(
-                                self.sys.get('builtin_module_names')):
-            modname = self.str_w(w_modname)
-            mod = self.interpclass_w(self.getbuiltinmodule(modname))
-            if isinstance(mod, Module):
+        for w_mod in self.builtin_modules.values():
+            mod = self.interpclass_w(w_mod)
+            if isinstance(mod, Module) and mod.startup_called:
                 mod.shutdown(self)
         if self.config.objspace.std.withdictmeasurement:
             from pypy.objspace.std.dictmultiobject import report
@@ -339,14 +347,37 @@
 
         w_name = self.wrap(name)
         w_mod = self.wrap(Module(self, w_name))
-        w_modules = self.sys.get('modules')
-        self.setitem(w_modules, w_name, w_mod)
+        self.builtin_modules[name] = w_mod
         return name
 
     def getbuiltinmodule(self, name):
         w_name = self.wrap(name)
         w_modules = self.sys.get('modules')
-        return self.getitem(w_modules, w_name)
+        try:
+            return self.getitem(w_modules, w_name)
+        except OperationError, e:
+            if not e.match(self, self.w_KeyError):
+                raise
+
+        # If the module is a builtin but not yet imported,
+        # retrieve it and initialize it
+        try:
+            w_mod = self.builtin_modules[name]
+        except KeyError:
+            raise e
+        else:
+            # Add the module to sys.modules
+            self.setitem(w_modules, w_name, w_mod)
+
+            # And initialize it
+            from pypy.interpreter.module import Module
+            mod = self.interpclass_w(w_mod)
+            if isinstance(mod, Module) and not mod.startup_called:
+                self.timer.start("startup " + name)
+                mod.startup(self)
+                mod.startup_called = True
+                self.timer.stop("startup " + name)
+            return w_mod
 
     def get_builtinmodule_to_install(self):
         """NOT_RPYTHON"""
@@ -397,16 +428,15 @@
         w_name = self.wrap('sys')
         self.sys = Module(self, w_name)
         w_modules = self.sys.get('modules')
-        self.setitem(w_modules, w_name, self.wrap(self.sys))
+        self.builtin_modules['sys'] = self.wrap(self.sys)
 
-        self.setitem(w_modules, w_name_exceptions,
-                     self.wrap(self.exceptions_module))
+        self.builtin_modules['exceptions'] = self.wrap(self.exceptions_module)
 
         from pypy.module.__builtin__ import Module
         w_name = self.wrap('__builtin__')
         self.builtin = Module(self, w_name)
         w_builtin = self.wrap(self.builtin)
-        self.setitem(w_modules, w_name, w_builtin)
+        self.builtin_modules['__builtin__'] = self.wrap(w_builtin)
         self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)
 
         bootstrap_modules = ['sys', '__builtin__', 'exceptions']
@@ -480,12 +510,9 @@
 
     def setup_builtin_modules(self):
         "NOT_RPYTHON: only for initializing the space."
-        from pypy.interpreter.module import Module
-        for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')):
-            modname = self.unwrap(w_modname)
-            mod = self.getbuiltinmodule(modname)
-            if isinstance(mod, Module):
-                mod.setup_after_space_initialization()
+        self.getbuiltinmodule('sys')
+        for mod in self.builtin_modules.values():
+            mod.setup_after_space_initialization()
 
     def initialize(self):
         """NOT_RPYTHON: Abstract method that should put some minimal

Modified: pypy/trunk/pypy/interpreter/mixedmodule.py
==============================================================================
--- pypy/trunk/pypy/interpreter/mixedmodule.py	(original)
+++ pypy/trunk/pypy/interpreter/mixedmodule.py	Mon Dec  7 15:46:29 2009
@@ -87,6 +87,7 @@
 
     def _freeze_(self):
         self.getdict()
+        self.startup_called = False
         # hint for the annotator: Modules can hold state, so they are
         # not constant
         return False

Modified: pypy/trunk/pypy/interpreter/module.py
==============================================================================
--- pypy/trunk/pypy/interpreter/module.py	(original)
+++ pypy/trunk/pypy/interpreter/module.py	Mon Dec  7 15:46:29 2009
@@ -16,21 +16,22 @@
         self.w_name = w_name 
         if w_name is not None:
             space.setitem(w_dict, space.new_interned_str('__name__'), w_name) 
+        self.startup_called = False
 
     def setup_after_space_initialization(self):
         """NOT_RPYTHON: to allow built-in modules to do some more setup
         after the space is fully initialized."""
 
     def startup(self, space):
-        """This is called at runtime before the space gets uses to allow
-        the module to do initialization at runtime.
+        """This is called at runtime on import to allow the module to
+        do initialization when it is imported for the first time.
         """
 
     def shutdown(self, space):
         """This is called when the space is shut down, just after
-        sys.exitfunc().
+        sys.exitfunc(), if the module has been imported.
         """
-        
+
     def getdict(self):
         return self.w_dict
 

Modified: pypy/trunk/pypy/module/__builtin__/importing.py
==============================================================================
--- pypy/trunk/pypy/module/__builtin__/importing.py	(original)
+++ pypy/trunk/pypy/module/__builtin__/importing.py	Mon Dec  7 15:46:29 2009
@@ -228,7 +228,11 @@
 
 def _absolute_import(space, modulename, baselevel, w_fromlist, tentative):
     w = space.wrap
-    
+
+    # check the builtin modules
+    if modulename in space.builtin_modules:
+        return space.getbuiltinmodule(modulename)
+
     w_mod = None
     parts = modulename.split('.')
     prefix = []

Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py
==============================================================================
--- pypy/trunk/pypy/module/__builtin__/test/test_import.py	(original)
+++ pypy/trunk/pypy/module/__builtin__/test/test_import.py	Mon Dec  7 15:46:29 2009
@@ -41,7 +41,9 @@
              relative_a = "import a",
              abs_b      = "import b",
              abs_x_y    = "import x.y",
+             abs_sys    = "import sys",
              string     = "inpackage = 1",
+             errno      = "",
              absolute   = "from __future__ import absolute_import\nimport string",
              relative_b = "from __future__ import absolute_import\nfrom . import string",
              relative_c = "from __future__ import absolute_import\nfrom .string import inpackage",
@@ -251,6 +253,17 @@
             import pkg_r.inpkg
         raises(ImportError,imp)
 
+    def test_import_builtin_inpackage(self):
+        def imp():
+            import pkg.sys
+        raises(ImportError,imp)
+
+        import sys, pkg.abs_sys
+        assert pkg.abs_sys.sys is sys
+
+        import errno, pkg.errno
+        assert pkg.errno is not errno
+
     def test_import_Globals_Are_None(self):
         import sys
         m = __import__('sys')

Modified: pypy/trunk/pypy/module/_demo/__init__.py
==============================================================================
--- pypy/trunk/pypy/module/_demo/__init__.py	(original)
+++ pypy/trunk/pypy/module/_demo/__init__.py	Mon Dec  7 15:46:29 2009
@@ -12,3 +12,13 @@
     appleveldefs = {
         'DemoError'        : 'app_demo.DemoError',
     }
+
+    # Used in tests
+    demo_events = []
+    def setup_after_space_initialization(self):
+        Module.demo_events.append('setup')
+    def startup(self, space):
+        Module.demo_events.append('startup')
+    def shutdown(self, space):
+        Module.demo_events.append('shutdown')
+

Modified: pypy/trunk/pypy/module/oracle/__init__.py
==============================================================================
--- pypy/trunk/pypy/module/oracle/__init__.py	(original)
+++ pypy/trunk/pypy/module/oracle/__init__.py	Mon Dec  7 15:46:29 2009
@@ -39,6 +39,7 @@
     def startup(self, space):
         from pypy.module.oracle.interp_error import get
         state = get(space)
+        state.startup(space)
         (state.w_DecimalType,
          state.w_DateTimeType, state.w_DateType, state.w_TimedeltaType,
          ) = space.fixedview(space.appexec([], """():

Modified: pypy/trunk/pypy/module/oracle/interp_error.py
==============================================================================
--- pypy/trunk/pypy/module/oracle/interp_error.py	(original)
+++ pypy/trunk/pypy/module/oracle/interp_error.py	Mon Dec  7 15:46:29 2009
@@ -4,32 +4,47 @@
 from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.error import OperationError
+
 from pypy.module.oracle import roci, config
+from pypy.rlib.unroll import unrolling_iterable
+
+exported_names = unrolling_iterable("""
+    DatabaseError OperationalError InterfaceError ProgrammingError
+    NotSupportedError IntegrityError InternalError DataError
+    Variable Connection""".split())
 
 class State:
     # XXX move to another file
+
     def __init__(self, space):
-        w_module = space.getbuiltinmodule('cx_Oracle')
-        def get(name):
-            return space.getattr(w_module, space.wrap(name))
+        "NOT_RPYTHON"
+        self.variableTypeByPythonType = {}
+        self.w_DecimalType = None
+        self.w_DateTimeType = None
+        self.w_DateType = None
+        self.w_TimedeltaType = None
 
-        self.w_DatabaseError = get('DatabaseError')
-        self.w_OperationalError = get('OperationalError')
-        self.w_InterfaceError = get('InterfaceError')
-        self.w_ProgrammingError = get('ProgrammingError')
-        self.w_NotSupportedError = get('NotSupportedError')
-        self.w_IntegrityError = get('IntegrityError')
-        self.w_InternalError = get('InternalError')
-        self.w_DataError = get('DataError')
-        self.w_Variable = get('Variable')
-        self.w_Connection = get('Connection')
+        for name in exported_names:
+            setattr(self, 'w_' + name, None)
+
+    def startup(self, space):
+        w_module = space.getbuiltinmodule('cx_Oracle')
+        for name in exported_names:
+            setattr(self, 'w_' + name, space.getattr(w_module, space.wrap(name)))
 
         from pypy.module.oracle.interp_variable import all_variable_types
-        self.variableTypeByPythonType = {}
         for varType in all_variable_types:
             w_type = space.gettypeobject(varType.typedef)
             self.variableTypeByPythonType[w_type] = varType
 
+        (self.w_DecimalType,
+         self.w_DateTimeType, self.w_DateType, self.w_TimedeltaType,
+         ) = space.fixedview(space.appexec([], """():
+             import decimal, datetime
+             return (decimal.Decimal,
+                     datetime.datetime, datetime.date, datetime.timedelta)
+        """))
+
 def get(space): 
     return space.fromcache(State) 
 



More information about the Pypy-commit mailing list