Import from database

[Steve Holden]

I'm trying to load module code from a database, which stores for each 
module its full name, code, load date and a Boolean indicating whether 
it's a package or not.

The following simple program:

import dbimp, sys

if __name__ == "__main__":
     dbimp.install()
     #import bsddb.db
     import a.b.c.d
     import bsddb

gives a traceback from its last line. The whole output is

$ python -i test.py
Accepted *db*
found a in db
load_module: a
a loaded: <module 'a' from 'db:a'> pkg: 1
found a.b in db
load_module: a.b
a.b loaded: <module 'a.b' from 'db:a.b'> pkg: 1
found a.b.c in db
load_module: a.b.c
a.b.c loaded: <module 'a.b.c' from 'db:a.b.c'> pkg: 1
found a.b.c.d in db
load_module: a.b.c.d
a.b.c.d loaded: <module 'a.b.c.d' from 'db:a.b.c.d'> pkg: 0
found bsddb in db
load_module: bsddb
found weakref in db
load_module: weakref
weakref loaded: <module 'weakref' from 'db:weakref'> pkg: 0
Traceback (most recent call last):
   File "test.py", line 7, in ?
     import bsddb
   File "/c/steve/Projects/Python/dbimp/dbimp.py", line 49, in load_module
     exec code in module.__dict__
   File "db:bsddb", line 62, in ?
   File "/usr/lib/python2.4/os.py", line 133, in ?
     from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, 
altsep,
ImportError: No module named path
 >>>

It appears that for some reason execution of the code in 
bsddb/__init__.pyc (loaded from the database using my importer) is 
causing the os module to execute again although it has already run and 
been cached in sys.modules.

Any explanations would be welcome. The importer module is shown below (a 
posting with an attachment to avoid code folding didn't appear to make 
it out)

Also if someone can suggest how associate the source file with 
module/package code from the database that would make tracebacks look 
more conventional, though I'd like to retain the indication the module 
was loaded by dbimp somehow.

#
# Import modules from a database
#
import sys, db, marshal, imp, new
conn = db.conn()
curs = conn.cursor()

curs.execute("select modName from module")
impdict = {}
for n in [x[0] for x in curs.fetchall()]:
     impdict[n] = 1


class dbimporter(object):

     def __init__(self, item, *args, **kw):
         if item != "*db*":
             raise ImportError
         print "Accepted", item

     def find_module(self, fullname, path=None):
         #print "find_module:", fullname, "from", path
         if fullname not in impdict:
             #print "Bailed on", fullname
             return None
         else:
             print "found", fullname, "in db"
             return self

     def load_module(self, modname):
         print "load_module:", modname
         if modname in sys.modules:
             return sys.modules[modname]
         curs.execute("select modCode, modPackage from module where 
modName=%s", (modname, ))
         row = curs.fetchone() # should only BE one ...S
         if not row:
             #print modname, "not found in db"
             raise ImportError, "DB module %s not found in modules"
         code, package = row
         code = marshal.loads(code)
         module = new.module(modname)
         sys.modules[modname] = module
         module.__name__ = modname
         exec code in module.__dict__
         module.__file__ = "db:%s" % modname
         module.__loader__ = dbimporter
         if package:
             module.__path__ = sys.path
         exec code in module.__dict__
         print modname, "loaded:", repr(module), "pkg:", package
         return module

def install():
     sys.path_hooks.append(dbimporter)
     sys.path_importer_cache.clear() # probably not necessary
     sys.path.insert(0, "*db*") # probably not needed with a metea-path 
hook?


regards
  Steve
-- 
Steve Holden               http://www.holdenweb.com/
Python Web Programming  http://pydish.holdenweb.com/
Holden Web LLC      +1 703 861 4237  +1 800 494 3119