Mailman 3 February 2007 - Python-checkins

r53630 - sandbox/trunk/import_in_py/test_controlled_importlib.py
by brett.cannon Feb. 3, 2007

Feb. 3, 2007

Author: brett.cannon Date: Sat Feb 3 23:09:02 2007 New Revision: 53630 Modified: sandbox/trunk/import_in_py/test_controlled_importlib.py Log: Test that keys starting with a dot in sys.modules and whitelisting of built-in modules works for ControlledImport. Modified: sandbox/trunk/import_in_py/test_controlled_importlib.py ============================================================================== --- sandbox/trunk/import_in_py/test_controlled_importlib.py (original) +++ sandbox/trunk/import_in_py/test_controlled_importlib.py Sat Feb 3 23:09:02 2007 @@ -110,6 +110,7 @@ @contextmanager def mutate_sys_modules(module, name): + """Temporarily mutate sys.modules with a new module.""" try: old_module = sys.modules.get(name) sys.modules[name] = module @@ -120,6 +121,19 @@ else: del sys.modules[name] +@contextmanager +def remove_from_sys_modules(*modules): + """Temporarily remove modules from sys.modules.""" + try: + cached = [] + for name in modules: + cached.append(sys.modules[name]) + del sys.modules[name] + yield + finally: + for name, module in zip(modules, cached): + sys.modules[name] = module + class ControlledImportMethodTests(unittest.TestCase): @@ -153,10 +167,20 @@ def test_block_dot_modules(self): # Modules with a leading dot should not be imported. - pass + import_ = controlled_importlib.ControlledImport() + module = mock_importlib.MockModule() + module_name = '.block' + with mutate_sys_modules(module, module_name): + self.failUnlessRaises(ImportError, import_, module_name, {}, {}, 0) def test_builtin_whitelisting(self): - pass + whitelist = sys.builtin_module_names[0] + blacklist = sys.builtin_module_names[1] + with remove_from_sys_modules(whitelist, blacklist): + import_ = controlled_importlib.ControlledImport([whitelist]) + module = import_(whitelist, {}, {}, 0) + self.failUnlessEqual(module.__name__, whitelist) + self.failUnlessRaises(ImportError, import_, blacklist, {}, {}, 0) def test_frozen_whitelisting(self): pass @@ -175,6 +199,7 @@ def test_no_loader_attribute(self): # No __loader__ attribute should be exposed on any module or package. # XXX check both modules, packages, and submodules. + pass def test_main():

1 0

r53629 - sandbox/trunk/import_in_py/importlib.py sandbox/trunk/import_in_py/test_importlib.py
by brett.cannon Feb. 3, 2007

Feb. 3, 2007

Author: brett.cannon Date: Sat Feb 3 23:08:12 2007 New Revision: 53629 Modified: sandbox/trunk/import_in_py/importlib.py sandbox/trunk/import_in_py/test_importlib.py Log: Make the built-in and frozen module meta_path objects not use classmethods. Modified: sandbox/trunk/import_in_py/importlib.py ============================================================================== --- sandbox/trunk/import_in_py/importlib.py (original) +++ sandbox/trunk/import_in_py/importlib.py Sat Feb 3 23:08:12 2007 @@ -158,22 +158,19 @@ """Base class for meta_path importers for built-in and frozen modules. - Subclasses must provide the _find and _load methods. The methods are - expected to be defined on the subclass itself and not on an instance. + Subclasses must provide the _find and _load methods. """ - @classmethod - def find_module(cls, fullname, path=None): + def find_module(self, fullname, path=None): """See if a built-in or frozen module can be imported based on the specified name.""" - if cls._find(fullname): - return cls + if self._find(fullname): + return self else: return None - @classmethod - def load_module(cls, fullname): + def load_module(self, fullname): """Load a built-in or frozen module. 'imp' code for loading a built-in or frozen module handles the setting @@ -184,7 +181,7 @@ try: return sys.modules[fullname] except KeyError: - mod = cls._load(fullname) + mod = self._load(fullname) if not mod: raise ImportError("expected built-in module not loaded") return mod @@ -605,7 +602,7 @@ """ def __init__(self, default_path_hook=None, - extended_meta_path=(BuiltinImporter, FrozenImporter)): + extended_meta_path=(BuiltinImporter(), FrozenImporter())): """Store a default path hook entry and a sequence to internally extend sys.meta_path by.""" self.extended_meta_path = extended_meta_path Modified: sandbox/trunk/import_in_py/test_importlib.py ============================================================================== --- sandbox/trunk/import_in_py/test_importlib.py (original) +++ sandbox/trunk/import_in_py/test_importlib.py Sat Feb 3 23:08:12 2007 @@ -85,7 +85,7 @@ """Test the built-in module importer.""" - importer = importlib.BuiltinImporter + importer = importlib.BuiltinImporter() module_name = 'sys' bad_module_names = ('tokenize', 'time', '__hello__') @@ -100,7 +100,7 @@ """ - importer = importlib.FrozenImporter + importer = importlib.FrozenImporter() module_name = '__hello__' bad_module_names = ('tokenize', 'time', 'sys')

1 0

r53628 - sandbox/trunk/import_in_py/controlled_importlib.py
by brett.cannon Feb. 3, 2007

Feb. 3, 2007

Author: brett.cannon Date: Sat Feb 3 23:05:21 2007 New Revision: 53628 Modified: sandbox/trunk/import_in_py/controlled_importlib.py Log: Prep the 'sys' module more in ControlledImport.__init__. Also provide default arguments. Modified: sandbox/trunk/import_in_py/controlled_importlib.py ============================================================================== --- sandbox/trunk/import_in_py/controlled_importlib.py (original) +++ sandbox/trunk/import_in_py/controlled_importlib.py Sat Feb 3 23:05:21 2007 @@ -64,14 +64,18 @@ """Represent a controllable version of import that allows for more fine-grained control over what can and cannot be imported.""" - def __init__(self, safe_builtins, safe_frozen, safe_extensions): + def __init__(self, safe_builtins=(), safe_frozen=(), safe_extensions=()): """Set up importation where built-in, frozen, and extension modules must be whitelisted and .pyc files are completely blocked while all. py files are allowed.""" + importlib.Import.__init__(self, (), ()) # Clear out any traces of the previous import machinery. sys.path_importer_cache.clear() + sys.meta_path = [] + sys.path_hooks = [] # Whitelist built-in and frozen modules on sys.meta_path. - # XXX + sys.meta_path.append(WhitelistBuiltin(safe_builtins)) + sys.meta_path.append(WhitelistFrozen(safe_frozen)) # Whitelist extension modules on sys.path. # XXX # Allow all .py files but not .pyc files on sys.path.

1 0

r53626 - sandbox/trunk/import_in_py/test_controlled_importlib.py
by brett.cannon Feb. 3, 2007

Feb. 3, 2007

Author: brett.cannon Date: Sat Feb 3 06:20:26 2007 New Revision: 53626 Modified: sandbox/trunk/import_in_py/test_controlled_importlib.py Log: Note some specifics on what to test when checking for __loader__ removal in integration tests. Modified: sandbox/trunk/import_in_py/test_controlled_importlib.py ============================================================================== --- sandbox/trunk/import_in_py/test_controlled_importlib.py (original) +++ sandbox/trunk/import_in_py/test_controlled_importlib.py Sat Feb 3 06:20:26 2007 @@ -172,6 +172,10 @@ # not whitelisted. pass + def test_no_loader_attribute(self): + # No __loader__ attribute should be exposed on any module or package. + # XXX check both modules, packages, and submodules. + def test_main(): test_support.run_unittest(WhitelistTests,

1 0

r53625 - sandbox/trunk/import_in_py/controlled_importlib.py sandbox/trunk/import_in_py/importlib.py sandbox/trunk/import_in_py/test_controlled_importlib.py sandbox/trunk/import_in_py/test_importlib.py
by brett.cannon Feb. 3, 2007

Feb. 3, 2007

Author: brett.cannon Date: Sat Feb 3 06:18:40 2007 New Revision: 53625 Modified: sandbox/trunk/import_in_py/controlled_importlib.py sandbox/trunk/import_in_py/importlib.py sandbox/trunk/import_in_py/test_controlled_importlib.py sandbox/trunk/import_in_py/test_importlib.py Log: Add a post-import method. This allows for the stripping of __loader__ from newly imported modules so that the global namespace of the loader is not exposed. Also add tests for module_from_cache. Modified: sandbox/trunk/import_in_py/controlled_importlib.py ============================================================================== --- sandbox/trunk/import_in_py/controlled_importlib.py (original) +++ sandbox/trunk/import_in_py/controlled_importlib.py Sat Feb 3 06:18:40 2007 @@ -84,3 +84,12 @@ raise ImportError("module names starting with '.' cannot be " "imported") return importlib.Import.module_from_cache(self, name) + + def post_import(self, module): + """Strip off the __loader__ attribute so that the object's global + namespace is not exposed.""" + try: + del module.__loader__ + except AttributeError: + pass + return module Modified: sandbox/trunk/import_in_py/importlib.py ============================================================================== --- sandbox/trunk/import_in_py/importlib.py (original) +++ sandbox/trunk/import_in_py/importlib.py Sat Feb 3 06:18:40 2007 @@ -702,6 +702,10 @@ return sys.modules[name] else: return False + + def post_import(self, module): + """Perform any desired post-import processing on the module.""" + return module def _import_module(self, name, path=None): """Import the specified module with no handling of parent modules. @@ -725,7 +729,7 @@ loader = self._search_std_path(name, path) # A loader was found. It is the loader's responsibility to have put an # entry in sys.modules. - return loader.load_module(name) + return self.post_import(loader.load_module(name)) def _import_full_module(self, name): """Import a module along with its parent modules and set into Modified: sandbox/trunk/import_in_py/test_controlled_importlib.py ============================================================================== --- sandbox/trunk/import_in_py/test_controlled_importlib.py (original) +++ sandbox/trunk/import_in_py/test_controlled_importlib.py Sat Feb 3 06:18:40 2007 @@ -1,5 +1,7 @@ import controlled_importlib +import mock_importlib +from contextlib import contextmanager import StringIO import sys import unittest @@ -106,11 +108,77 @@ self.failUnlessRaises(ImportError, imp_load.load_module, blacklist) +@contextmanager +def mutate_sys_modules(module, name): + try: + old_module = sys.modules.get(name) + sys.modules[name] = module + yield + finally: + if old_module: + sys.modules[name] = old_module + else: + del sys.modules[name] + + +class ControlledImportMethodTests(unittest.TestCase): + + """Test explicit methods of ControlledImport.""" + + def setUp(self): + self.import_ = controlled_importlib.ControlledImport([], [], []) + + def test_module_from_cache(self): + # Importing of module names with a leading dot should not occur. + module = 'module' + module_name = '.blocked' + assert module_name.startswith('.') + with mutate_sys_modules(module, module_name): + self.failUnlessRaises(ImportError, self.import_.module_from_cache, + module_name) + + def test_post_import(self): + # Any __loader__ attribute should be indiscriminately removed. + module = mock_importlib.MockModule() + self.failUnless(self.import_.post_import(module) is module) + module.__loader__ = None + stripped_module = self.import_.post_import(module) + self.failUnless(stripped_module is module) + self.failUnless(not hasattr(stripped_module, '__loader__')) + + +class ControlledImportUsageTests(unittest.TestCase): + + """Make sure that usage of ControlledImport works properly.""" + + def test_block_dot_modules(self): + # Modules with a leading dot should not be imported. + pass + + def test_builtin_whitelisting(self): + pass + + def test_frozen_whitelisting(self): + pass + + def test_extension_whitelisting(self): + pass + + def test_pyc_blocking(self): + pass + + def test_py(self): + # XXX try importing something with the same name as a built-in that is + # not whitelisted. + pass + def test_main(): test_support.run_unittest(WhitelistTests, WhitelistBuiltinTests, - WhitelistFrozenTests) + WhitelistFrozenTests, + ControlledImportMethodTests, + ControlledImportUsageTests) if __name__ == '__main__': Modified: sandbox/trunk/import_in_py/test_importlib.py ============================================================================== --- sandbox/trunk/import_in_py/test_importlib.py (original) +++ sandbox/trunk/import_in_py/test_importlib.py Sat Feb 3 06:18:40 2007 @@ -1152,6 +1152,12 @@ returned = self.importer.module_from_cache(mod_name + 'asdfeddf') self.failUnless(returned is False) + def test_post_import(self): + # Post-import processing should do nothing but return the module + # unscathed. + module = "mod" + self.failUnless(self.importer.post_import(module) is module) + class ImportMetaPathTests(ImportHelper):

1 0

buildbot warnings in x86 XP trunk
by buildbot＠python.org Feb. 2, 2007

Feb. 2, 2007

The Buildbot has detected a new failure of x86 XP trunk. Full details are available at: http://www.python.org/dev/buildbot/all/x86%2520XP%2520trunk/builds/185 Buildbot URL: http://www.python.org/dev/buildbot/all/ Build Reason: Build Source Stamp: [branch trunk] HEAD Blamelist: peter.astrand Build had warnings: warnings failed slave lost sincerely, -The Buildbot

1 0

r53624 - python/trunk/Lib/subprocess.py
by peter.astrand Feb. 2, 2007

Feb. 2, 2007

Author: peter.astrand Date: Fri Feb 2 20:06:36 2007 New Revision: 53624 Modified: python/trunk/Lib/subprocess.py Log: We had several if statements checking the value of a fd. This is unsafe, since valid fds might be zero. We should check for not None instead. Modified: python/trunk/Lib/subprocess.py ============================================================================== --- python/trunk/Lib/subprocess.py (original) +++ python/trunk/Lib/subprocess.py Fri Feb 2 20:06:36 2007 @@ -592,14 +592,14 @@ c2pread, c2pwrite, errread, errwrite) - if p2cwrite: + if p2cwrite is not None: self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) - if c2pread: + if c2pread is not None: if universal_newlines: self.stdout = os.fdopen(c2pread, 'rU', bufsize) else: self.stdout = os.fdopen(c2pread, 'rb', bufsize) - if errread: + if errread is not None: if universal_newlines: self.stderr = os.fdopen(errread, 'rU', bufsize) else: @@ -986,29 +986,29 @@ # Child try: # Close parent's pipe ends - if p2cwrite: + if p2cwrite is not None: os.close(p2cwrite) - if c2pread: + if c2pread is not None: os.close(c2pread) - if errread: + if errread is not None: os.close(errread) os.close(errpipe_read) # Dup fds for child - if p2cread: + if p2cread is not None: os.dup2(p2cread, 0) - if c2pwrite: + if c2pwrite is not None: os.dup2(c2pwrite, 1) - if errwrite: + if errwrite is not None: os.dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the same # fd more than once, or standard fds. - if p2cread and p2cread not in (0,): + if p2cread is not None and p2cread not in (0,): os.close(p2cread) - if c2pwrite and c2pwrite not in (p2cread, 1): + if c2pwrite is not None and c2pwrite not in (p2cread, 1): os.close(c2pwrite) - if errwrite and errwrite not in (p2cread, c2pwrite, 2): + if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2): os.close(errwrite) # Close all other fds, if asked for @@ -1041,11 +1041,11 @@ # Parent os.close(errpipe_write) - if p2cread and p2cwrite: + if p2cread is not None and p2cwrite is not None: os.close(p2cread) - if c2pwrite and c2pread: + if c2pwrite is not None and c2pread is not None: os.close(c2pwrite) - if errwrite and errread: + if errwrite is not None and errread is not None: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising exception

1 0

buildbot failure in MIPS Debian 2.5
by buildbot＠python.org Feb. 2, 2007

Feb. 2, 2007

The Buildbot has detected a new failure of MIPS Debian 2.5. Full details are available at: http://www.python.org/dev/buildbot/all/MIPS%2520Debian%25202.5/builds/125 Buildbot URL: http://www.python.org/dev/buildbot/all/ Build Reason: The web-page 'force build' button was pressed by 'James': test Build Source Stamp: [branch sytem] net Blamelist: BUILD FAILED: failed svn sincerely, -The Buildbot

1 0

r53623 - sandbox/trunk/import_in_py/controlled_importlib.py sandbox/trunk/import_in_py/test_controlled_importlib.py
by brett.cannon Feb. 2, 2007

Feb. 2, 2007

Author: brett.cannon Date: Fri Feb 2 01:27:13 2007 New Revision: 53623 Added: sandbox/trunk/import_in_py/controlled_importlib.py (contents, props changed) sandbox/trunk/import_in_py/test_controlled_importlib.py (contents, props changed) Log: Begin implementation of a controlling import that usees whitelisting to restrict what can and cannot be imported. Added: sandbox/trunk/import_in_py/controlled_importlib.py ============================================================================== --- (empty file) +++ sandbox/trunk/import_in_py/controlled_importlib.py Fri Feb 2 01:27:13 2007 @@ -0,0 +1,86 @@ +"""Create a controllable import. + +One should be able to do several things to control imports: + + * Whitelisting of modules based on name and type + + built-ins + + frozen + + extensions + * Block all modules based on type + + .pyc + * Allow all modules of a certain type + + .py +""" +import importlib +import sys + +class Whitelister(object): + + """Whitelist the finding and/or loading of modules.""" + + def __init__(self, whitelist, *args, **kwargs): + """Store whitelist and continue on with instantiation.""" + self._whitelist = frozenset(whitelist) + super(Whitelister, self).__init__(*args, **kwargs) + + def check_name(self, name): + """Check a module name against the whitelist, returning True if it + passes or False otherwise.""" + if name in self._whitelist: + return True + return False + + def find_module(self, name, path=None): + """If the module name passes the whitelist, allow the finding of the + modules to continue, otherwise return None.""" + if self.check_name(name): + return super(Whitelister, self).find_module(name, path) + return None + + def load_module(self, name): + """If the module names passes the whitelist, allow the loading of the + module to continue, otherwise raise ImportError.""" + if self.check_name(name): + return super(Whitelister, self).load_module(name) + raise ImportError("cannot import module") + + +class WhitelistBuiltin(Whitelister, importlib.BuiltinImporter): + + """Whitelisting importer/loader for built-in modules.""" + + pass + + +class WhitelistFrozen(Whitelister, importlib.FrozenImporter): + + """Whitelisting importer/loader for frozen modules.""" + + pass + + +class ControlledImport(importlib.Import): + + """Represent a controllable version of import that allows for more + fine-grained control over what can and cannot be imported.""" + + def __init__(self, safe_builtins, safe_frozen, safe_extensions): + """Set up importation where built-in, frozen, and extension modules + must be whitelisted and .pyc files are completely blocked + while all. py files are allowed.""" + # Clear out any traces of the previous import machinery. + sys.path_importer_cache.clear() + # Whitelist built-in and frozen modules on sys.meta_path. + # XXX + # Whitelist extension modules on sys.path. + # XXX + # Allow all .py files but not .pyc files on sys.path. + # XXX + + def module_from_cache(self, name): + """Override so that any module name starting with a dot raises + ImportError.""" + if name.startswith('.'): + raise ImportError("module names starting with '.' cannot be " + "imported") + return importlib.Import.module_from_cache(self, name) Added: sandbox/trunk/import_in_py/test_controlled_importlib.py ============================================================================== --- (empty file) +++ sandbox/trunk/import_in_py/test_controlled_importlib.py Fri Feb 2 01:27:13 2007 @@ -0,0 +1,117 @@ +import controlled_importlib + +import StringIO +import sys +import unittest +from test import test_support + + +class DummyImporterLoader(object): + + def find_module(self, name, path=None): + return True + + def load_module(self, name): + return True + +class DummyWhitelist(controlled_importlib.Whitelister, DummyImporterLoader): + + pass + + +class WhitelistTests(unittest.TestCase): + + """Test the general whitelisting mechanism.""" + + def test_whitelist_module(self): + # A direct module name should be allowed. + whitelist = 'ok_mod' + imp_load = DummyWhitelist([whitelist]) + self.failUnless(imp_load.find_module(whitelist) is True) + self.failUnless(imp_load.find_module('fdssdf') is None) + self.failUnless(imp_load.load_module(whitelist) is True) + self.failUnlessRaises(ImportError, imp_load.load_module, 'asdfadsf') + + def test_whitelist_list(self): + # When the whitelist is a list it should be able to properly whitelist + # no matter where a module is listed. + whitelist1 = 'A' + whitelist2 = 'B' + imp_load = DummyWhitelist([whitelist1, whitelist2]) + for whitelist in (whitelist1, whitelist2): + self.failUnless(imp_load.find_module(whitelist) is True) + self.failUnless(imp_load.load_module(whitelist) is True) + + def test_block_partial_name(self): + # A module that happens to be a prefix of a whitelisted module or has a + # whitelisted module as a prefix should still be blocked. + whitelist = 'mod' + imp_load = DummyWhitelist([whitelist]) + # Module has a whitelisted module as a prefix. + self.failUnless(imp_load.find_module(whitelist+'2') is None) + self.failUnlessRaises(ImportError, imp_load.load_module, whitelist+'2') + # Module is a prefix of a whitelisted module. + self.failUnless(imp_load.find_module(whitelist[:-1]) is None) + self.failUnlessRaises(ImportError, imp_load.load_module, + whitelist[:-1]) + + def test_package(self): + # Whitelisting a package does not automatically allow the submodules. + whitelist = 'pkg' + imp_load = DummyWhitelist([whitelist]) + mod = whitelist + '.' + 'mod' + self.failUnless(imp_load.find_module(mod) is None) + self.failUnlessRaises(ImportError, imp_load.load_module, mod) + + +class WhitelistBuiltinTests(unittest.TestCase): + + """Test the whitelisting support for built-in modules.""" + + def setUp(self): + self.whitelist = sys.builtin_module_names[0] + self.blacklist = sys.builtin_module_names[1] + + def test_whitelist(self): + # Only modules on the whitelist should be allowed to be imported. + # Everything else should return None. + imp_load = controlled_importlib.WhitelistBuiltin([self.whitelist]) + # Importer + self.failUnless(imp_load.find_module(self.whitelist) is not None) + self.failUnless(imp_load.find_module(self.blacklist) is None) + # Loader + self.failUnless(imp_load.load_module(self.whitelist)) + self.failUnlessRaises(ImportError, imp_load.load_module, + self.blacklist) + + +class WhitelistFrozenTests(unittest.TestCase): + + """Test whitelisting of frozen modules.""" + + def setUp(self): + sys.stdout = StringIO.StringIO() + self.whitelist = '__phello__' + self.blacklist = ('__hello__', '__phello__.spam') + + def tearDown(self): + sys.stdout = sys.__stdout__ + + def test_whitelist(self): + imp_load = controlled_importlib.WhitelistFrozen([self.whitelist]) + self.failUnless(imp_load.find_module(self.whitelist) is not None) + self.failUnless(imp_load.load_module(self.whitelist)) + for blacklist in self.blacklist: + self.failUnless(imp_load.find_module(blacklist) is None) + self.failUnlessRaises(ImportError, imp_load.load_module, blacklist) + + + +def test_main(): + test_support.run_unittest(WhitelistTests, + WhitelistBuiltinTests, + WhitelistFrozenTests) + + +if __name__ == '__main__': + test_main()

1 0

buildbot warnings in x86 Ubuntu edgy (icc) trunk
by buildbot＠python.org Feb. 1, 2007

Feb. 1, 2007

The Buildbot has detected a new failure of x86 Ubuntu edgy (icc) trunk. Full details are available at: http://www.python.org/dev/buildbot/all/x86%2520Ubuntu%2520edgy%2520%2528icc… Buildbot URL: http://www.python.org/dev/buildbot/all/ Build Reason: Build Source Stamp: [branch trunk] HEAD Blamelist: raymond.hettinger Build had warnings: warnings test Excerpt from the test logfile: 1 test failed: test_timeout sincerely, -The Buildbot

1 0