NB Sorry for a lengthy reply -- more like notes for myself I guess ;) Short story -- imp.get_suffixes()[0] != get_config_var('SO') on Debian multiarch sid Perspective patch is at the bottom. On Wed, 25 Jul 2012, Stefan Behnel wrote:

...

anything familiar? (I see PY3 fix bf7981fb37b19f08a331c704df8bf25d3b299be5 but it doesn't look relevant for this one, or am I wrong?)

No, I haven't seen this before.

cool -- I am glad then to bring the freshiest stuff to the table ;)

...

ImportError: /tmp/cython_inline_zo_0tz/_cython_inline_d0fe156ce72658e73c2b9a9438fd8d6a.cpython-32mu-x86_64-linux-gnu.so: cannot open shared object file: No such file or directory

That's weird, and I don't think this is Cython's fault. According to the logs, the shared library file that it built is called

_cython_inline_d0fe156ce72658e73c2b9a9438fd8d6a.cpython-32mu.so

i.e. the "cpython-32mu-x86_64-linux-gnu" should be "cpython-32mu".

Thanks for the analysis! Additional suffix comes from $> cc --print-multiarch x86_64-linux-gnu for multiarch Debian support

The module path that Cython tries (and fails) to load the module from is built like this in the code:

""" so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0]

just a note: [0] -- so only the first possible one is considered

So, apparently, imp.get_suffixes() returns an unexpected result here. Could you figure out what exactly that function returns in your Py3.2 build environment?

here it is in my local environment: $> python3 -c 'import imp; print(imp.get_suffixes())' [('.cpython-32mu.so', 'rb', 3), ('module.cpython-32mu.so', 'rb', 3), ('.abi3.so', 'rb', 3), ('module.abi3.so', 'rb', 3), ('.so', 'rb', 3), ('module.so', 'rb', 3), ('.py', 'U', 1), ('.pyc', 'rb', 2)] BUT it is with python 3.2.3-2 while sid (the build environment) was with up-to-date python 3.2.3-3, which according to http://packages.debian.org/changelogs/pool/main/p/python3.2/python3.2_3.2.3-... did: * Lookup extension modules with a multiarch suffix too. updated my local env to match the build env: $> python3 -c 'import imp; print(imp.get_suffixes())' [('.cpython-32mu-x86_64-linux-gnu.so', 'rb', 3), ('.cpython-32mu.so', 'rb', 3), ('module.cpython-32mu-x86_64-linux-gnu.so', 'rb', 3), ('module.cpython-32mu.so', 'rb', 3), ('.abi3.so', 'rb', 3), ('module.abi3.so', 'rb', 3), ('.so', 'rb', 3), ('module.so', 'rb', 3), ('.py', 'U', 1), ('.pyc', 'rb', 2)] so here we get it ;): (gdb) py-down #3 Frame 0x2814760, for file /home/yoh/deb/gits/cython/build/work-dir/Cy3/Cython/Build/Inline.py, line 194, in cython_inline (code='return global_value + 1', get_type=<function at remote 0x260b4b0>, lib_dir='/home/yoh/.tmp/cython_inline_mr_ml9', cython_include_dirs=['.'], force=True, quiet=True, locals={'self': <TestInline(_outcomeForDoCleanups=<_Outcome(unexpectedSuccess=None, errors=[], success=True, expectedFailure=None, skipped=None, failures=[]) at remote 0x297d290>, _testMethodName='test_globals', test_kwds={'lib_dir': '/home/yoh/.tmp/cython_inline_mr_ml9', 'force': True, 'quiet': True}, _cleanups=[], _type_equality_funcs={<type at remote 0x9aac20>: 'assertDictEqual', <type at remote 0x8a54a0>: 'assertMultiLineEqual', <type at remote 0x89fb60>: 'assertTupleEqual', <type at remote 0x89ebc0>: 'assertSetEqual', <type at remote 0x89f0a0>: 'assertSetEqual', <type at remote 0x9a8de0>: 'assertListEqual'}, listing_file=None, _testMethodDoc=None, echo_file=None) at remote 0x260cfb0>}, globals={'CythonTest': <type a...(truncated) module = imp.load_dynamic(module_name, module_path) (gdb) py-print module_name local 'module_name' = '_cython_inline_7bc6bab54c5c46dc2ccbec2dc048aa42' (gdb) py-print module_path local 'module_path' = '/home/yoh/.tmp/cython_inline_mr_ml9/_cython_inline_7bc6bab54c5c46dc2ccbec2dc048aa42.cpython-32dmu-x86_64-linux-gnu.so' while there is only: $> ls -l /home/yoh/.tmp/cython_inline_mr_ml9/_cython_inline_7bc6bab54c5c46dc2ccbec2dc048aa42.cpython-32dmu*.so -rwx------ 1 yoh yoh 49027 Jul 25 14:22 /home/yoh/.tmp/cython_inline_mr_ml9/_cython_inline_7bc6bab54c5c46dc2ccbec2dc048aa42.cpython-32dmu.so* So -- there is a dichotomy between what is the first in the list of imp.get_suffixes() and what is the default extension filename from build_ext which is based on (Pdb) from distutils.sysconfig import get_config_var (Pdb) get_config_var('SO') '.cpython-32dmu.so' used by .get_ext_filename(): (Pdb) print build_extension.get_ext_filename(module_name) '_cython_inline_7bc6bab54c5c46dc2ccbec2dc048aa42.cpython-32dmu.so' So I wonder, wouldn't it be reasonable (i.e. more robust) in cython_inline to instantiate first build_extension and seek full name for the resultant extension from it? That should eliminate any possibility to get different names. e.g. smth like: $> quilt diff --- a/Cython/Build/Inline.py +++ b/Cython/Build/Inline.py @@ -139,8 +139,15 @@ def cython_inline(code, key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() - so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0] - module_path = os.path.join(lib_dir, module_name+so_ext) + dist = Distribution() + # Ensure the build respects distutils configuration by parsing + # the configuration files + config_files = dist.find_config_files() + dist.parse_config_files(config_files) + build_extension = build_ext(dist) + build_extension.finalize_options() + + module_path = os.path.join(lib_dir, build_extension.get_ext_filename(module_name)) if not os.path.exists(lib_dir): os.makedirs(lib_dir) @@ -178,13 +185,6 @@ def __invoke(%(params)s): sources = [pyx_file], include_dirs = c_include_dirs, extra_compile_args = cflags) - dist = Distribution() - # Ensure the build respects distutils configuration by parsing - # the configuration files - config_files = dist.find_config_files() - dist.parse_config_files(config_files) - build_extension = build_ext(dist) - build_extension.finalize_options() build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir -- Yaroslav O. Halchenko Postdoctoral Fellow, Department of Psychological and Brain Sciences Dartmouth College, 419 Moore Hall, Hinman Box 6207, Hanover, NH 03755 Phone: +1 (603) 646-9834 Fax: +1 (603) 646-1419 WWW: http://www.linkedin.com/in/yarik

On Wed, Jul 25, 2012 at 6:12 PM, Yaroslav Halchenko <lists@onerussian.com> wrote:

actually I have not stated alternative variant since I thought it would not be wise to 'waste' memory : just store association between a particular build and target module_name but now I have mentioned that such code is pretty much there ... but incorrect and not used:

$> grep -e '\Wkey\W' -e '^def cython_inline' -e 'code_cache' Cython/Build/Inline.py _code_cache = {} def cython_inline(code, key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() for key, value in literals.items(): module_code = module_code.replace(key, value) _code_cache[key] = module_name

so 1. key in for loop overrides the key tuple identifying the module_path 2. _code_cache is not used anywhere (but does waste memory although probably not much since there is not that many values of key I guess which it would get)

thus I wondered if it is ok to waste memory ( ;) ) should following patch be used? or should _code_cache be removed altogether (or am I missing its role?)

$> quilt diff --- a/Cython/Build/Inline.py +++ b/Cython/Build/Inline.py @@ -138,13 +138,11 @@ def cython_inline(code, arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() - - so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0] - module_path = os.path.join(lib_dir, module_name+so_ext) + module_path = _code_cache.get(module_name, None)

if not os.path.exists(lib_dir): os.makedirs(lib_dir) - if force or not os.path.isfile(module_path): + if force or module_path is None or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') @@ -189,7 +187,9 @@ def __invoke(%(params)s): build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() - _code_cache[key] = module_name + # Should we check if module_path is not None either it still matches? + module_path = os.path.join(lib_dir, build_extension.get_ext_filename(module_name)) + _code_cache[module_name] = module_path

module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names]

Compiled modules can persist between sessions as well. I like your solution of caching the module name, what if we computed the module name iff it wasn't in the cache, then compiled the file iff the .so file didn't exist (with a check that the module name was OK). Alternatively, could we just rename the compiled library to be what we expect if it wasn't already? - Robert

On Wed, Jul 25, 2012 at 7:08 PM, Yaroslav Halchenko <lists@onerussian.com> wrote:

On Wed, 25 Jul 2012, Robert Bradshaw wrote:

...

...

module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names]

...

Compiled modules can persist between sessions as well.

yeah -- figured it down also while working on another version of this trivial patch ;)

...

I like your solution of caching the module name, what if we computed the module name iff it wasn't in the cache, then compiled the file iff the .so file didn't exist (with a check that the module name was OK).

...

Alternatively, could we just rename the compiled library to be what we expect if it wasn't already?

we could do that ... but altogether -- do you really like caching the names? imho it is somewhat wasteful for long-running interactive sessions where people might try different things. Ok -- here is my next version (stop me eventually) which just does what you wanted -- cache the ultimate suffix under assumption that it would not change (also removed unused _code_cache) (it came out a bit longer simply due to me adding helper function _get_build_extension() to avoid duplication):

Looks good. Thanks. File a pull request and we'll merge it in.

--- a/Cython/Build/Inline.py +++ b/Cython/Build/Inline.py @@ -29,8 +29,6 @@ if sys.version_info[0] < 3: else: to_unicode = lambda x: x

-_code_cache = {} -

class AllSymbols(CythonTransform, SkipDeclarations): def __init__(self): @@ -94,6 +92,16 @@ def safe_type(arg, context=None): return '%s.%s' % (base_type.__module__, base_type.__name__) return 'object'

+def _get_build_extension(): + dist = Distribution() + # Ensure the build respects distutils configuration by parsing + # the configuration files + config_files = dist.find_config_files() + dist.parse_config_files(config_files) + build_extension = build_ext(dist) + build_extension.finalize_options() + return build_extension + def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), @@ -139,8 +147,13 @@ def cython_inline(code, key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()

- so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0] - module_path = os.path.join(lib_dir, module_name+so_ext) + build_extension = None + if cython_inline.so_ext is None: + # Figure out and cache current extension suffix + build_extension = _get_build_extension() + cython_inline.so_ext = build_extension.get_ext_filename('') + + module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext)

if not os.path.exists(lib_dir): os.makedirs(lib_dir) @@ -178,23 +191,21 @@ def __invoke(%(params)s): sources = [pyx_file], include_dirs = c_include_dirs, extra_compile_args = cflags) - dist = Distribution() - # Ensure the build respects distutils configuration by parsing - # the configuration files - config_files = dist.find_config_files() - dist.parse_config_files(config_files) - build_extension = build_ext(dist) - build_extension.finalize_options() + if build_extension is None: + build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() - _code_cache[key] = module_name

module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list)

+# Cached suffix used by cython_inline above. None should get +# overridden with actual value upon the first cython_inline invocation +cython_inline.so_ext = None + non_space = re.compile('[^ ]') def strip_common_indent(code): min_indent = None

-- Yaroslav O. Halchenko Postdoctoral Fellow, Department of Psychological and Brain Sciences Dartmouth College, 419 Moore Hall, Hinman Box 6207, Hanover, NH 03755 Phone: +1 (603) 646-9834 Fax: +1 (603) 646-1419 WWW: http://www.linkedin.com/in/yarik _______________________________________________ cython-devel mailing list cython-devel@python.org http://mail.python.org/mailman/listinfo/cython-devel