[Python-checkins] bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)
Victor Stinner
webhook-mailer at python.org
Tue Sep 24 20:10:39 EDT 2019
https://github.com/python/cpython/commit/52ad33abbfb6637d74932617c7013bae0ccf6e32
commit: 52ad33abbfb6637d74932617c7013bae0ccf6e32
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-09-25T02:10:35+02:00
summary:
bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)
Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.
Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.
files:
M Lib/sysconfig.py
M Lib/test/test_embed.py
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
index e76e6927cb1f..b9e2fafbc084 100644
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -412,7 +412,7 @@ def _generate_posix_vars():
pprint.pprint(vars, stream=f)
# Create file used for sys.path fixup -- see Modules/getpath.c
- with open('pybuilddir.txt', 'w', encoding='ascii') as f:
+ with open('pybuilddir.txt', 'w', encoding='utf8') as f:
f.write(pybuilddir)
def _init_posix(vars):
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 7808a1a2cf6a..e7a10b6defea 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -3,11 +3,14 @@
import unittest
from collections import namedtuple
+import contextlib
import json
import os
import re
+import shutil
import subprocess
import sys
+import tempfile
import textwrap
@@ -25,6 +28,12 @@
API_ISOLATED = 3
+def debug_build(program):
+ program = os.path.basename(program)
+ name = os.path.splitext(program)[0]
+ return name.endswith("_d")
+
+
def remove_python_envvars():
env = dict(os.environ)
# Remove PYTHON* environment variables to get deterministic environment
@@ -40,7 +49,7 @@ def setUp(self):
basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
exename = "_testembed"
if MS_WINDOWS:
- ext = ("_d" if "_d" in sys.executable else "") + ".exe"
+ ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
exename += ext
exepath = os.path.dirname(sys.executable)
else:
@@ -58,7 +67,8 @@ def tearDown(self):
os.chdir(self.oldcwd)
def run_embedded_interpreter(self, *args, env=None,
- timeout=None, returncode=0, input=None):
+ timeout=None, returncode=0, input=None,
+ cwd=None):
"""Runs a test in the embedded interpreter"""
cmd = [self.test_exe]
cmd.extend(args)
@@ -72,7 +82,8 @@ def run_embedded_interpreter(self, *args, env=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
- env=env)
+ env=env,
+ cwd=cwd)
try:
(out, err) = p.communicate(input=input, timeout=timeout)
except:
@@ -460,6 +471,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
EXPECTED_CONFIG = None
+ @classmethod
+ def tearDownClass(cls):
+ # clear cache
+ cls.EXPECTED_CONFIG = None
+
def main_xoptions(self, xoptions_list):
xoptions = {}
for opt in xoptions_list:
@@ -490,11 +506,12 @@ def _get_expected_config_impl(self):
args = [sys.executable, '-S', '-c', code]
proc = subprocess.run(args, env=env,
stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
+ stderr=subprocess.PIPE)
if proc.returncode:
raise Exception(f"failed to get the default config: "
f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
stdout = proc.stdout.decode('utf-8')
+ # ignore stderr
try:
return json.loads(stdout)
except json.JSONDecodeError:
@@ -506,8 +523,15 @@ def _get_expected_config(self):
cls.EXPECTED_CONFIG = self._get_expected_config_impl()
# get a copy
- return {key: dict(value)
- for key, value in cls.EXPECTED_CONFIG.items()}
+ configs = {}
+ for config_key, config_value in cls.EXPECTED_CONFIG.items():
+ config = {}
+ for key, value in config_value.items():
+ if isinstance(value, list):
+ value = value.copy()
+ config[key] = value
+ configs[config_key] = config
+ return configs
def get_expected_config(self, expected_preconfig, expected, env, api,
modify_path_cb=None):
@@ -612,7 +636,7 @@ def check_global_config(self, configs):
def check_all_configs(self, testname, expected_config=None,
expected_preconfig=None, modify_path_cb=None, stderr=None,
- *, api, env=None, ignore_stderr=False):
+ *, api, env=None, ignore_stderr=False, cwd=None):
new_env = remove_python_envvars()
if env is not None:
new_env.update(env)
@@ -642,7 +666,8 @@ def check_all_configs(self, testname, expected_config=None,
expected_config, env,
api, modify_path_cb)
- out, err = self.run_embedded_interpreter(testname, env=env)
+ out, err = self.run_embedded_interpreter(testname,
+ env=env, cwd=cwd)
if stderr is None and not expected_config['verbose']:
stderr = ""
if stderr is not None and not ignore_stderr:
@@ -994,6 +1019,48 @@ def test_init_setpath(self):
api=API_COMPAT, env=env,
ignore_stderr=True)
+ def module_search_paths(self, prefix=None, exec_prefix=None):
+ config = self._get_expected_config()
+ if prefix is None:
+ prefix = config['config']['prefix']
+ if exec_prefix is None:
+ exec_prefix = config['config']['prefix']
+ if MS_WINDOWS:
+ return config['config']['module_search_paths']
+ else:
+ ver = sys.version_info
+ return [
+ os.path.join(prefix, 'lib',
+ f'python{ver.major}{ver.minor}.zip'),
+ os.path.join(prefix, 'lib',
+ f'python{ver.major}.{ver.minor}'),
+ os.path.join(exec_prefix, 'lib',
+ f'python{ver.major}.{ver.minor}', 'lib-dynload'),
+ ]
+
+ @contextlib.contextmanager
+ def tmpdir_with_python(self):
+ # Temporary directory with a copy of the Python program
+ with tempfile.TemporaryDirectory() as tmpdir:
+ if MS_WINDOWS:
+ # Copy pythonXY.dll (or pythonXY_d.dll)
+ ver = sys.version_info
+ dll = f'python{ver.major}{ver.minor}'
+ if debug_build(sys.executable):
+ dll += '_d'
+ dll += '.dll'
+ dll = os.path.join(os.path.dirname(self.test_exe), dll)
+ dll_copy = os.path.join(tmpdir, os.path.basename(dll))
+ shutil.copyfile(dll, dll_copy)
+
+ # Copy Python program
+ exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe))
+ shutil.copyfile(self.test_exe, exec_copy)
+ shutil.copystat(self.test_exe, exec_copy)
+ self.test_exe = exec_copy
+
+ yield tmpdir
+
def test_init_setpythonhome(self):
# Test Py_SetPythonHome(home) + PYTHONPATH env var
# + Py_SetProgramName()
@@ -1012,13 +1079,7 @@ def test_init_setpythonhome(self):
prefix = exec_prefix = home
ver = sys.version_info
- if MS_WINDOWS:
- expected_paths = paths
- else:
- expected_paths = [
- os.path.join(prefix, 'lib', f'python{ver.major}{ver.minor}.zip'),
- os.path.join(home, 'lib', f'python{ver.major}.{ver.minor}'),
- os.path.join(home, 'lib', f'python{ver.major}.{ver.minor}/lib-dynload')]
+ expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
config = {
'home': home,
@@ -1033,6 +1094,95 @@ def test_init_setpythonhome(self):
self.check_all_configs("test_init_setpythonhome", config,
api=API_COMPAT, env=env)
+ def copy_paths_by_env(self, config):
+ all_configs = self._get_expected_config()
+ paths = all_configs['config']['module_search_paths']
+ paths_str = os.path.pathsep.join(paths)
+ config['pythonpath_env'] = paths_str
+ env = {'PYTHONPATH': paths_str}
+ return env
+
+ @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt')
+ def test_init_pybuilddir(self):
+ # Test path configuration with pybuilddir.txt configuration file
+
+ with self.tmpdir_with_python() as tmpdir:
+ # pybuilddir.txt is a sub-directory relative to the current
+ # directory (tmpdir)
+ subdir = 'libdir'
+ libdir = os.path.join(tmpdir, subdir)
+ os.mkdir(libdir)
+
+ filename = os.path.join(tmpdir, 'pybuilddir.txt')
+ with open(filename, "w", encoding="utf8") as fp:
+ fp.write(subdir)
+
+ module_search_paths = self.module_search_paths()
+ module_search_paths[-1] = libdir
+
+ executable = self.test_exe
+ config = {
+ 'base_executable': executable,
+ 'executable': executable,
+ 'module_search_paths': module_search_paths,
+ }
+ env = self.copy_paths_by_env(config)
+ self.check_all_configs("test_init_compat_config", config,
+ api=API_COMPAT, env=env,
+ ignore_stderr=True, cwd=tmpdir)
+
+ def test_init_pyvenv_cfg(self):
+ # Test path configuration with pyvenv.cfg configuration file
+
+ with self.tmpdir_with_python() as tmpdir, \
+ tempfile.TemporaryDirectory() as pyvenv_home:
+ ver = sys.version_info
+
+ if not MS_WINDOWS:
+ lib_dynload = os.path.join(pyvenv_home,
+ 'lib',
+ f'python{ver.major}.{ver.minor}',
+ 'lib-dynload')
+ os.makedirs(lib_dynload)
+ else:
+ lib_dynload = os.path.join(pyvenv_home, 'lib')
+ os.makedirs(lib_dynload)
+ # getpathp.c uses Lib\os.py as the LANDMARK
+ shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py'))
+
+ filename = os.path.join(tmpdir, 'pyvenv.cfg')
+ with open(filename, "w", encoding="utf8") as fp:
+ print("home = %s" % pyvenv_home, file=fp)
+ print("include-system-site-packages = false", file=fp)
+
+ paths = self.module_search_paths()
+ if not MS_WINDOWS:
+ paths[-1] = lib_dynload
+ else:
+ for index, path in enumerate(paths):
+ if index == 0:
+ paths[index] = os.path.join(tmpdir, os.path.basename(path))
+ else:
+ paths[index] = os.path.join(pyvenv_home, os.path.basename(path))
+ paths[-1] = pyvenv_home
+
+ executable = self.test_exe
+ exec_prefix = pyvenv_home
+ config = {
+ 'base_exec_prefix': exec_prefix,
+ 'exec_prefix': exec_prefix,
+ 'base_executable': executable,
+ 'executable': executable,
+ 'module_search_paths': paths,
+ }
+ if MS_WINDOWS:
+ config['base_prefix'] = pyvenv_home
+ config['prefix'] = pyvenv_home
+ env = self.copy_paths_by_env(config)
+ self.check_all_configs("test_init_compat_config", config,
+ api=API_COMPAT, env=env,
+ ignore_stderr=True, cwd=tmpdir)
+
class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
def test_open_code_hook(self):
More information about the Python-checkins
mailing list