https://github.com/python/cpython/commit/03082a836b707528f885080bda9732d8984... commit: 03082a836b707528f885080bda9732d89849d4e3 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> committer: GitHub <noreply@github.com> date: 2019-01-30T14:14:35-08:00 summary: bpo-35854: Fix EnvBuilder and --symlinks in venv on Windows (GH-11700) (cherry picked from commit a1f9a3332bd4767e47013ea787022f06b6dbcbbd) Co-authored-by: Steve Dower <steve.dower@microsoft.com> files: A Misc/NEWS.d/next/Windows/2019-01-29-15-44-46.bpo-35854.Ww3z19.rst M Doc/library/venv.rst M Doc/using/venv-create.inc M Lib/test/test_venv.py M Lib/venv/__init__.py diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 53f3d2640566..fefc522b45d1 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -114,8 +114,7 @@ creation according to their needs, the :class:`EnvBuilder` class. any existing target directory, before creating the environment. * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the - Python binary (and any necessary DLLs or other binaries, - e.g. ``pythonw.exe``), rather than copying. + Python binary rather than copying. * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing environment with the running Python - for use when that Python has been @@ -181,15 +180,15 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: setup_python(context) - Creates a copy of the Python executable in the environment on POSIX - systems. If a specific executable ``python3.x`` was used, symlinks to - ``python`` and ``python3`` will be created pointing to that executable, - unless files with those names already exist. + Creates a copy or symlink to the Python executable in the environment. + On POSIX systems, if a specific executable ``python3.x`` was used, + symlinks to ``python`` and ``python3`` will be created pointing to that + executable, unless files with those names already exist. .. method:: setup_scripts(context) Installs activation scripts appropriate to the platform into the virtual - environment. On Windows, also installs the ``python[w].exe`` scripts. + environment. .. method:: post_setup(context) @@ -199,8 +198,13 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.7.2 Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries, and so :meth:`setup_python` does nothing - unless running from a build in the source tree. + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. In addition, :class:`EnvBuilder` provides this utility method that can be called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index ba5096abd370..1ba538bec48b 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -70,6 +70,11 @@ The command, if run with ``-h``, will show the available options:: In earlier versions, if the target directory already existed, an error was raised, unless the ``--clear`` or ``--upgrade`` option was provided. +.. note:: + While symlinks are supported on Windows, they are not recommended. Of + particular note is that double-clicking ``python.exe`` in File Explorer + will resolve the symlink eagerly and ignore the virtual environment. + The created ``pyvenv.cfg`` file also includes the ``include-system-site-packages`` key, set to ``true`` if ``venv`` is run with the ``--system-site-packages`` option, ``false`` otherwise. diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 34c2234493bc..6096b9df45bf 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -243,7 +243,6 @@ def test_isolation(self): self.assertIn('include-system-site-packages = %s\n' % s, data) @unittest.skipUnless(can_symlink(), 'Needs symlinks') - @unittest.skipIf(os.name == 'nt', 'Symlinks are never used on Windows') def test_symlinking(self): """ Test symlinking works as expected diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 5438b0d4e508..8f9e3138474a 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -64,11 +64,10 @@ def create(self, env_dir): self.system_site_packages = False self.create_configuration(context) self.setup_python(context) - if not self.upgrade: - self.setup_scripts(context) if self.with_pip: self._setup_pip(context) if not self.upgrade: + self.setup_scripts(context) self.post_setup(context) if true_system_site_packages: # We had set it to False before, now @@ -176,6 +175,23 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): logger.warning('Unable to symlink %r to %r', src, dst) force_copy = True if force_copy: + if os.name == 'nt': + # On Windows, we rewrite symlinks to our base python.exe into + # copies of venvlauncher.exe + basename, ext = os.path.splitext(os.path.basename(src)) + if basename.endswith('_d'): + ext = '_d' + ext + basename = basename[:-2] + if sysconfig.is_python_build(True): + if basename == 'python': + basename = 'venvlauncher' + elif basename == 'pythonw': + basename = 'venvwlauncher' + scripts = os.path.dirname(src) + else: + scripts = os.path.join(os.path.dirname(__file__), "scripts", "nt") + src = os.path.join(scripts, basename + ext) + shutil.copyfile(src, dst) def setup_python(self, context): @@ -202,23 +218,31 @@ def setup_python(self, context): if not os.path.islink(path): os.chmod(path, 0o755) else: - # For normal cases, the venvlauncher will be copied from - # our scripts folder. For builds, we need to copy it - # manually. - if sysconfig.is_python_build(True): - suffix = '.exe' - if context.python_exe.lower().endswith('_d.exe'): - suffix = '_d.exe' - - src = os.path.join(dirname, "venvlauncher" + suffix) - dst = os.path.join(binpath, context.python_exe) - copier(src, dst) + if self.symlinks: + # For symlinking, we need a complete copy of the root directory + # If symlinks fail, you'll get unnecessary copies of files, but + # we assume that if you've opted into symlinks on Windows then + # you know what you're doing. + suffixes = [ + f for f in os.listdir(dirname) if + os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') + ] + if sysconfig.is_python_build(True): + suffixes = [ + f for f in suffixes if + os.path.normcase(f).startswith(('python', 'vcruntime')) + ] + else: + suffixes = ['python.exe', 'python_d.exe', 'pythonw.exe', + 'pythonw_d.exe'] - src = os.path.join(dirname, "venvwlauncher" + suffix) - dst = os.path.join(binpath, "pythonw" + suffix) - copier(src, dst) + for suffix in suffixes: + src = os.path.join(dirname, suffix) + if os.path.exists(src): + copier(src, os.path.join(binpath, suffix)) - # copy init.tcl over + if sysconfig.is_python_build(True): + # copy init.tcl for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: tcldir = os.path.basename(root) @@ -304,6 +328,9 @@ def install_scripts(self, context, path): dirs.remove(d) continue # ignore files in top level for f in files: + if (os.name == 'nt' and f.startswith('python') + and f.endswith(('.exe', '.pdb'))): + continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] if not suffix: diff --git a/Misc/NEWS.d/next/Windows/2019-01-29-15-44-46.bpo-35854.Ww3z19.rst b/Misc/NEWS.d/next/Windows/2019-01-29-15-44-46.bpo-35854.Ww3z19.rst new file mode 100644 index 000000000000..a1c761458d35 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2019-01-29-15-44-46.bpo-35854.Ww3z19.rst @@ -0,0 +1 @@ +Fix EnvBuilder and --symlinks in venv on Windows