[Python-checkins] bpo-36610: shutil.copyfile(): use sendfile() on Linux only (GH-13675)

[Giampaolo Rodola]

https://github.com/python/cpython/commit/413d955f8ec88a7183f91d7ad8b0ff7def803de3
commit: 413d955f8ec88a7183f91d7ad8b0ff7def803de3
branch: master
author: Giampaolo Rodola <g.rodola at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-05-30T14:05:41+08:00
summary:

bpo-36610: shutil.copyfile(): use sendfile() on Linux only (GH-13675)

...and avoid using it on Solaris as it can raise EINVAL if offset is equal or bigger than the size of the file

files:
M Doc/library/shutil.rst
M Doc/whatsnew/3.8.rst
M Lib/shutil.py
M Lib/test/test_shutil.py
M Misc/NEWS.d/3.8.0a1.rst

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 4af5a1680608..dcb2a16cff98 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -420,8 +420,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``".
 
 On macOS `fcopyfile`_ is used to copy the file content (not metadata).
 
-On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports
-copies between 2 regular file descriptors :func:`os.sendfile` is used.
+On Linux :func:`os.sendfile` is used.
 
 On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB
 instead of 64 KiB) and a :func:`memoryview`-based variant of
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 5ee9cf07ebaf..98f0c3474f26 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -772,7 +772,7 @@ Optimizations
 
 * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
   :func:`shutil.copytree` and :func:`shutil.move` use platform-specific
-  "fast-copy" syscalls on Linux, macOS and Solaris in order to copy the file
+  "fast-copy" syscalls on Linux and macOS in order to copy the file
   more efficiently.
   "fast-copy" means that the copying operation occurs within the kernel,
   avoiding the use of userspace buffers in Python as in
diff --git a/Lib/shutil.py b/Lib/shutil.py
index 2dfae87c9ce9..dae916b41605 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -50,7 +50,7 @@
     import nt
 
 COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024
-_HAS_SENDFILE = posix and hasattr(os, "sendfile")
+_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux")
 _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile")  # macOS
 
 __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
@@ -111,7 +111,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags):
 def _fastcopy_sendfile(fsrc, fdst):
     """Copy data from one regular mmap-like fd to another by using
     high-performance sendfile(2) syscall.
-    This should work on Linux >= 2.6.33 and Solaris only.
+    This should work on Linux >= 2.6.33 only.
     """
     # Note: copyfileobj() is left alone in order to not introduce any
     # unexpected breakage. Possible risks by using zero-copy calls
@@ -122,7 +122,7 @@ def _fastcopy_sendfile(fsrc, fdst):
     #   GzipFile (which decompresses data), HTTPResponse (which decodes
     #   chunks).
     # - possibly others (e.g. encrypted fs/partition?)
-    global _HAS_SENDFILE
+    global _USE_CP_SENDFILE
     try:
         infd = fsrc.fileno()
         outfd = fdst.fileno()
@@ -152,7 +152,7 @@ def _fastcopy_sendfile(fsrc, fdst):
                 # sendfile() on this platform (probably Linux < 2.6.33)
                 # does not support copies between regular files (only
                 # sockets).
-                _HAS_SENDFILE = False
+                _USE_CP_SENDFILE = False
                 raise _GiveupOnFastCopy(err)
 
             if err.errno == errno.ENOSPC:  # filesystem is full
@@ -260,8 +260,8 @@ def copyfile(src, dst, *, follow_symlinks=True):
                     return dst
                 except _GiveupOnFastCopy:
                     pass
-            # Linux / Solaris
-            elif _HAS_SENDFILE:
+            # Linux
+            elif _USE_CP_SENDFILE:
                 try:
                     _fastcopy_sendfile(fsrc, fdst)
                     return dst
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index eeebb97ff692..208718bb1281 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -2315,7 +2315,7 @@ def test_file2file_not_supported(self):
         # Emulate a case where sendfile() only support file->socket
         # fds. In such a case copyfile() is supposed to skip the
         # fast-copy attempt from then on.
-        assert shutil._HAS_SENDFILE
+        assert shutil._USE_CP_SENDFILE
         try:
             with unittest.mock.patch(
                     self.PATCHPOINT,
@@ -2324,13 +2324,13 @@ def test_file2file_not_supported(self):
                     with self.assertRaises(_GiveupOnFastCopy):
                         shutil._fastcopy_sendfile(src, dst)
                 assert m.called
-            assert not shutil._HAS_SENDFILE
+            assert not shutil._USE_CP_SENDFILE
 
             with unittest.mock.patch(self.PATCHPOINT) as m:
                 shutil.copyfile(TESTFN, TESTFN2)
                 assert not m.called
         finally:
-            shutil._HAS_SENDFILE = True
+            shutil._USE_CP_SENDFILE = True
 
 
 @unittest.skipIf(not MACOS, 'macOS only')
diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst
index 3d5e6336246e..f4b0a0483300 100644
--- a/Misc/NEWS.d/3.8.0a1.rst
+++ b/Misc/NEWS.d/3.8.0a1.rst
@@ -4450,7 +4450,7 @@ data_received() being called before connection_made().
 
 :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
 :func:`shutil.copytree` and :func:`shutil.move` use platform-specific
-fast-copy syscalls on Linux, Solaris and macOS in order to copy the file
+fast-copy syscalls on Linux and macOS in order to copy the file
 more efficiently. On Windows :func:`shutil.copyfile` uses a bigger default
 buffer size (1 MiB instead of 16 KiB) and a :func:`memoryview`-based variant
 of :func:`shutil.copyfileobj` is used. The speedup for copying a 512MiB file