Mailman 3 May 2013 - Python-checkins

cpython (merge 3.3 -> default): Merge with 3.3
by jason.coombs May 28, 2013

May 28, 2013

http://hg.python.org/cpython/rev/138ddf15b220 changeset: 83944:138ddf15b220 parent: 83942:c5d4c041ab47 parent: 83943:29a2557d693e user: Jason R. Coombs <jaraco(a)jaraco.com> date: Mon May 27 23:26:36 2013 -0400 summary: Merge with 3.3 files: Doc/library/os.rst | 8 +- Lib/test/test_os.py | 44 ++++++++- Modules/posixmodule.c | 129 +++++++++++++++++++++++++++++- 3 files changed, 168 insertions(+), 13 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2024,9 +2024,10 @@ Create a symbolic link pointing to *source* named *link_name*. On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If *target_is_directory* is set to ``True``, - the symlink will be created as a directory symlink, otherwise as a file symlink - (the default). On non-Window platforms, *target_is_directory* is ignored. + morph to the target dynamically. If the target is present, the type of the + symlink will be created to match. Otherwise, the symlink will be created + as a directory if *target_is_directory* is ``True`` or a file symlink (the + default) otherwise. On non-Window platforms, *target_is_directory* is ignored. Symbolic link support was introduced in Windows 6.0 (Vista). :func:`symlink` will raise a :exc:`NotImplementedError` on Windows versions earlier than 6.0. @@ -2042,6 +2043,7 @@ to the administrator level. Either obtaining the privilege or running your application as an administrator are ways to successfully create symlinks. + :exc:`OSError` is raised when the function is called by an unprivileged user. diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -686,13 +686,8 @@ f.write("I'm " + path + " and proud of it. Blame test_os.\n") f.close() if support.can_symlink(): - if os.name == 'nt': - def symlink_to_dir(src, dest): - os.symlink(src, dest, True) - else: - symlink_to_dir = os.symlink - symlink_to_dir(os.path.abspath(t2_path), link_path) - symlink_to_dir('broken', broken_link_path) + os.symlink(os.path.abspath(t2_path), link_path) + symlink_to_dir('broken', broken_link_path, True) sub2_tree = (sub2_path, ["link"], ["broken_link", "tmp3"]) else: sub2_tree = (sub2_path, [], ["tmp3"]) @@ -1516,7 +1511,7 @@ os.remove(self.missing_link) def test_directory_link(self): - os.symlink(self.dirlink_target, self.dirlink, True) + os.symlink(self.dirlink_target, self.dirlink) self.assertTrue(os.path.exists(self.dirlink)) self.assertTrue(os.path.isdir(self.dirlink)) self.assertTrue(os.path.islink(self.dirlink)) @@ -1610,6 +1605,38 @@ shutil.rmtree(level1) +(a)support.skip_unless_symlink +class NonLocalSymlinkTests(unittest.TestCase): + + def setUp(self): + """ + Create this structure: + + base + \___ some_dir + """ + os.makedirs('base/some_dir') + + def tearDown(self): + shutil.rmtree('base') + + def test_directory_link_nonlocal(self): + """ + The symlink target should resolve relative to the link, not relative + to the current directory. + + Then, link base/some_link -> base/some_dir and ensure that some_link + is resolved as a directory. + + In issue13772, it was discovered that directory detection failed if + the symlink target was not specified relative to the current + directory, which was a defect in the implementation. + """ + src = os.path.join('base', 'some_link') + os.symlink('some_dir', src) + assert os.path.isdir(src) + + class FSEncodingTests(unittest.TestCase): def test_nop(self): self.assertEqual(os.fsencode(b'abc\xff'), b'abc\xff') @@ -2243,6 +2270,7 @@ Pep383Tests, Win32KillTests, Win32SymlinkTests, + NonLocalSymlinkTests, FSEncodingTests, DeviceEncodingTests, PidTests, diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6735,6 +6735,124 @@ return (Py_CreateSymbolicLinkW && Py_CreateSymbolicLinkA); } +void _dirnameW(WCHAR *path) { + /* Remove the last portion of the path */ + + WCHAR *ptr; + + /* walk the path from the end until a backslash is encountered */ + for(ptr = path + wcslen(path); ptr != path; ptr--) + { + if(*ptr == *L"\\" || *ptr == *L"/") { + break; + } + } + *ptr = 0; +} + +void _dirnameA(char *path) { + /* Remove the last portion of the path */ + + char *ptr; + + /* walk the path from the end until a backslash is encountered */ + for(ptr = path + strlen(path); ptr != path; ptr--) + { + if(*ptr == '\\' || *ptr == '/') { + break; + } + } + *ptr = 0; +} + +int _is_absW(WCHAR *path) { + /* Is this path absolute? */ + + return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; + +} + +int _is_absA(char *path) { + /* Is this path absolute? */ + + return path[0] == '\\' || path[0] == '/' || path[1] == ':'; + +} + +void _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { + /* join root and rest with a backslash */ + int root_len; + + if(_is_absW(rest)) { + wcscpy(dest_path, rest); + return; + } + + root_len = wcslen(root); + + wcscpy(dest_path, root); + if(root_len) { + dest_path[root_len] = *L"\\"; + root_len += 1; + } + wcscpy(dest_path+root_len, rest); +} + +void _joinA(char *dest_path, const char *root, const char *rest) { + /* join root and rest with a backslash */ + int root_len; + + if(_is_absA(rest)) { + strcpy(dest_path, rest); + return; + } + + root_len = strlen(root); + + strcpy(dest_path, root); + if(root_len) { + dest_path[root_len] = '\\'; + root_len += 1; + } + strcpy(dest_path+root_len, rest); +} + +int _check_dirW(WCHAR *src, WCHAR *dest) +{ + /* Return True if the path at src relative to dest is a directory */ + WIN32_FILE_ATTRIBUTE_DATA src_info; + WCHAR dest_parent[MAX_PATH]; + WCHAR src_resolved[MAX_PATH] = L""; + + /* dest_parent = os.path.dirname(dest) */ + wcscpy(dest_parent, dest); + _dirnameW(dest_parent); + /* src_resolved = os.path.join(dest_parent, src) */ + _joinW(src_resolved, dest_parent, src); + return ( + GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) + && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + ); +} + +int _check_dirA(char *src, char *dest) +{ + /* Return True if the path at src relative to dest is a directory */ + WIN32_FILE_ATTRIBUTE_DATA src_info; + char dest_parent[MAX_PATH]; + char src_resolved[MAX_PATH] = ""; + + /* dest_parent = os.path.dirname(dest) */ + strcpy(dest_parent, dest); + _dirnameW(dest_parent); + /* src_resolved = os.path.join(dest_parent, src) */ + _joinW(src_resolved, dest_parent, src); + return ( + GetFileAttributesExA(src_resolved, GetFileExInfoStandard, &src_info) + && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + ); +} + #endif static PyObject * @@ -6793,13 +6911,20 @@ } #ifdef MS_WINDOWS + Py_BEGIN_ALLOW_THREADS - if (dst.wide) + if (dst.wide) { + /* if src is a directory, ensure target_is_directory==1 */ + target_is_directory |= _check_dirW(src.wide, dst.wide); result = Py_CreateSymbolicLinkW(dst.wide, src.wide, target_is_directory); - else + } + else { + /* if src is a directory, ensure target_is_directory==1 */ + target_is_directory |= _check_dirA(src.narrow, dst.narrow); result = Py_CreateSymbolicLinkA(dst.narrow, src.narrow, target_is_directory); + } Py_END_ALLOW_THREADS if (!result) { -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.3): Issue #13772: Restored directory detection of targets in `os.symlink` on
by jason.coombs May 28, 2013

May 28, 2013

http://hg.python.org/cpython/rev/29a2557d693e changeset: 83943:29a2557d693e branch: 3.3 parent: 83941:24c3e7e08168 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Mon May 27 23:21:28 2013 -0400 summary: Issue #13772: Restored directory detection of targets in `os.symlink` on Windows, which was temporarily removed in Python 3.2.3 due to an incomplete implementation. The implementation now works even if the symlink is created in a location other than the current directory. files: Doc/library/os.rst | 8 +- Lib/test/test_os.py | 44 ++++++++- Misc/NEWS | 5 + Modules/posixmodule.c | 129 +++++++++++++++++++++++++++++- 4 files changed, 173 insertions(+), 13 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2023,9 +2023,10 @@ Create a symbolic link pointing to *source* named *link_name*. On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If *target_is_directory* is set to ``True``, - the symlink will be created as a directory symlink, otherwise as a file symlink - (the default). On non-Window platforms, *target_is_directory* is ignored. + morph to the target dynamically. If the target is present, the type of the + symlink will be created to match. Otherwise, the symlink will be created + as a directory if *target_is_directory* is ``True`` or a file symlink (the + default) otherwise. On non-Window platforms, *target_is_directory* is ignored. Symbolic link support was introduced in Windows 6.0 (Vista). :func:`symlink` will raise a :exc:`NotImplementedError` on Windows versions earlier than 6.0. @@ -2041,6 +2042,7 @@ to the administrator level. Either obtaining the privilege or running your application as an administrator are ways to successfully create symlinks. + :exc:`OSError` is raised when the function is called by an unprivileged user. diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -686,13 +686,8 @@ f.write("I'm " + path + " and proud of it. Blame test_os.\n") f.close() if support.can_symlink(): - if os.name == 'nt': - def symlink_to_dir(src, dest): - os.symlink(src, dest, True) - else: - symlink_to_dir = os.symlink - symlink_to_dir(os.path.abspath(t2_path), link_path) - symlink_to_dir('broken', broken_link_path) + os.symlink(os.path.abspath(t2_path), link_path) + symlink_to_dir('broken', broken_link_path, True) sub2_tree = (sub2_path, ["link"], ["broken_link", "tmp3"]) else: sub2_tree = (sub2_path, [], ["tmp3"]) @@ -1516,7 +1511,7 @@ os.remove(self.missing_link) def test_directory_link(self): - os.symlink(self.dirlink_target, self.dirlink, True) + os.symlink(self.dirlink_target, self.dirlink) self.assertTrue(os.path.exists(self.dirlink)) self.assertTrue(os.path.isdir(self.dirlink)) self.assertTrue(os.path.islink(self.dirlink)) @@ -1610,6 +1605,38 @@ shutil.rmtree(level1) +(a)support.skip_unless_symlink +class NonLocalSymlinkTests(unittest.TestCase): + + def setUp(self): + """ + Create this structure: + + base + \___ some_dir + """ + os.makedirs('base/some_dir') + + def tearDown(self): + shutil.rmtree('base') + + def test_directory_link_nonlocal(self): + """ + The symlink target should resolve relative to the link, not relative + to the current directory. + + Then, link base/some_link -> base/some_dir and ensure that some_link + is resolved as a directory. + + In issue13772, it was discovered that directory detection failed if + the symlink target was not specified relative to the current + directory, which was a defect in the implementation. + """ + src = os.path.join('base', 'some_link') + os.symlink('some_dir', src) + assert os.path.isdir(src) + + class FSEncodingTests(unittest.TestCase): def test_nop(self): self.assertEqual(os.fsencode(b'abc\xff'), b'abc\xff') @@ -2137,6 +2164,7 @@ Pep383Tests, Win32KillTests, Win32SymlinkTests, + NonLocalSymlinkTests, FSEncodingTests, DeviceEncodingTests, PidTests, diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,11 @@ Library ------- +- Issue #13772: Restored directory detection of targets in ``os.symlink`` on + Windows, which was temporarily removed in Python 3.2.3 due to an incomplete + implementation. The implementation now works even if the symlink is created + in a location other than the current directory. + - Issue #16986: ElementTree now correctly parses a string input not only when an internal XML encoding is UTF-8 or US-ASCII. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7200,6 +7200,124 @@ return (Py_CreateSymbolicLinkW && Py_CreateSymbolicLinkA); } +void _dirnameW(WCHAR *path) { + /* Remove the last portion of the path */ + + WCHAR *ptr; + + /* walk the path from the end until a backslash is encountered */ + for(ptr = path + wcslen(path); ptr != path; ptr--) + { + if(*ptr == *L"\\" || *ptr == *L"/") { + break; + } + } + *ptr = 0; +} + +void _dirnameA(char *path) { + /* Remove the last portion of the path */ + + char *ptr; + + /* walk the path from the end until a backslash is encountered */ + for(ptr = path + strlen(path); ptr != path; ptr--) + { + if(*ptr == '\\' || *ptr == '/') { + break; + } + } + *ptr = 0; +} + +int _is_absW(WCHAR *path) { + /* Is this path absolute? */ + + return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; + +} + +int _is_absA(char *path) { + /* Is this path absolute? */ + + return path[0] == '\\' || path[0] == '/' || path[1] == ':'; + +} + +void _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { + /* join root and rest with a backslash */ + int root_len; + + if(_is_absW(rest)) { + wcscpy(dest_path, rest); + return; + } + + root_len = wcslen(root); + + wcscpy(dest_path, root); + if(root_len) { + dest_path[root_len] = *L"\\"; + root_len += 1; + } + wcscpy(dest_path+root_len, rest); +} + +void _joinA(char *dest_path, const char *root, const char *rest) { + /* join root and rest with a backslash */ + int root_len; + + if(_is_absA(rest)) { + strcpy(dest_path, rest); + return; + } + + root_len = strlen(root); + + strcpy(dest_path, root); + if(root_len) { + dest_path[root_len] = '\\'; + root_len += 1; + } + strcpy(dest_path+root_len, rest); +} + +int _check_dirW(WCHAR *src, WCHAR *dest) +{ + /* Return True if the path at src relative to dest is a directory */ + WIN32_FILE_ATTRIBUTE_DATA src_info; + WCHAR dest_parent[MAX_PATH]; + WCHAR src_resolved[MAX_PATH] = L""; + + /* dest_parent = os.path.dirname(dest) */ + wcscpy(dest_parent, dest); + _dirnameW(dest_parent); + /* src_resolved = os.path.join(dest_parent, src) */ + _joinW(src_resolved, dest_parent, src); + return ( + GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) + && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + ); +} + +int _check_dirA(char *src, char *dest) +{ + /* Return True if the path at src relative to dest is a directory */ + WIN32_FILE_ATTRIBUTE_DATA src_info; + char dest_parent[MAX_PATH]; + char src_resolved[MAX_PATH] = ""; + + /* dest_parent = os.path.dirname(dest) */ + strcpy(dest_parent, dest); + _dirnameW(dest_parent); + /* src_resolved = os.path.join(dest_parent, src) */ + _joinW(src_resolved, dest_parent, src); + return ( + GetFileAttributesExA(src_resolved, GetFileExInfoStandard, &src_info) + && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + ); +} + #endif static PyObject * @@ -7256,13 +7374,20 @@ } #ifdef MS_WINDOWS + Py_BEGIN_ALLOW_THREADS - if (dst.wide) + if (dst.wide) { + /* if src is a directory, ensure target_is_directory==1 */ + target_is_directory |= _check_dirW(src.wide, dst.wide); result = Py_CreateSymbolicLinkW(dst.wide, src.wide, target_is_directory); - else + } + else { + /* if src is a directory, ensure target_is_directory==1 */ + target_is_directory |= _check_dirA(src.narrow, dst.narrow); result = Py_CreateSymbolicLinkA(dst.narrow, src.narrow, target_is_directory); + } Py_END_ALLOW_THREADS if (!result) { -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.3): Issue #15392: Create a unittest framework for IDLE.
by terry.reedy May 28, 2013

May 28, 2013

http://hg.python.org/cpython/rev/24c3e7e08168 changeset: 83941:24c3e7e08168 branch: 3.3 parent: 83934:e57c8a90b2df user: Terry Jan Reedy <tjreedy(a)udel.edu> date: Mon May 27 21:32:03 2013 -0400 summary: Issue #15392: Create a unittest framework for IDLE. Preliminary patch by Rajagopalasarma Jayakrishnan. files: Lib/idlelib/CallTips.py | 4 +- Lib/idlelib/PathBrowser.py | 3 +- Lib/idlelib/idle_test/(a)README.txt | 63 +++++++++++ Lib/idlelib/idle_test/__init__.py | 9 + Lib/idlelib/idle_test/test_calltips.py | 11 + Lib/idlelib/idle_test/test_pathbrowser.py | 12 ++ Lib/test/test_idle.py | 14 ++ Misc/ACKS | 1 + Misc/NEWS | 3 + 9 files changed, 118 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py --- a/Lib/idlelib/CallTips.py +++ b/Lib/idlelib/CallTips.py @@ -264,4 +264,6 @@ print("%d of %d tests failed" % (num_fail, num_tests)) if __name__ == '__main__': - main() + #main() + from unittest import main + main('idlelib.idle_test.test_calltips', verbosity=2, exit=False) diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py --- a/Lib/idlelib/PathBrowser.py +++ b/Lib/idlelib/PathBrowser.py @@ -95,4 +95,5 @@ mainloop() if __name__ == "__main__": - main() + from unittest import main + main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/(a)README.txt b/Lib/idlelib/idle_test/(a)README.txt new file mode 100644 --- /dev/null +++ b/Lib/idlelib/idle_test/(a)README.txt @@ -0,0 +1,63 @@ +README FOR IDLE TESTS IN IDLELIB.IDLE_TEST + +The idle directory, idlelib, has over 60 xyz.py files. The idle_test +subdirectory should contain a test_xyy.py for each one. (For test modules, +make 'xyz' lower case.) Each should start with the following cut-paste +template, with the blanks after after '.'. 'as', and '_' filled in. +--- +import unittest +import idlelib. as + +class Test_(unittest.TestCase): + + def test_(self): + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) +--- +Idle tests are run with unittest; do not use regrtest's test_main. + +Once test_xyy is written, the following should go at the end of xyy.py, +with xyz (lowercased) added after 'test_'. +--- +if __name__ == "__main__": + import unittest + unittest.main('idlelib.idle_test.test_', verbosity=2, exit=False) +--- + +In Idle, pressing F5 in an editor window with either xyz.py or test_xyz.py +loaded will then run the test with the version of Python running Idle and +tracebacks will appear in the Shell window. The options are appropriate for +developers running (as opposed to importing) either type of file during +development: verbosity=2 lists all test_y methods; exit=False avoids a +spurious sys.exit traceback when running in Idle. The following command +lines also run test_xyz.py + +python -m idlelib.xyz # With the capitalization of the xyz module +python -m unittest -v idlelib.idle_test.test_xyz + +To run all idle tests either interactively ('>>>', with unittest imported) +or from a command line, use one of the following. + +>>> unittest.main('idlelib.idle_test', verbosity=2, exit=False) +python -m unittest -v idlelib.idle_test +python -m test.test_idle +python -m test test_idle + +The idle tests are 'discovered' in idlelib.idle_test.__init__.load_tests, +which is also imported into test.test_idle. Normally, neither file should be +changed when working on individual test modules. The last command runs runs +unittest indirectly through regrtest. The same happens when the entire test +suite is run with 'python -m test'. So it must work for buildbots to stay green. + +To run an individual Testcase or test method, extend the +dotted name given to unittest on the command line. + +python -m unittest -v idlelib.idle_test.text_xyz.Test_case.test_meth + +To disable test/test_idle.py, there are at least two choices. +a. Comment out 'load_tests' line, no no tests are discovered (simple and safe); +Running no tests passes, so there is no indication that nothing was run. +b.Before that line, make module an unexpected skip for regrtest with +import unittest; raise unittest.SkipTest('skip for buildbots') +When run directly with unittest, this causes a normal exit and traceback. \ No newline at end of file diff --git a/Lib/idlelib/idle_test/__init__.py b/Lib/idlelib/idle_test/__init__.py new file mode 100644 --- /dev/null +++ b/Lib/idlelib/idle_test/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname + +def load_tests(loader, standard_tests, pattern): + this_dir = dirname(__file__) + top_dir = dirname(dirname(this_dir)) + package_tests = loader.discover(start_dir=this_dir, pattern='test*.py', + top_level_dir=top_dir) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py new file mode 100644 --- /dev/null +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -0,0 +1,11 @@ +import unittest +import idlelib.CallTips as ct + +class Test_get_entity(unittest.TestCase): + def test_bad_entity(self): + self.assertIsNone(ct.get_entity('1/0')) + def test_good_entity(self): + self.assertIs(ct.get_entity('int'), int) + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py new file mode 100644 --- /dev/null +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -0,0 +1,12 @@ +import unittest +import idlelib.PathBrowser as PathBrowser + +class PathBrowserTest(unittest.TestCase): + + def test_DirBrowserTreeItem(self): + # Issue16226 - make sure that getting a sublist works + d = PathBrowser.DirBrowserTreeItem('') + d.GetSubList() + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_idle.py @@ -0,0 +1,14 @@ +# Skip test if tkinter wasn't built or idlelib was deleted. +from test.support import import_module +import_module('tkinter') # discard return +itdir = import_module('idlelib.idle_test') + +# Without test_main present, regrtest.runtest_inner (line1219) +# imitates unittest.main by calling +# unittest.TestLoader().loadTestsFromModule(this_module) +# which look for load_tests and uses it if found. +load_tests = itdir.load_tests + +if __name__ == '__main__': + import unittest + unittest.main(verbosity=2, exit=False) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -575,6 +575,7 @@ Bill Janssen Thomas Jarosch Juhana Jauhiainen +Rajagopalasarma Jayakrishnan Zbigniew Jędrzejewski-Szmek Julien Jehannet Drew Jenkins diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -46,6 +46,9 @@ IDLE ---- +- Issue #15392: Create a unittest framework for IDLE. + Rajagopalasarma Jayakrishnan + - Issue #14146: Highlight source line while debugging on Windows. - Issue #17532: Always include Options menu for IDLE on OS X. -- Repository URL: http://hg.python.org/cpython

1 0

cpython: NEWS entry for issue #18072
by brett.cannon May 28, 2013

May 28, 2013

http://hg.python.org/cpython/rev/2ea849fde22b changeset: 83940:2ea849fde22b user: Brett Cannon <brett(a)python.org> date: Mon May 27 21:12:40 2013 -0400 summary: NEWS entry for issue #18072 files: Misc/NEWS | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -100,6 +100,9 @@ Library ------- +- Issue #18072: Implement importlib.abc.InspectLoader.get_code() and + importlib.abc.ExecutionLoader.get_code(). + - Issue #8240: Set the SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER flag on SSL sockets. -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Issue #18072: Implement get_code() for importlib.abc.InspectLoader and
by brett.cannon May 28, 2013

May 28, 2013

http://hg.python.org/cpython/rev/11510db74223 changeset: 83939:11510db74223 user: Brett Cannon <brett(a)python.org> date: Mon May 27 21:11:04 2013 -0400 summary: Issue #18072: Implement get_code() for importlib.abc.InspectLoader and ExecutionLoader. files: Doc/library/importlib.rst | 12 +- Lib/importlib/abc.py | 30 ++++- Lib/test/test_importlib/test_abc.py | 96 +++++++++++++++- 3 files changed, 121 insertions(+), 17 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -318,16 +318,20 @@ .. method:: get_code(fullname) - An abstract method to return the :class:`code` object for a module. - ``None`` is returned if the module does not have a code object + Return the code object for a module. + ``None`` should be returned if the module does not have a code object (e.g. built-in module). :exc:`ImportError` is raised if loader cannot find the requested module. + .. note:: + While the method has a default implementation, it is suggested that + it be overridden if possible for performance. + .. index:: single: universal newlines; importlib.abc.InspectLoader.get_source method .. versionchanged:: 3.4 - Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. + No longer abstract and a concrete implementation is provided. .. method:: get_source(fullname) @@ -410,7 +414,7 @@ .. method:: get_data(path) - Returns the open, binary file for *path*. + Reads *path* as a binary file and returns the bytes from it. .. class:: SourceLoader diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -147,14 +147,18 @@ """ raise ImportError - @abc.abstractmethod def get_code(self, fullname): - """Abstract method which when implemented should return the code object - for the module. The fullname is a str. Returns a types.CodeType. + """Method which returns the code object for the module. - Raises ImportError if the module cannot be found. + The fullname is a str. Returns a types.CodeType if possible, else + returns None if a code object does not make sense + (e.g. built-in module). Raises ImportError if the module cannot be + found. """ - raise ImportError + source = self.get_source(fullname) + if source is None: + return None + return self.source_to_code(source) @abc.abstractmethod def get_source(self, fullname): @@ -194,6 +198,22 @@ """ raise ImportError + def get_code(self, fullname): + """Method to return the code object for fullname. + + Should return None if not applicable (e.g. built-in module). + Raise ImportError if the module cannot be found. + """ + source = self.get_source(fullname) + if source is None: + return None + try: + path = self.get_filename(fullname) + except ImportError: + return self.source_to_code(source) + else: + return self.source_to_code(source, path) + class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -9,6 +9,7 @@ import os import sys import unittest +from unittest import mock from . import util @@ -166,9 +167,6 @@ def is_package(self, fullname): return super().is_package(fullname) - def get_code(self, fullname): - return super().get_code(fullname) - def get_source(self, fullname): return super().get_source(fullname) @@ -181,10 +179,6 @@ with self.assertRaises(ImportError): self.ins.is_package('blah') - def test_get_code(self): - with self.assertRaises(ImportError): - self.ins.get_code('blah') - def test_get_source(self): with self.assertRaises(ImportError): self.ins.get_source('blah') @@ -206,7 +200,7 @@ ##### InspectLoader concrete methods ########################################### -class InspectLoaderConcreteMethodTests(unittest.TestCase): +class InspectLoaderSourceToCodeTests(unittest.TestCase): def source_to_module(self, data, path=None): """Help with source_to_code() tests.""" @@ -248,6 +242,92 @@ self.assertEqual(code.co_filename, '<string>') +class InspectLoaderGetCodeTests(unittest.TestCase): + + def test_get_code(self): + # Test success. + module = imp.new_module('blah') + with mock.patch.object(InspectLoaderSubclass, 'get_source') as mocked: + mocked.return_value = 'attr = 42' + loader = InspectLoaderSubclass() + code = loader.get_code('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + def test_get_code_source_is_None(self): + # If get_source() is None then this should be None. + with mock.patch.object(InspectLoaderSubclass, 'get_source') as mocked: + mocked.return_value = None + loader = InspectLoaderSubclass() + code = loader.get_code('blah') + self.assertIsNone(code) + + def test_get_code_source_not_found(self): + # If there is no source then there is no code object. + loader = InspectLoaderSubclass() + with self.assertRaises(ImportError): + loader.get_code('blah') + + +##### ExecutionLoader concrete methods ######################################### +class ExecutionLoaderGetCodeTests(unittest.TestCase): + + def mock_methods(self, *, get_source=False, get_filename=False): + source_mock_context, filename_mock_context = None, None + if get_source: + source_mock_context = mock.patch.object(ExecutionLoaderSubclass, + 'get_source') + if get_filename: + filename_mock_context = mock.patch.object(ExecutionLoaderSubclass, + 'get_filename') + return source_mock_context, filename_mock_context + + def test_get_code(self): + path = 'blah.py' + source_mock_context, filename_mock_context = self.mock_methods( + get_source=True, get_filename=True) + with source_mock_context as source_mock, filename_mock_context as name_mock: + source_mock.return_value = 'attr = 42' + name_mock.return_value = path + loader = ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertEqual(code.co_filename, path) + module = imp.new_module('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + def test_get_code_source_is_None(self): + # If get_source() is None then this should be None. + source_mock_context, _ = self.mock_methods(get_source=True) + with source_mock_context as mocked: + mocked.return_value = None + loader = ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertIsNone(code) + + def test_get_code_source_not_found(self): + # If there is no source then there is no code object. + loader = ExecutionLoaderSubclass() + with self.assertRaises(ImportError): + loader.get_code('blah') + + def test_get_code_no_path(self): + # If get_filename() raises ImportError then simply skip setting the path + # on the code object. + source_mock_context, filename_mock_context = self.mock_methods( + get_source=True, get_filename=True) + with source_mock_context as source_mock, filename_mock_context as name_mock: + source_mock.return_value = 'attr = 42' + name_mock.side_effect = ImportError + loader = ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertEqual(code.co_filename, '<string>') + module = imp.new_module('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + + ##### SourceLoader concrete methods ############################################ class SourceOnlyLoaderMock(abc.SourceLoader): -- Repository URL: http://hg.python.org/cpython

1 0

cpython: grammar
by benjamin.peterson May 27, 2013

May 27, 2013

http://hg.python.org/cpython/rev/789f8bbf0ae7 changeset: 83938:789f8bbf0ae7 user: Benjamin Peterson <benjamin(a)python.org> date: Mon May 27 14:49:31 2013 -0700 summary: grammar files: Misc/NEWS | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,9 +10,9 @@ Core and Builtins ----------------- -- Issue #17206: Py_CLEAR(), Py_DECREF(), Py_XINCREF() and Py_XDECREF() now - expands their arguments once instead of multiple times. - Patch written by Illia Polosukhin. +- Issue #17206: Py_CLEAR(), Py_DECREF(), Py_XINCREF() and Py_XDECREF() now + expand their arguments once instead of multiple times. Patch written by Illia + Polosukhin. - Issue #17937: Try harder to collect cyclic garbage at shutdown. -- Repository URL: http://hg.python.org/cpython

1 0

cpython: don't expand the operand to Py_XINCREF/XDECREF/CLEAR/DECREF multiple times
by benjamin.peterson May 27, 2013

May 27, 2013

http://hg.python.org/cpython/rev/aff41a6421c2 changeset: 83937:aff41a6421c2 parent: 83935:39e0be9106c1 user: Benjamin Peterson <benjamin(a)python.org> date: Mon May 27 14:46:14 2013 -0700 summary: don't expand the operand to Py_XINCREF/XDECREF/CLEAR/DECREF multiple times (closes #17206) A patch from Illia Polosukhin. files: Include/object.h | 34 +++++++++++-------- Misc/ACKS | 1 + Misc/NEWS | 4 ++ Modules/_testcapimodule.c | 46 +++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/Include/object.h b/Include/object.h --- a/Include/object.h +++ b/Include/object.h @@ -680,12 +680,6 @@ complications in the deallocation function. (This is actually a decision that's up to the implementer of each new type so if you want, you can count such references to the type object.) - -*** WARNING*** The Py_DECREF macro must have a side-effect-free argument -since it may evaluate its argument multiple times. (The alternative -would be to mace it a proper function or assign it to a global temporary -variable first, both of which are slower; and in a multi-threaded -environment the global variable trick is not safe.) */ /* First define a pile of simple helper macros, one set per special @@ -764,15 +758,16 @@ #define Py_INCREF(op) ( \ _Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA \ - ((PyObject*)(op))->ob_refcnt++) + ((PyObject *)(op))->ob_refcnt++) #define Py_DECREF(op) \ do { \ + PyObject *_py_decref_tmp = (PyObject *)(op); \ if (_Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA \ - --((PyObject*)(op))->ob_refcnt != 0) \ - _Py_CHECK_REFCNT(op) \ + --(_py_decref_tmp)->ob_refcnt != 0) \ + _Py_CHECK_REFCNT(_py_decref_tmp) \ else \ - _Py_Dealloc((PyObject *)(op)); \ + _Py_Dealloc(_py_decref_tmp); \ } while (0) /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear @@ -811,16 +806,27 @@ */ #define Py_CLEAR(op) \ do { \ - if (op) { \ - PyObject *_py_tmp = (PyObject *)(op); \ + PyObject *_py_tmp = (PyObject *)(op); \ + if (_py_tmp != NULL) { \ (op) = NULL; \ Py_DECREF(_py_tmp); \ } \ } while (0) /* Macros to use in case the object pointer may be NULL: */ -#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0) -#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0) +#define Py_XINCREF(op) \ + do { \ + PyObject *_py_xincref_tmp = (PyObject *)(op); \ + if (_py_xincref_tmp != NULL) \ + Py_INCREF(_py_xincref_tmp); \ + } while (0) + +#define Py_XDECREF(op) \ + do { \ + PyObject *_py_xdecref_tmp = (PyObject *)(op); \ + if (_py_xdecref_tmp != NULL) \ + Py_DECREF(_py_xdecref_tmp); \ + } while (0) /* These are provided as conveniences to Python runtime embedders, so that diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -983,6 +983,7 @@ Remi Pointel Ariel Poliak Guilherme Polo +Illia Polosukhin Michael Pomraning Iustin Pop Claudiu Popa diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Core and Builtins ----------------- +- Issue #17206: Py_CLEAR(), Py_DECREF(), Py_XINCREF() and Py_XDECREF() now + expands their arguments once instead of multiple times. + Patch written by Illia Polosukhin. + - Issue #17937: Try harder to collect cyclic garbage at shutdown. - Issue #12370: Prevent class bodies from interfering with the __class__ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2468,6 +2468,48 @@ return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec); } +static PyObject * +_test_incref(PyObject *ob) +{ + Py_INCREF(ob); + return ob; +} + +static PyObject * +test_xincref_doesnt_leak(PyObject *ob) +{ + PyObject *obj = PyLong_FromLong(0); + Py_XINCREF(_test_incref(obj)); + Py_DECREF(obj); + Py_DECREF(obj); + Py_DECREF(obj); + Py_RETURN_NONE; +} + +static PyObject * +test_incref_doesnt_leak(PyObject *ob) +{ + PyObject *obj = PyLong_FromLong(0); + Py_INCREF(_test_incref(obj)); + Py_DECREF(obj); + Py_DECREF(obj); + Py_DECREF(obj); + Py_RETURN_NONE; +} + +static PyObject * +test_xdecref_doesnt_leak(PyObject *ob) +{ + Py_XDECREF(PyLong_FromLong(0)); + Py_RETURN_NONE; +} + +static PyObject * +test_decref_doesnt_leak(PyObject *ob) +{ + Py_DECREF(PyLong_FromLong(0)); + Py_RETURN_NONE; +} static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -2478,6 +2520,10 @@ {"test_dict_iteration", (PyCFunction)test_dict_iteration,METH_NOARGS}, {"test_lazy_hash_inheritance", (PyCFunction)test_lazy_hash_inheritance,METH_NOARGS}, {"test_long_api", (PyCFunction)test_long_api, METH_NOARGS}, + {"test_xincref_doesnt_leak",(PyCFunction)test_xincref_doesnt_leak, METH_NOARGS}, + {"test_incref_doesnt_leak", (PyCFunction)test_incref_doesnt_leak, METH_NOARGS}, + {"test_xdecref_doesnt_leak",(PyCFunction)test_xdecref_doesnt_leak, METH_NOARGS}, + {"test_decref_doesnt_leak", (PyCFunction)test_decref_doesnt_leak, METH_NOARGS}, {"test_long_and_overflow", (PyCFunction)test_long_and_overflow, METH_NOARGS}, {"test_long_as_double", (PyCFunction)test_long_as_double,METH_NOARGS}, -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): Issue #18015: Fix unpickling of 2.7.3 and 2.7.4 namedtuples.
by raymond.hettinger May 27, 2013

May 27, 2013

http://hg.python.org/cpython/rev/687295c6c8f2 changeset: 83936:687295c6c8f2 branch: 2.7 parent: 83906:f4981d8eb401 user: Raymond Hettinger <python(a)rcn.com> date: Mon May 27 10:58:55 2013 -0700 summary: Issue #18015: Fix unpickling of 2.7.3 and 2.7.4 namedtuples. files: Doc/library/collections.rst | 6 +++ Lib/collections.py | 6 +++ Lib/test/test_collections.py | 44 +++++++++++++++++++++++- Misc/NEWS | 2 + 4 files changed, 57 insertions(+), 1 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -639,6 +639,12 @@ 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) <BLANKLINE> + __dict__ = _property(_asdict) + <BLANKLINE> + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + pass + <BLANKLINE> x = _property(_itemgetter(0), doc='Alias for field number 0') <BLANKLINE> y = _property(_itemgetter(1), doc='Alias for field number 1') diff --git a/Lib/collections.py b/Lib/collections.py --- a/Lib/collections.py +++ b/Lib/collections.py @@ -270,6 +270,12 @@ 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + __dict__ = _property(_asdict) + + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + pass + {field_defs} ''' diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -17,6 +17,43 @@ TestNT = namedtuple('TestNT', 'x y z') # type used for pickle tests +py273_named_tuple_pickle = '''\ +ccopy_reg +_reconstructor +p0 +(ctest.test_collections +TestNT +p1 +c__builtin__ +tuple +p2 +(I10 +I20 +I30 +tp3 +tp4 +Rp5 +ccollections +OrderedDict +p6 +((lp7 +(lp8 +S'x' +p9 +aI10 +aa(lp10 +S'y' +p11 +aI20 +aa(lp12 +S'z' +p13 +aI30 +aatp14 +Rp15 +b. +''' + class TestNamedTuple(unittest.TestCase): def test_factory(self): @@ -78,12 +115,12 @@ self.assertRaises(TypeError, eval, 'Point(XXX=1, y=2)', locals()) # wrong keyword argument self.assertRaises(TypeError, eval, 'Point(x=1)', locals()) # missing keyword argument self.assertEqual(repr(p), 'Point(x=11, y=22)') - self.assertNotIn('__dict__', dir(p)) # verify instance has no dict self.assertNotIn('__weakref__', dir(p)) self.assertEqual(p, Point._make([11, 22])) # test _make classmethod self.assertEqual(p._fields, ('x', 'y')) # test _fields attribute self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method + self.assertEqual(vars(p), p._asdict()) # verify that vars() works try: p._replace(x=1, error=2) @@ -215,6 +252,11 @@ # test __getnewargs__ self.assertEqual(t.__getnewargs__(), values) + def test_pickling_bug_18015(self): + # http://bugs.python.org/issue18015 + pt = pickle.loads(py273_named_tuple_pickle) + self.assertEqual(pt.x, 10) + class ABCTestCase(unittest.TestCase): def validate_abstract_methods(self, abc, *names): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,8 @@ - Issue #17981: Closed socket on error in SysLogHandler. +- Issue #18015: Fix unpickling of 2.7.3 and 2.7.4 namedtuples. + - Issue #17754: Make ctypes.util.find_library() independent of the locale. - Fix typos in the multiprocessing module. -- Repository URL: http://hg.python.org/cpython

1 0

peps: Make consistent use of the term "implementation" versus "overload"
by lukasz.langa May 27, 2013

May 27, 2013

http://hg.python.org/peps/rev/2f9702702b7c changeset: 4915:2f9702702b7c user: Łukasz Langa <lukasz(a)langa.pl> date: Mon May 27 14:23:30 2013 +0200 summary: Make consistent use of the term "implementation" versus "overload" files: pep-0443.txt | 72 ++++++++++++++++++++++----------------- 1 files changed, 41 insertions(+), 31 deletions(-) diff --git a/pep-0443.txt b/pep-0443.txt --- a/pep-0443.txt +++ b/pep-0443.txt @@ -19,10 +19,11 @@ module that provides a simple form of generic programming known as single-dispatch generic functions. -A **generic function** is composed of multiple functions sharing the -same name. Which form should be used during a call is determined by the -dispatch algorithm. When the implementation is chosen based on the type -of a single argument, this is known as **single dispatch**. +A **generic function** is composed of multiple functions implementing +the same operation for different types. Which implementation should be +used during a call is determined by the dispatch algorithm. When the +implementation is chosen based on the type of a single argument, this is +known as **single dispatch**. Rationale and Goals @@ -72,8 +73,9 @@ ... print(arg) To add overloaded implementations to the function, use the -``register()`` attribute of the generic function. It takes a type -parameter:: +``register()`` attribute of the generic function. It is a decorator, +taking a type parameter and decorating a function implementing the +operation for that type:: >>> @fun.register(int) ... def _(arg, verbose=False): @@ -110,7 +112,8 @@ >>> fun_num is fun False -When called, the generic function dispatches on the first argument:: +When called, the generic function dispatches on the type of the first +argument:: >>> fun("Hello, world.") Hello, world. @@ -129,15 +132,17 @@ >>> fun(1.23) 0.615 -To get the implementation for a specific type, use the ``dispatch()`` -attribute:: +Where there is no registered implementation for a specific type, its +method resolution order is used to find a more generic implementation. +To check which implementation will the generic function choose for +a given type, use the ``dispatch()`` attribute:: >>> fun.dispatch(float) <function fun_num at 0x104319058> >>> fun.dispatch(dict) <function fun at 0x103fe4788> -To access all registered overloads, use the read-only ``registry`` +To access all registered implementations, use the read-only ``registry`` attribute:: >>> fun.registry.keys() @@ -180,9 +185,10 @@ handling of old-style classes and Zope's ExtensionClasses. More importantly, it introduces support for Abstract Base Classes (ABC). -When a generic function overload is registered for an ABC, the dispatch -algorithm switches to a mode of MRO calculation for the provided -argument which includes the relevant ABCs. The algorithm is as follows:: +When a generic function implementation is registered for an ABC, the +dispatch algorithm switches to a mode of MRO calculation for the +provided argument which includes the relevant ABCs. The algorithm is as +follows:: def _compose_mro(cls, haystack): """Calculates the MRO for a given class `cls`, including relevant @@ -218,9 +224,10 @@ While this mode of operation is significantly slower, all dispatch decisions are cached. The cache is invalidated on registering new -overloads on the generic function or when user code calls ``register()`` -on an ABC to register a new virtual subclass. In the latter case, it is -possible to create a situation with ambiguous dispatch, for instance:: +implementations on the generic function or when user code calls +``register()`` on an ABC to register a new virtual subclass. In the +latter case, it is possible to create a situation with ambiguous +dispatch, for instance:: >>> from collections import Iterable, Container >>> class P: @@ -274,24 +281,27 @@ sense that we need not expect people to randomly redefine the behavior of existing functions in unpredictable ways. To the contrary, generic function usage in actual programs tends to follow very predictable -patterns and overloads are highly-discoverable in the common case. +patterns and registered implementations are highly-discoverable in the +common case. If a module is defining a new generic operation, it will usually also -define any required overloads for existing types in the same place. -Likewise, if a module is defining a new type, then it will usually -define overloads there for any generic functions that it knows or cares -about. As a result, the vast majority of overloads can be found adjacent -to either the function being overloaded, or to a newly-defined type for -which the overload is adding support. +define any required implementations for existing types in the same +place. Likewise, if a module is defining a new type, then it will +usually define implementations there for any generic functions that it +knows or cares about. As a result, the vast majority of registered +implementations can be found adjacent to either the function being +overloaded, or to a newly-defined type for which the implementation is +adding support. -It is only in rather infrequent cases that one will have overloads in -a module that contains neither the function nor the type(s) for which -the overload is added. In the absence of incompetence or deliberate -intention to be obscure, the few overloads that are not adjacent to the -relevant type(s) or function(s), will generally not need to be -understood or known about outside the scope where those overloads are -defined. (Except in the "support modules" case, where best practice -suggests naming them accordingly.) +It is only in rather infrequent cases that one will have implementations +registered in a module that contains neither the function nor the +type(s) for which the implementation is added. In the absence of +incompetence or deliberate intention to be obscure, the few +implementations that are not registered adjacent to the relevant type(s) +or function(s), will generally not need to be understood or known about +outside the scope where those implementations are defined. (Except in +the "support modules" case, where best practice suggests naming them +accordingly.) As mentioned earlier, single-dispatch generics are already prolific throughout the standard library. A clean, standard way of doing them -- Repository URL: http://hg.python.org/peps

1 0

peps: Publish the JSON-based metadata 2.0 spec
by nick.coghlan May 27, 2013

May 27, 2013

http://hg.python.org/peps/rev/6b4a11ae1c10 changeset: 4914:6b4a11ae1c10 user: Nick Coghlan <ncoghlan(a)gmail.com> date: Mon May 27 21:20:13 2013 +1000 summary: Publish the JSON-based metadata 2.0 spec files: pep-0426.txt | 3316 ++++++++++++++++++++----------------- pep-0440.txt | 995 +++++----- 2 files changed, 2304 insertions(+), 2007 deletions(-) diff --git a/pep-0426.txt b/pep-0426.txt --- a/pep-0426.txt +++ b/pep-0426.txt @@ -2,9 +2,9 @@ Title: Metadata for Python Software Packages 2.0 Version: $Revision$ Last-Modified: $Date$ -Author: Daniel Holth <dholth(a)gmail.com>, - Donald Stufft <donald(a)stufft.io>, - Nick Coghlan <ncoghlan(a)gmail.com> +Author: Nick Coghlan <ncoghlan(a)gmail.com>, + Daniel Holth <dholth(a)gmail.com>, + Donald Stufft <donald(a)stufft.io> BDFL-Delegate: Nick Coghlan <ncoghlan(a)gmail.com> Discussions-To: Distutils SIG <distutils-sig(a)python.org> Status: Draft @@ -12,15 +12,16 @@ Content-Type: text/x-rst Requires: 440 Created: 30 Aug 2012 -Post-History: 14 Nov 2012, 5 Feb 2013, 7 Feb 2013, 9 Feb 2013 +Post-History: 14 Nov 2012, 5 Feb 2013, 7 Feb 2013, 9 Feb 2013, 27-May-2013 Replaces: 345 Abstract ======== -This PEP describes a mechanism for adding metadata to Python distributions. -It includes specifics of the field names, and their semantics and +This PEP describes a mechanism for publishing and exchanging metadata +related to Python distributions. It includes specifics of the field names, +and their semantics and usage. This document specifies version 2.0 of the metadata format. @@ -28,89 +29,351 @@ Version 1.1 is specified in PEP 314. Version 1.2 is specified in PEP 345. -Version 2.0 of the metadata format adds fields designed to make -third-party packaging of Python Software easier and defines a formal -extension mechanism. It also adds support for optional features of -distributions and allows the description to be placed into a payload -section. Finally, this version addresses several issues with the -previous iteration of the standard version identification scheme. +Version 2.0 of the metadata format migrates from a custom key-value format +to a JSON-compatible in-memory representation. + +This version also adds fields designed to make third-party packaging of +Python software easier, defines a formal extension mechanism, and adds +support for optional dependencies. Finally, this version addresses +several issues with the previous iteration of the standard version +identification scheme. .. note:: - Portions of this PEP are in the process of being broken out into a - separate PEP (PEP 440). Old versions of much of that material currently - still exists in this PEP - the duplication will be eliminated in the - next major update. + "I" in this doc refers to Nick Coghlan. Daniel and Donald either wrote or + contributed to earlier versions, and have been providing feedback as this + initial draft of the JSON-based rewrite has taken shape. + + Metadata 2.0 represents a major upgrade to the Python packaging ecosystem, + and attempts to incorporate experience gained over the 15 years(!) since + distutils was first added to the standard library. Some of that is just + incorporating existing practices from setuptools/pip/etc, some of it is + copying from other distribution systems (like Linux distros or other + development language communities) and some of it is attempting to solve + problems which haven't yet been well solved by anyone (like supporting + clean conversion of Python source packages to distro policy compliant + source packages for at least Debian and Fedora, and perhaps other + platform specific distribution systems). + + There will eventually be a suite of PEPs covering various aspects of + the metadata 2.0 format and related systems: + + * this PEP, covering the core metadata format + * PEP 440, covering the versioning identification and selection scheme + * a new PEP to define v2.0 of the sdist format + * an updated wheel PEP (v1.1) to add pymeta.json + * an updated installation database PEP both for pymeta.json and to add + a linking scheme to better support runtime selection of dependencies, + as well as recording which extras are currently available + * a new static config PEP to standardise metadata generation and + creation of sdists + * PEP 439, covering a bootstrapping mechanism for ``pip`` + * a distutils upgrade PEP, adding metadata v2.0 and wheel support. + + It's going to take a while to work through all of these and make them + a reality. The main change from our last attempt at this is that we're + trying to design the different pieces so we can implement them + independently of each other, without requiring users to switch to + a whole new tool chain (although they may have to upgrade their existing + ones to start enjoying the benefits in their own work). + + Many of the inline notes in this version of the PEP are there to aid + reviewers that are familiar with the old metadata standards. Before this + version is finalised, most of that content will be moved down to the + "rationale" section at the end of the document, as it would otherwise be + an irrelevant distraction for future readers. + + +Definitions +=========== + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", +"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this +document are to be interpreted as described in RFC 2119. + +"Distributions" are deployable software components published through an index +server or otherwise made available for installation. + +"Versions" are uniquely identified snapshots of a distribution. + +"Distribution archives" are the packaged files which are used to publish +and distribute the software. + +"Source archives" require build tools to be available on the target +system. + +"Binary archives" only require that prebuilt files be moved to the correct +location on the target system. As Python is a dynamically bound +cross-platform language, many "binary" archives will contain only pure +Python source code. + +"Build tools" are automated tools intended to run on development systems, +producing source and binary distribution archives. Build tools may also be +invoked by installation tools in order to install software distributed as +source archives rather than prebuilt binary archives. + +"Index servers" are active distribution registries which publish version and +dependency metadata and place constraints on the permitted metadata. + +"Publication tools" are automated tools intended to run on development +systems and upload source and binary distribution archives to index servers. + +"Installation tools" are automated tools intended to run on production +systems, consuming source and binary distribution archives from an index +server or other designated location and deploying them to the target system. + +"Automated tools" is a collective term covering build tools, index servers, +publication tools, installation tools and any other software that produces +or consumes distribution version and dependency metadata. + +"Projects" refers to the developers that manage the creation of a particular +distribution. + +"Legacy metadata" refers to earlier versions of this metadata specification, +along with the supporting metadata file formats defined by the +``setuptools`` project. + + +Development and distribution activities +======================================= + +Making effective use of a common metadata format requires a common +understanding of the most complex development and distribution model +the format is intended to support. The metadata format described in this +PEP is based on the following activities: + +* Development: during development, a user is operating from a + source checkout (or equivalent) for the current project. Dependencies must + be available in order to build, test and create a source archive of the + distribution. + + .. note:: + As a generated file, the full distribution metadata often won't be + available in a raw source checkout or tarball. In such cases, the + relevant distribution metadata is generally obtained from another + location, such as the last published release, or by generating it + based on a command given in a standard input file. This spec + deliberately avoids handling that scenario, instead falling back on + the existing ``setup.py`` functionality. + +* Build: the build step is the process of turning a source archive into a + binary archive. Dependencies must be available in order to build and + create a binary archive of the distribution (including any documentation + that is installed on target systems). + +* Deployment: the deployment phase consists of two subphases: + + * Installation: the installation phase involves getting the distribution + and all of its runtime dependencies onto the target system. In this + phase, the distribution may already be on the system (when upgrading or + reinstalling) or else it may be a completely new installation. + + * Usage: the usage phase, also referred to as "runtime", is normal usage + of the distribution after it has been installed on the target system. + +The metadata format described in this PEP is designed to enable the +following: + +* It should be practical to have separate development systems, build systems + and deployment systems. +* It should be practical to install dependencies needed specifically to + build source archives only on development systems. +* It should be practical to install dependencies needed specifically to + build the software only on development and build systems, as well as + optionally on deployment systems if installation from source archives + is needed. +* It should be practical to install dependencies needed to run the + distribution only on development and deployment systems. +* It should be practical to install the dependencies needed to run a + distribution's test suite only on development systems, as well as + optionally on deployment systems. +* It should be practical for repackagers to separate out the build + dependencies needed to build the application itself from those required + to build its documentation (as the documentation often doesn't need to + be rebuilt when porting an application to a different platform). + +.. note:: + + This "most complex supported scenario" is almost *exactly* what has to + happen to get an upstream Python package into a Linux distribution, and + is why the current crop of automatic Python metadata -> Linux distro + metadata converters have some serious issues, at least from the point of + view of complying with distro packaging policies: the information + they need to comply with those policies isn't available from the + upstream projects, and all current formats for publishing it are + distro specific. This means either upstreams have to maintain metadata + for multiple distributions (which rarely happens) or else repackagers + have to do a lot of work manually in order to separate out these + dependencies in a way that complies with those policies. + + One thing this PEP aims to do is define a metadata format that at least + has the *potential* to provide the info repackagers need, thus allowing + upstream Python projects and Linux distro repackagers to collaborate more + effectively (and, ideally, make it possible to reliably automate + the process of converting upstream Python distributions into policy + compliant distro packages). + + Some items in this section (and the contents of this note) will likely + end up moving down to the "Rationale for changes from PEP 345" section. + + +Metadata format +=============== + +The format defined in this PEP is an in-memory representation of Python +distribution metadata as a string-keyed dictionary. Permitted values for +individual entries are strings, lists of strings, and additional +nested string-keyed dictionaries. + +Except where otherwise noted, dictionary keys in distribution metadata MUST +be valid Python identifiers in order to support attribute based metadata +access APIs. + +The individual field descriptions show examples of the key name and value +as they would be serialised as part of a JSON mapping. + +The fields identified as core metadata are required. Automated tools MUST +NOT accept distributions with missing core metadata as valid Python +distributions. + +All other fields are optional. Automated tools MUST operate correctly +if a distribution does not provide them, except for those operations +which specifically require the omitted fields. + +Automated tools MUST NOT insert dummy data for missing fields. If a valid +value is not provided for a required field then the metadata and the +associated distribution MUST be rejected as invalid. If a valid value +is not provided for an optional field, that field MUST be omitted entirely. +Automated tools MAY automatically derive valid values from other +information sources (such as a version control system). Metadata files -============== - -The syntax defined in this PEP is for use with Python distribution -metadata files. The file format is a simple UTF-8 encoded Key: value -format with case-insensitive keys and no maximum line length, optionally -followed by a blank line and a payload containing a description of the -distribution. - -This format is parseable by the ``email`` module with an appropriate -``email.policy.Policy()`` (see `Appendix A`_). When ``metadata`` is a -Unicode string, ```email.parser.Parser().parsestr(metadata)`` is a -serviceable parser. +-------------- + +The information defined in this PEP is serialised to ``pymeta.json`` +files for some use cases. As indicated by the extension, these +are JSON-encoded files. Each file consists of a single serialised mapping, +with fields as described in this PEP. There are three standard locations for these metadata files: -* the ``PKG-INFO`` file included in the base directory of Python - source distribution archives (as created by the distutils ``sdist`` - command) -* the ``{distribution}-{version}.dist-info/METADATA`` file in a ``wheel`` - binary distribution archive (as described in PEP 425, or a later version - of that specification) -* the ``{distribution}-{version}.dist-info/METADATA`` files in a local - Python installation database (as described in PEP 376, or a later version - of that specification) +* as a ``{distribution}-{version}.dist-info/pymeta.json`` file in an + ``sdist`` source distribution archive +* as a ``{distribution}-{version}.dist-info/pymeta.json`` file in a ``wheel`` + binary distribution archive +* as a ``{distribution}-{version}.dist-info/pymeta.json`` file in a local + Python installation database + +.. note:: + + These locations are to be confirmed, since they depend on the definition + of sdist 2.0 and the revised installation database standard. There will + also be a wheel 1.1 format update after this PEP is approved that + mandates 2.0+ metadata. Other tools involved in Python distribution may also use this format. - -Encoding -======== - -Metadata 2.0 files are UTF-8 with the restriction that keys must be -ASCII. Parser implementations should be aware that older versions of -the Metadata specification do not specify an encoding. - - -Metadata header fields -======================= - -This section specifies the names and semantics of each of the -supported fields in the metadata header. - -In a single Metadata 2.0 file, fields marked with "(optional)" may occur -0 or 1 times. Fields marked with "(multiple use)" may be specified -0, 1 or more times. Only "Metadata-Version", "Name", "Version", and -"Summary" must appear exactly once. - -The fields may appear in any order within the header section of the file. - - -Metadata-Version +It is expected that these metadata files will be generated by build tools +based on other input formats (such as ``setup.py``) rather than being +edited by hand. + +.. note:: + + It may be appropriate to add a "./setup.py dist_info" command to + setuptools to allow just the sdist metadata files to be generated + without having to build the full sdist archive. This would be + similar to the existing "./setup.py egg_info" command in setuptools, + which would continue to emit the legacy metadata format. + +For backwards compatibility with older installation tools, metadata 2.0 +files MAY be distributed alongside legacy metadata. + +Index servers MAY allow distributions to be uploaded and installation tools +MAY allow distributions to be installed with only legacy metadata. + + +Essential dependency resolution metadata +---------------------------------------- + +For dependency resolution purposes, it is useful to have a minimal subset +of the metadata that contains only those fields needed to identify +distributions and resolve dependencies between them. + +The essential dependency resolution metadata consists of the following +fields: + +* ``metadata_version`` +* ``name`` +* ``version`` +* ``build_label`` +* ``version_url`` +* ``extras`` +* ``requires`` +* ``may-require`` +* ``build-requires`` +* ``build-may-require`` +* ``dev-requires`` +* ``dev-may-require`` +* ``provides`` +* ``obsoleted_by`` +* ``supports_environments`` + +When serialised to a file, the name used for this metadata set SHOULD +be ``pymeta-minimal.json``. + +Abbreviated metadata +-------------------- + +Some metadata fields have the potential to contain a lot of information +that will rarely be referenced, greatly increasing storage requirements +without providing significant benefits. + +The abbreviated metadata for a distribution consists of all fields +*except* the following: + +* ``description`` +* ``contributors`` + +When serialised to a file, the name used for this metadata set SHOULD +be ``pymeta-short.json``. + + +Core metadata +============= + +This section specifies the core metadata fields that are required for every +Python distribution. + +Publication tools MUST ensure at least these fields are present when +publishing a distribution. + +Index servers MUST ensure at least these fields are present in the metadata +when distributions are uploaded. + +Installation tools MUST refuse to install distributions with one or more +of these fields missing by default, but MAY allow users to force such an +installation to occur. + + +Metadata version ---------------- -Version of the file format; "2.0" is the only legal value. - -Automated tools consuming metadata should warn if ``Metadata-Version`` is -greater than the highest version they support, and must fail if -``Metadata-Version`` has a greater major version than the highest -version they support. - -For broader compatibility, automated tools may choose to produce +Version of the file format; ``"2.0"`` is the only legal value. + +Automated tools consuming metadata SHOULD warn if ``metadata_version`` is +greater than the highest version they support, and MUST fail if +``metadata_version`` has a greater major version than the highest +version they support (as described in PEP 440, the major version is the +value before the first dot). + +For broader compatibility, build tools MAY choose to produce distribution metadata using the lowest metadata version that includes all of the needed fields. Example:: - Metadata-Version: 2.0 + "metadata_version": "2.0" Name @@ -118,1057 +381,1100 @@ The name of the distribution. +As distribution names are used as part of URLs, filenames, command line +parameters and must also interoperate with other packaging systems, the +permitted characters are constrained to: + +* ASCII letters (``[a-zA-Z]``) +* ASCII digits (``[0-9]``) +* underscores (``_``) +* hyphens (``-``) +* periods (``.``) + +Distributions named MUST start and end with an ASCII letter or digit. + +Automated tools MUST reject non-compliant names. + +All comparisons of distribution names MUST be case insensitive, and MUST +consider hyphens and underscores to be equivalent. + +Index servers MAY consider "confusable" characters (as defined by the +Unicode Consortium in `TR39: Unicode Security Mechanisms <TR39>`__) to be +equivalent. + +Index servers that permit arbitrary distribution name registrations from +untrusted sources SHOULD consider confusable characters to be equivalent +when registering new distributions (and hence reject them as duplicates). + +Installation tools MUST NOT silently accept a confusable alternate +spelling as matching a requested distribution name. + +At time of writing, the characters in the ASCII subset designated as +confusables by the Unicode Consortium are: + +* ``1`` (DIGIT ONE), ``l`` (LATIN SMALL LETTER L), and ``I`` (LATIN CAPITAL + LETTER I) +* ``0`` (DIGIT ZERO), and ``O`` (LATIN CAPITAL LETTER O) + + Example:: - Name: BeagleVote + "name": "ComfyChair" + +.. note:: + + Debian doesn't actually permit underscores in names, but that seems + unduly restrictive for this spec given the common practice of using + valid Python identifiers as Python distribution names. A Debian side + policy of converting underscores to hyphens seems easy enough to + implement (and the requirement to consider hyphens and underscores as + equivalent ensures that doing so won't introduce any conflicts). + + We're deliberately *not* following Python 3 down the path of arbitrary + unicode identifiers at this time. The security implications of doing so + are substantially worse in the software distribution use case (it opens + up far more interesting attack vectors than mere code obfuscation), the + existing tooling really only works properly if you abide by the stated + restrictions and changing it would require a *lot* of work for all + the automated tools in the chain. + + PyPI has recently been updated to reject non-compliant names for newly + registered projects, but existing non-compliant names are still + tolerated when using legacy metadata formats. Affected distributions + will need to change their names (typically be replacing spaces with + hyphens) before they can migrate to the new metadata formats. + + Donald Stufft ran an analysis, and the new restrictions impact less + than 230 projects out of the ~31k already on PyPI. This isn't that + surprising given the fact that many existing tools could already + exhibit odd behaviour when attempting to deal with non-compliant + names, implicitly discouraging the use of more exotic names. + + Of those projects, ~200 have the only non-compliant character as an + internal space (e.g. "Twisted Web"). These will be automatically + migrated by replacing the spaces with hyphens (e.g. "Twisted-Web"), + which is what you have to actually type to install these distributions + with ``setuptools`` (which powers both ``easy_install`` and ``pip``). + + The remaining ~30 will be investigated manually and decided upon on a + case by case basis how to migrate them to the new naming rules (in + consultation with the maintainers of those projects where possible). Version ------- -The distribution's public version identifier. Public versions are designed -for consumption by automated tools and are strictly ordered according -to a defined scheme. See `Version scheme`_ below. +The distribution's public version identifier, as defined in PEP 440. Public +versions are designed for consumption by automated tools and support a +variety of flexible version specification mechanisms (see PEP 440 for +details). Example:: - Version: 1.0a2 - + "version": "1.0a2" + + +Additional identifying metadata +=============================== + +This section specifies fields that provide other identifying details +that are unique to this distribution. + +All of these fields are optional. Automated tools MUST operate correctly if +a distribution does not provide them, including failing cleanly when an +operation depending on one of these fields is requested. + + +Build label +----------- + +A constrained identifying text string, as defined in PEP 440. Build labels +cannot be used in ordered version comparisons, but may be used to select +an exact version (see PEP 440 for details). + + +Examples:: + + "build_label": "1.0.0-alpha.1" + + "build_label": "1.3.7+build.11.e0f985a" + + "build_label": "v1.8.1.301.ga0df26f" + + "build_label": "2013.02.17.dev123" + + +Version URL +----------- + +A string containing a full URL where this specific version of the +distribution can be downloaded. (This means that the URL can't be +something like ``"https://github.com/pypa/pip/archive/master.zip"``, but +instead must be ``"https://github.com/pypa/pip/archive/1.3.1.zip"``.) + +Some appropriate targets for a version URL are a source tarball, an sdist +archive or a direct reference to a tag or specific commit in an online +version control system. + +All version URL references SHOULD either specify a secure transport +mechanism (such as ``https``) or else include an expected hash value in the +URL for verification purposes. If an insecure transport is specified without +any hash information (or with hash information that the tool doesn't +understand), automated tools SHOULD at least emit a warning and MAY +refuse to rely on the URL. + +It is RECOMMENDED that only hashes which are unconditionally provided by +the latest version of the standard library's ``hashlib`` module be used +for source archive hashes. At time of writing, that list consists of +``'md5'``, ``'sha1'``, ``'sha224'``, ``'sha256'``, ``'sha384'``, and +``'sha512'``. + +For source archive references, an expected hash value may be specified by +including a ``<hash-algorithm>=<expected-hash>`` as part of the URL +fragment. + +For version control references, the ``VCS+protocol`` scheme SHOULD be +used to identify both the version control system and the secure transport. + +To support version control systems that do not support including commit or +tag references directly in the URL, that information may be appended to the +end of the URL using the ``@<tag>`` notation. + +Example:: + + "version_url": "https://github.com/pypa/pip/archive/1.3.1.zip" + "version_url": "http://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a…" + "version_url": "git+https://github.com/pypa/pip.git@1.3.1" + +.. note:: + + This was called "Download-URL" in previous versions of the metadata. It + has been renamed, since there are plenty of other download locations and + this URL is meant to be a way to get the original source for development + purposes (or to generate an SRPM or other platform specific equivalent). + + For extra fun and games, it appears that unlike "svn+ssh://", + neither "git+ssh://" nor "hg+ssh://" natively support direct linking to a + particular tag (hg does support direct links to bookmarks through the URL + fragment, but that doesn't help for git and doesn't appear to be what I + want anyway). + + However pip does have a `defined convention + <http://www.pip-installer.org/en/latest/logic.html#vcs-support>`__ for + this kind of link, which effectively splits a "URL" into "<repo>@<tag>". + + The PEP simply adopts pip's existing solution to this problem. + + This field is separate from the project URLs, as it's expected to + change for each version, while the project URLs are expected to + be fairly stable. + + +Additional descriptive metadata +=============================== + +This section specifies fields that provide additional information regarding +the distribution and its contents. + +All of these fields are optional. Automated tools MUST operate correctly if +a distribution does not provide them, including failing cleanly when an +operation depending on one of these fields is requested. Summary ------- A one-line summary of what the distribution does. +Publication tools SHOULD emit a warning if this field is not provided. Index +servers MAY require that this field be present before allowing a +distribution to be uploaded. + Example:: - Summary: A module for collecting votes from beagles. - - -Private-Version (optional) --------------------------- - -An arbitrary private version label. Private version labels are intended -for internal use by a project, and cannot be used in version specifiers. -See `Compatibility with other version schemes`_ below. - -Examples:: - - Private-Version: 1.0.0-alpha.1 - Private-Version: 1.3.7+build.11.e0f985a - Private-Version: v1.8.1.301.ga0df26f - Private-Version: 2013.02.17.dev123 - - -Description (optional, deprecated) ----------------------------------- - -Starting with Metadata 2.0, the recommended place for the description is in -the payload section of the document, after the last header. The description -does not need to be reformatted when it is included in the payload. - -See `Describing the Distribution`_ for more information on the expected -contents of this field. - -Since a line separator immediately followed by another line separator -indicates the end of the headers section, any line separators in a -``Description`` header field must be suffixed by whitespace to -indicate continuation. - -It is an error to provide both a ``Description`` header and a metadata -payload. - - -Keywords (optional) -------------------- - -A list of additional whitespace separated keywords to be used to assist -searching for the distribution in a larger catalog. - -Example:: - - Keywords: dog puppy voting election - - -Home-page (optional) --------------------- - -A string containing the URL for the distribution's home page. - -Example:: - - Home-page: http://www.example.com/~cschultz/bvote/ - - -Download-URL (optional) ------------------------ - -A string containing the URL from which this version of the distribution -can be downloaded. (This means that the URL can't be something like -".../BeagleVote-latest.tgz", but instead must be ".../BeagleVote-0.45.tgz".) - - -Project-URL (multiple use) --------------------------- - -A string containing a label and a browsable URL for the project, separated -by the last occurrence of comma and space ", ". - -The label consists of any permitted header text, including commas. - -Example:: - - Bug, Issue Tracker, http://bitbucket.org/tarek/distribute/issues/ - - -Author (optional) ------------------ - -A string containing the author's name at a minimum; additional -contact information may be provided. - -Example:: - - Author: C. Schultz, Universal Features Syndicate, - Los Angeles, CA <cschultz(a)peanuts.example.com> - - -Author-email (optional) ------------------------ - -A string containing the author's e-mail address. It contains a name -and e-mail address in the RFC 5322 recommended ``Address Specification`` -format. - -Example:: - - Author-email: "C. Schultz" <cschultz(a)example.com> - - -Maintainer (optional) ---------------------- - -A string containing the maintainer's name at a minimum; additional -contact information may be provided. - -Note that this field is intended for use when a project is being -maintained by someone other than the original author: it should be -omitted if it is identical to ``Author``. - -Example:: - - Maintainer: C. Schultz, Universal Features Syndicate, - Los Angeles, CA <cschultz(a)peanuts.example.com> - - -Maintainer-email (optional) ---------------------------- - -A string containing the maintainer's e-mail address. It has the same -format as ``Author-email``. - -Note that this field is intended for use when a project is being -maintained by someone other than the original author: it should be -omitted if it is identical to ``Author-email``. - -Example:: - - Maintainer-email: "C. Schultz" <cschultz(a)example.com> - - -License (optional) ------------------- - -Text indicating the license covering the distribution where the license -is not a selection from the "License" Trove classifiers. See -"Classifier" below. This field may also be used to specify a -particular version of a license which is named via the ``Classifier`` -field, or to indicate a variation or exception to such a license. - -Examples:: - - License: This software may only be obtained by sending the - author a postcard, and then the user promises not - to redistribute it. - - License: GPL version 3, excluding DRM provisions - -The full text of the license would normally be included in a separate -file. - - -Classifier (multiple use) -------------------------- - -Each entry is a string giving a single classification value -for the distribution. Classifiers are described in PEP 301 [2]. - -`Environment markers`_ may be used with this field. - -Examples:: - - Classifier: Development Status :: 4 - Beta - Classifier: Environment :: Console (Text Based) - - -Provides-Dist (multiple use) ----------------------------- - -Each entry contains a string naming a requirement that is satisfied by -installing this distribution. These strings must be of the form -``Name`` or ``Name (Version)``, following the formats of the corresponding -field definitions. - -A distribution may provide additional names, e.g. to indicate that -multiple projects have been merged into and replaced by a single -distribution or to indicate that this project is a substitute for another. -For instance, distribute (a fork of setuptools) can include a -``Provides-Dist: setuptools`` entry to prevent the conflicting -package from being downloaded and installed when distribute is already -installed. A distribution that has been merged with another might -``Provides-Dist`` the obsolete name(s) to satisfy any projects that -require the obsolete distribution's name. - -A distribution may also provide a "virtual" project name, which does -not correspond to any separately-distributed project: such a name -might be used to indicate an abstract capability which could be supplied -by one of multiple projects. E.g., multiple projects might supply -RDBMS bindings for use by a given ORM: each project might declare -that it provides ``ExampleORM-somedb-bindings``, allowing other -projects to depend only on having at least one of them installed. - -A version declaration may be supplied and must follow the rules described -in `Version scheme`_. The distribution's version identifier will be implied -if none is specified. - -`Environment markers`_ may be used with this field. - -Examples:: - - Provides-Dist: AnotherProject (3.4) - Provides-Dist: virtual_package - - -Provides-Extra (multiple use) ------------------------------ - -A string containing the name of an optional feature or "extra" that may -only be available when additional dependencies have been installed. Must -be printable ASCII, not containing whitespace, comma (,), or square -brackets []. - -See `Optional Features`_ for details on the use of this field. - -Example:: - - Name: beaglevote - Provides-Extra: pdf - Requires-Dist: reportlab; extra == 'pdf' - Requires-Dist: nose; extra == 'test' - Requires-Dist: sphinx; extra == 'doc' - - -Obsoleted-By (optional) ------------------------ - -Indicates that this project is no longer being developed. The named -project provides a substitute or replacement. - -A version declaration may be supplied and must follow the rules described -in `Version specifiers`_. - -Possible uses for this field include handling project name changes and -project mergers. - -Examples:: - - Name: BadName - Obsoleted-By: AcceptableName - - Name: SeparateProject - Obsoleted-By: MergedProject (>=4.0.0) - - -Requires-Dist (multiple use) ----------------------------- - -Each entry contains a string naming some other distutils -project required by this distribution. - -The format of a requirement string is identical to that of a distribution -name (e.g., as found in the ``Name:`` field) optionally followed by a -version declaration within parentheses. - -The distribution names should correspond to names as found on the `Python -Package Index`_; often the same as, but distinct from, the module names -as accessed with ``import x``. - -`Environment markers`_ may be used with this field. - -Version declarations must follow the rules described in -`Version specifiers`_ - -Distributions may also depend on optional features of other distributions. -See `Optional Features`_ for details. - -Examples:: - - Requires-Dist: pkginfo - Requires-Dist: PasteDeploy - Requires-Dist: zope.interface (>3.5.0) - -Dependencies mentioned in ``Requires-Dist`` may be installed exclusively -at run time and are not guaranteed to be available when creating or -installing a package. If a dependency is needed during distribution -creation or installation *and* at run time, it should be listed under -both ``Requires-Dist`` and ``Setup-Requires-Dist``. - - -Setup-Requires-Dist (multiple use) ----------------------------------- - -Like ``Requires-Dist``, but names dependencies needed in order to build, -package or install the distribution -- in distutils, a dependency imported -by ``setup.py`` itself. Commonly used to bring in extra compiler support -or a package needed to generate a manifest from version control. - -Examples:: - - Setup-Requires-Dist: custom_setup_command - -Dependencies mentioned in ``Setup-Requires-Dist`` may be installed -exclusively for setup and are not guaranteed to be available at run time. -If a dependency is needed during distribution creation or installation -*and* at run time, it should be listed under both ``Requires-Dist`` and -``Setup-Requires-Dist``. - - -Requires-Python (multiple use) ------------------------------- - -This field specifies the Python version(s) that the distribution is -guaranteed to be compatible with. - -`Environment markers`_ may be used with this field. - -Version declarations must be in the format specified in -`Version specifiers`_. - -Examples:: - - Requires-Python: 3.2 - Requires-Python: >3.1 - Requires-Python: >=2.3.4 - Requires-Python: >=2.5,<2.7 - -If specified multiple times, the Python version must satisfy all such -constraints to be considered compatible. This is most useful in combination -with appropriate `Environment markers`_. - -For example, if a feature was initially introduced to Python as a -Unix-specific addition, and then Windows support was added in the -subsequent release, this could be indicated with the following pair -of entries:: - - Requires-Python: >= 3.1 - Requires-Python: >= 3.2; sys.platform == 'win32' - - -Requires-External (multiple use) --------------------------------- - -Each entry contains a string describing some dependency in the -system that the distribution is to be used. This field is intended to -serve as a hint to downstream project maintainers, and has no -semantics which are meaningful to the ``distutils`` distribution. - -The format of a requirement string is a name of an external -dependency, optionally followed by a version declaration within -parentheses. - -`Environment markers`_ may be used with this field. - -Because they refer to non-Python software releases, version identifiers -for this field are **not** required to conform to the format -described in `Version scheme`_: they should correspond to the -version scheme used by the external dependency. - -Notice that there is no particular rule on the strings to be used. - -Examples:: - - Requires-External: C - Requires-External: libpng (>=1.5) - - -Platform (multiple use) ------------------------ - -A Platform specification describing an operating system supported by -the distribution which is not listed in the "Operating System" Trove -classifiers. See `Classifier`__ above. - -__ `Classifier (multiple use)`_ - -Examples:: - - Platform: ObscureUnix - Platform: RareDOS - - -Supported-Platform (multiple use) ---------------------------------- - -Binary distributions containing a metadata file will use the -Supported-Platform field in their metadata to specify the OS and -CPU for which the binary distribution was compiled. The semantics of -the Supported-Platform field are not specified in this PEP. - -Example:: - - Supported-Platform: RedHat 7.2 - Supported-Platform: i386-win32-2791 - - -Extension (multiple use) ------------------------- - -An ASCII string, not containing whitespace or the ``/`` character, that -indicates the presence of extended metadata. The additional fields -defined by the extension are then prefixed with the name of the extension -and the ``/`` character. The additional fields are optional (0..1). - -For example:: - - Extension: Chili - Chili/Type: Poblano - Chili/Heat: Mild - Chili/json: { - "type" : "Poblano", - "heat" : "Mild" } - -The special ``{extension name}/json`` permits embedded JSON. It may be -parsed automatically by a future tool. - -Values in extension fields must still respect the general formatting -requirements for metadata headers. - -To avoid name conflicts, it is recommended that distribution names be used -to identify metadata extensions. This practice will also make it easier to -find authoritative documentation for metadata extensions. - -As the order of the metadata headers is not constrained, the -``Extension: Chili`` field may appear before or after the corresponding -extension fields ``Chili/Type:`` etc. - -A bare ``Extension: Name`` entry with no corresponding extension fields is -permitted. It may, for example, indicate the expected presence of an -additional metadata file rather than the presence of extension fields. - -An extension field with no corresponding ``Extension: Name`` entry is an -error. - - -Describing the distribution -=========================== + "summary": "A module that is more fiendish than soft cushions." + +.. note:: + + This used to be mandatory, and it's still highly recommended, but really, + nothing should break even when it's missing. + + +Description +----------- The distribution metadata should include a longer description of the distribution that may run to several paragraphs. Software that deals with metadata should not assume any maximum size for the description. -The recommended location for the description is in the metadata payload, -separated from the header fields by at least one completely blank line -(that is, two successive line separators with no other characters -between them, not even whitespace). - -Alternatively, the description may be provided in the `Description`__ -metadata header field. Providing both a ``Description`` field and a -payload is an error. - -__ `Description (optional, deprecated)`_ - The distribution description can be written using reStructuredText markup [1]_. For programs that work with the metadata, supporting markup is optional; programs may also display the contents of the field as plain text without any special formatting. This means that authors should be conservative in the markup they use. - -Version scheme -============== - -Public version identifiers must comply with the following scheme:: - - N[.N]+[{a|b|c|rc}N][.postN][.devN] - -Version identifiers which do not comply with this scheme are an error. - -Version identifiers must not include leading or trailing whitespace. - -Any given version will be a "release", "pre-release", "post-release" or -"developmental release" as defined in the following sections. +Example:: + + "description": "The ComfyChair module replaces SoftCushions.\\n\\nUse until lunchtime, but pause for a cup of coffee at eleven." .. note:: - Some hard to read version identifiers are permitted by this scheme in - order to better accommodate the wide range of versioning practices - across existing public and private Python projects, given the - constraint that the package index is not yet sophisticated enough to - allow the introduction of a simpler, backwards-incompatible scheme. - - Accordingly, some of the versioning practices which are technically - permitted by the PEP are strongly discouraged for new projects. Where - this is the case, the relevant details are noted in the following - sections. - - -Releases + The difficulty of editing this field in a raw JSON file is one of the + main reasons this metadata interchange format is NOT recommended for + use as an input format for build tools. + + +Description Format +------------------ + +A field indicating the intended format of the text in the description field. +This allows index servers to render the description field correctly and +provide feedback on rendering errors, rather than having to guess the +intended format. + +If this field is omitted, or contains an unrecognised value, the default +rendering format MUST be plain text. + +The following format names SHOULD be used for the specified markup formats: + +* ``txt``: Plain text (default handling if field is omitted) +* ``rst``: reStructured Text +* ``md``: Markdown (exact syntax variant will be implementation dependent) +* ``adoc``: AsciiDoc +* ``html``: HTML + +Automated tools MAY render one or more of the listed formats as plain +text and MAY accept other markup formats beyond those listed. + +Example:: + + "description_format": "rst" + + +Keywords -------- -A release number is a version identifier that consists solely of one or -more non-negative integer values, separated by dots:: - - N[.N]+ - -Releases within a project must be numbered in a consistently increasing -fashion. Ordering considers the numeric value of each component -in turn, with "component does not exist" sorted ahead of all numeric -values. - -Date based release numbers are explicitly excluded from compatibility with -this scheme, as they hinder automatic translation to other versioning -schemes, as well as preventing the adoption of semantic versioning without -changing the name of the project. Accordingly, a leading release component -greater than or equal to ``1980`` is an error. - -While any number of additional components after the first are permitted -under this scheme, the most common variants are to use two components -("major.minor") or three components ("major.minor.micro"). - -For example:: - - 0.9 - 0.9.1 - 0.9.2 - ... - 0.9.10 - 0.9.11 - 1.0 - 1.0.1 - 1.1 - 2.0 - 2.0.1 - -A release series is any set of release numbers that start with a common -prefix. For example, ``3.3.1``, ``3.3.5`` and ``3.3.9.45`` are all -part of the ``3.3`` release series. +A list of additional keywords to be used to assist searching for the +distribution in a larger catalog. + +Example:: + + "keywords": ["comfy", "chair", "cushions", "too silly", "monty python"] + + +License +------- + +A string indicating the license covering the distribution where the license +is not a simple selection from the "License" Trove classifiers. See +Classifiers" below. This field may also be used to specify a +particular version of a license which is named via the ``Classifier`` +field, or to indicate a variation or exception to such a license. + +Example:: + + "license": "GPL version 3, excluding DRM provisions" + + +License URL +----------- + +A specific URL referencing the full licence text for this version of the +distribution. + +Example:: + + "license_url": "https://github.com/pypa/pip/blob/1.3.1/LICENSE.txt" .. note:: - Using both ``X.Y`` and ``X.Y.0`` as distinct release numbers within the - scope of a single release series is strongly discouraged, as it makes the - version ordering ambiguous for human readers. Automated tools should - either treat this case as an error, or else interpret an ``X.Y.0`` - release as coming *after* the corresponding ``X.Y`` release. - - The recommended practice is to always use release numbers of a consistent - length (that is, always include the trailing ``.0``). An acceptable - alternative is to consistently omit the trailing ``.0``. The example - above shows both styles, always including the ``.0`` at the second - level and consistently omitting it at the third level. - - -Pre-releases + Like Version URL, this is handled separately from the project URLs + as it is important that it remain accurate for this *specific* + version of the distribution, even if the project later switches to a + different license. + + The project URLs field is intended for more stable references. + + +Classifiers +----------- + +A list of strings, with each giving a single classification value +for the distribution. Classifiers are described in PEP 301 [2]. + +Example:: + + "classifiers": [ + "Development Status :: 4 - Beta", + "Environment :: Console (Text Based)" + ] + + +Contact metadata +================ + +Contact metadata for a distribution is provided to allow users to get +access to more information about the distribution and its maintainers. + +These details are recorded as mappings with the following subfields: + +* ``name``: the name of an individual or group +* ``email``: an email address (this may be a mailing list) +* ``url``: a URL (such as a profile page on a source code hosting service) +* ``type``: one of ``"author"``, ``"maintainer"``, ``"organization"`` + or ``"individual"`` + +The ``name`` subfield is required, the other subfields are optional. + +If no specific contact type is stated, the default is ``individual``. + +The different contact types are as follows: + +* ``author``: the original creator of a distribution +* ``maintainer``: the current lead contributor for a distribution, when + they are not the original creator +* ``individual``: any other individuals involved in the creation of the + distribution +* ``organization``: indicates that these contact details are for an + organization (formal or informal) rather than for a specific individual + +.. note:: + + This is admittedly a little complicated, but it's designed to replace the + Author, Author-Email, Maintainer, Maintainer-Email fields from metadata + 1.2 in a way that allows those distinctions to be fully represented for + lossless translation, while allowing future distributions to pretty + much ignore everything other than the contact/contributor distinction + if they so choose. + +Contact metadata is optional. Automated tools MUST operate correctly if +a distribution does not provide them, including failing cleanly when an +operation depending on one of these fields is requested. + + +Contacts +-------- + +A list of contact entries giving the recommended contact points for getting +more information about the project. + +The example below would be suitable for a project that was in the process +of handing over from the original author to a new lead maintainer, while +operating as part of a larger development group. + +Example:: + + "contacts": [ + { + "name": "Python Packaging Authority/Distutils-SIG", + "type": "organization", + "email": "distutils-sig(a)python.org", + "url": "https://bitbucket.org/pypa/" + }, + { + "name": "Samantha C.", + "type": "maintainer", + "email": "dontblameme(a)example.org" + }, + { + "name": "Charlotte C.", + "type": "author", + "email": "iambecomingasketchcomedian(a)example.com" + } + ] + + +Contributors ------------ -Some projects use an "alpha, beta, release candidate" pre-release cycle to -support testing by their users prior to a full release. - -If used as part of a project's development cycle, these pre-releases are -indicated by a suffix appended directly to the last component of the -release number:: - - X.YaN # Alpha release - X.YbN # Beta release - X.YcN # Release candidate (alternative notation: X.YrcN) - X.Y # Full release - -The pre-release suffix consists of an alphabetical identifier for the -pre-release phase, along with a non-negative integer value. Pre-releases for -a given release are ordered first by phase (alpha, beta, release candidate) -and then by the numerical component within that phase. +A list of contact entries for other contributors not already listed as +current project points of contact. The subfields within the list elements +are the same as those for the main contact field. + +Example:: + + "contributors": [ + {"name": "John C."}, + {"name": "Erik I."}, + {"name": "Terry G."}, + {"name": "Mike P."}, + {"name": "Graeme C."}, + {"name": "Terry J."} + ] + + +Project URLs +------------ + +A mapping of arbitrary text labels to additional URLs relevant to the +project. + +While projects are free to choose their own labels and specific URLs, +it is RECOMMENDED that home page, source control, issue tracker and +documentation links be provided using the labels in the example below. + +URL labels MUST be treated as case insensitive by automated tools, but they +are not required to be valid Python identifiers. Any legal JSON string is +permitted as a URL label. + +Example:: + + "project_urls": { + "Documentation": "https://distlib.readthedocs.org" + "Home": "https://bitbucket.org/pypa/distlib" + "Source": "https://bitbucket.org/pypa/distlib/src" + "Tracker": "https://bitbucket.org/pypa/distlib/issues" + } + + +Dependency metadata +=================== + +Dependency metadata allows distributions to make use of functionality +provided by other distributions, without needing to bundle copies of those +distributions. + +Dependency management is heavily dependent on the version identification +and specification scheme defined in PEP 440. .. note:: - Using both ``c`` and ``rc`` to identify release candidates within - the scope of a single release is strongly discouraged, as it makes the - version ordering ambiguous for human readers. Automated tools should - either treat this case as an error, or else interpret all ``rc`` versions - as coming after all ``c`` versions (that is, ``rc1`` indicates a later - version than ``c2``). - - -Post-releases + This substantially changes the old two-phase setup vs runtime dependency + model in metadata 1.2 (which was in turn derived from the setuptools + dependency parameters). The translation is that ``dev_requires`` and + ``build_requires`` both map to ``Setup-Requires-Dist`` + in 1.2, while ``requires`` maps to ``Requires-Dist``. To go the other + way, ``Setup-Requires-Dist`` maps to ``build_requires`` and + ``Requires-Dist`` maps to ``requires``. + +All of these fields are optional. Automated tools MUST operate correctly if +a distribution does not provide them, by assuming that a missing field +indicates "Not applicable for this distribution". + + +Dependency specifications +------------------------- + +Individual dependencies are typically defined as strings containing a +distribution name (as found in the ``name`` field). The dependency name +may be followed by an extras specifier (enclosed in square +brackets) and by a version specification (within parentheses). + +See `Extras (optional dependencies)`_ for details on extras and PEP 440 +for details on version specifiers. + +The distribution names should correspond to names as found on the `Python +Package Index`_; while these names are often the same as the module names +as accessed with ``import x``, this is not always the case (especially +for distributions that provide multiple top level modules or packages). + +Example dependency specifications:: + + "Flask" + "Django" + "Pyramid" + "SciPy (0.12)" + "ComfyChair[warmup]" + "ComfyChair[warmup] (> 0.1)" + + +Conditional dependencies +------------------------ + +While many dependencies will be needed to use a distribution at all, others +are needed only on particular platforms or only when particular optional +features of the distribution are needed. To enable this, dependency fields +are marked as either unconditional (indicated by ``requires`` in the field +name) or conditional (indicated by ``may_require``) in the field name. + +Unconditional dependency fields are lists of dependency specifications, with +each entry indicated a required dependency. + +Conditional dependencies are lists of mappings with the following fields: + +* ``dependencies``: a list of relevant dependency specifications +* ``extra``: the name of a set of optional dependencies that are requested + and installed together. See `Extras (optional dependencies)`_ for details. +* ``environment``: an environment marker defining the environment that + needs these dependencies. See `Environment markers`_ for details. + +The ``dependencies`` field is required, as is at least one of ``extra`` and +``environment``. All three fields may be supplied, indicating that the +dependency is needed only when that particular set of additional +dependencies is requested in a particular environment. + +Note that the same extras and environment markers MAY appear in multiple +conditional dependencies. This may happen, for example, if an extra itself +only needs some of its dependencies in specific environments. + +.. note:: + + Technically, you could store the conditional and unconditional + dependencies in a single list and switch based on the entry type + (string or mapping), but the ``*requires`` vs ``*may-require`` two + list design seems easier to understand and work with. + + +Mapping dependencies to development and distribution activities +--------------------------------------------------------------- + +The different categories of dependency are based on the various distribution +and development activities identified above, and govern which dependencies +should be installed for the specified activities: + +* Deployment dependencies: + + * ``requires`` + * ``may_require`` + * Request the ``test`` extra to also install + + * ``test_requires`` + * ``test_may_require`` + +* Build dependencies: + + * ``build_requires`` + * ``build_may_require`` + +* Development dependencies: + + * ``requires`` + * ``may_require`` + * ``build_requires`` + * ``build_may_require`` + * ``test_requires`` + * ``test_may_require`` + * ``dev_requires`` + * ``dev_may_require`` + +To ease compatibility with existing two phase setup/deployment toolchains, +installation tools MAY treat ``dev_requires`` and ``dev_may_require`` as +additions to ``build_requires`` and ``build_may_require`` rather than +as separate fields. + +Installation tools SHOULD allow users to request at least the following +operations for a named distribution: + +* Install the distribution and any deployment dependencies. +* Install just the build dependencies without installing the distribution +* Install just the development dependencies without installing + the distribution + +The notation described in `Extras (optional dependencies)`_ SHOULD be used to +request additional optional dependencies when installing deployment +or build dependencies. + +Installation tools SHOULD report an error if dependencies cannot be found, +MUST at least emit a warning, and MAY allow the user to force the +installation to proceed regardless. + +.. note:: + + As an example of mapping this to Linux distro packages, assume an + example project without any extras defined is split into 2 RPMs + in a SPEC file: example and example-devel + + The ``requires`` and applicable ``may_require`` dependencies would be + mapped to the Requires dependencies for the "example" RPM (a mapping from + environment markers to SPEC file conditions would also allow those to + be handled correctly) + + The ``build_requires`` and ``build_may_require`` dependencies would be + mapped to the BuildRequires dependencies for the "example" RPM. + + All defined dependencies relevant to Linux, including those in + ``dev_requires`` and ``test_requires``, would become Requires + dependencies for the "example-devel" RPM. + + If a project defines any extras, those would be mapped to additional + virtual RPMs with appropriate BuildRequires and Requires entries based + on the details of the dependency specifications. + + A documentation toolchain dependency like Sphinx would either go in + ``build_requires`` (for example, if man pages were included in the + built distribution) or in ``dev_requires`` (for example, if the + documentation is published solely through ReadTheDocs or the + project website). This would be enough to allow an automated converter + to map it to an appropriate dependency in the spec file. + + +Requires +-------- + +A list of other distributions needed when this distribution is deployed. + +Example:: + + "requires": ["SciPy", "PasteDeploy", "zope.interface (>3.5.0)"] + + +Extras +------ + +A list of optional sets of dependencies that may be used to define +conditional dependencies in ``"may_require"`` and similar fields. See +`Extras (optional dependencies)`_ for details. + +The extra name``"test"`` is reserved for requesting the dependencies +specified in ``test_requires`` and ``test_may_require`` and is NOT +permitted in this field. + +Example:: + + "extras": ["warmup"] + + +May require +----------- + +A list of other distributions that may be needed when this distribution +is deployed, based on the extras requested and the target deployment +environment. + +Any extras referenced from this field MUST be named in the `Extras`_ field. + +Example:: + + "may_require": [ + { + "dependencies": ["pywin32 (>1.0)"], + "environment": "sys.platform == 'win32'" + }, + { + "dependencies": ["SoftCushions"], + "extra": "warmup" + } + ] + +Test requires ------------- -Some projects use post-releases to address minor errors in a release that -do not affect the distributed software (for example, correcting an error -in the release notes). - -If used as part of a project's development cycle, these post-releases are -indicated by a suffix appended directly to the last component of the -release number:: - - X.Y.postN # Post-release - -The post-release suffix consists of the string ``.post``, followed by a -non-negative integer value. Post-releases are ordered by their -numerical component, immediately following the corresponding release, -and ahead of any subsequent release. +A list of other distributions needed in order to run the automated tests +for this distribution, either during development or when running the +``test_installed_dist`` metabuild when deployed. + +Example:: + + "test_requires": ["unittest2"] + + +Test may require +---------------- + +A list of other distributions that may be needed in order to run the +automated tests for this distribution, either during development or when +running the ``test_installed_dist`` metabuild when deployed, based on the +extras requested and the target deployment environment. + +Any extras referenced from this field MUST be named in the `Extras`_ field. + +Example:: + + "test_may_require": [ + { + "dependencies": ["pywin32 (>1.0)"], + "environment": "sys.platform == 'win32'" + }, + { + "dependencies": ["CompressPadding"], + "extra": "warmup" + } + ] + + +Build requires +-------------- + +A list of other distributions needed when this distribution is being built +(creating a binary archive from a source archive). + +Note that while these are build dependencies for the distribution being +built, the installation is a *deployment* scenario for the dependencies. + +Example:: + + "build_requires": ["setuptools (>= 0.7)"] + + +Build may require +----------------- + +A list of other distributions that may be needed when this distribution +is built (creating a binary archive from a source archive), based on the +features requested and the build environment. + +Note that while these are build dependencies for the distribution being +built, the installation is a *deployment* scenario for the dependencies. + +Any extras referenced from this field MUST be named in the `Extras`_ field. + +Automated tools MAY assume that all extras are implicitly requested when +installing build dependencies. + +Example:: + + "build_may_require": [ + { + "dependencies": ["pywin32 (>1.0)"], + "environment": "sys.platform == 'win32'" + }, + { + "dependencies": ["cython"], + "extra": "c-accelerators" + } + ] + + +Dev requires +------------ + +A list of any additional distributions needed during development of this +distribution that aren't already covered by the deployment and build +dependencies. + +Additional dependencies that may be listed in this field include: + +* tools needed to create a source archive +* tools needed to generate project documentation that is published online + rather than distributed along with the rest of the software +* additional test dependencies for tests which are not executed when the + test is invoked through the ``test_installed_dist`` metabuild hook (for + example, tests that require a local database server and web server and + may not work when fully installed on a production system) + +Example:: + + "dev_requires": ["hgtools", "sphinx (>= 1.0)"] + + +Dev may require +--------------- + +A list of other distributions that may be needed during development of +this distribution, based on the features requested and the build environment. + +This should only be needed if the project's own utility scripts have +platform specific dependencies that aren't already defined as deployment +or build dependencies. + +Any extras referenced from this field MUST be named in the `Extras`_ field. + +Automated tools MAY assume that all extras are implicitly requested when +installing development dependencies. + +Example:: + + "dev_may_require": [ + { + "dependencies": ["pywin32 (>1.0)"], + "environment": "sys.platform == 'win32'" + } + ] + + +Provides +-------- + +A list of strings naming additional dependency requirements that are +satisfied by installing this distribution. These strings must be of the +form ``Name`` or ``Name (Version)``, as for the ``requires`` field. + +While dependencies are usually resolved based on distribution names and +versions, a distribution may provide additional names explicitly in the +``provides`` field. + +For example, this may be used to indicate that multiple projects have +been merged into and replaced by a single distribution or to indicate +that this project is a substitute for another. + +For instance, with distribute merged back into setuptools, the merged +project is able to include a ``"provides": ["distribute"]`` entry to +satisfy any projects that require the now obsolete distribution's name. + +A distribution may also provide a "virtual" project name, which does +not correspond to any separately distributed project: such a name +might be used to indicate an abstract capability which could be supplied +by one of multiple projects. For example, multiple projects might supply +PostgreSQL bindings for use with SQL Alchemy: each project might declare +that it provides ``sqlalchemy-postgresql-bindings``, allowing other +projects to depend only on having at least one of them installed. + +A version declaration may be supplied and must follow the rules described +in PEP 440. The distribution's version identifier will be implied +if none is specified. + +Example:: + + "provides": ["AnotherProject (3.4)", "virtual_package"] + + +Obsoleted by +------------ + +A string that indicates that this project is no longer being developed. The +named project provides a substitute or replacement. + +A version declaration may be supplied and must follow the rules described +in PEP 440. + +Possible uses for this field include handling project name changes and +project mergers. + +For instance, with distribute merging back into setuptools, a new version +of distribute may be released that depends on the new version of setuptools, +and also explicitly indicates that distribute itself is now obsolete. + +Note that without a corresponding ``provides``, there is no expectation +that the replacement project will be a "drop-in" replacement for the +obsolete project - at the very least, upgrading to the new distribution +is likely to require changes to import statements. + +Examples:: + + "name": "BadName", + "obsoleted_by": "AcceptableName" + + "name": "distribute", + "obsoleted_by": "setuptools (>= 0.7)" + + +Supports Environments +--------------------- + +A list of strings specifying the environments that the distribution +explicitly supports. An environment is considered supported if it +matches at least one of the environment markers given. + +If this field is not given in the metadata, it is assumed that the +distribution supports any platform supported by Python. + +Individual entries are environment markers, as described in +`Environment markers`_. + +Installation tools SHOULD report an error if supported platforms are +specified by the distribution and the current platform fails to match +any of them, MUST at least emit a warning, and MAY allow the user to +force the installation to proceed regardless. + +Examples:: + + "supports_environments": ["sys_platform == 'win32'"] + "supports_environments": ["sys_platform != 'win32'"] + "supports_environments": ["'linux' in sys_platform", + "'bsd' in sys_platform"] + .. note:: - The use of post-releases to publish maintenance releases containing - actual bug fixes is strongly discouraged. In general, it is better - to use a longer release number and increment the final component - for each maintenance release. - -Post-releases are also permitted for pre-releases:: - - X.YaN.postM # Post-release of an alpha release - X.YbN.postM # Post-release of a beta release - X.YcN.postM # Post-release of a release candidate - -.. note:: - - Creating post-releases of pre-releases is strongly discouraged, as - it makes the version identifier difficult to parse for human readers. - In general, it is substantially clearer to simply create a new - pre-release by incrementing the numeric component. - - -Developmental releases ----------------------- - -Some projects make regular developmental releases, and system packagers -(especially for Linux distributions) may wish to create early releases -which do not conflict with later project releases. - -If used as part of a project's development cycle, these developmental -releases are indicated by a suffix appended directly to the last -component of the release number:: - - X.Y.devN # Developmental release - -The developmental release suffix consists of the string ``.dev``, -followed by a non-negative integer value. Developmental releases are ordered -by their numerical component, immediately before the corresponding release -(and before any pre-releases), and following any previous release. - -Developmental releases are also permitted for pre-releases and -post-releases:: - - X.YaN.devM # Developmental release of an alpha release - X.YbN.devM # Developmental release of a beta release - X.YcN.devM # Developmental release of a release candidate - X.Y.postN.devM # Developmental release of a post-release - -.. note:: - - Creating developmental releases of pre-releases is strongly - discouraged, as it makes the version identifier difficult to parse for - human readers. In general, it is substantially clearer to simply create - additional pre-releases by incrementing the numeric component. - - Developmental releases of post-releases are also strongly discouraged, - but they may be appropriate for projects which use the post-release - notation for full maintenance releases which may include code changes. - - -Examples of compliant version schemes -------------------------------------- - -The standard version scheme is designed to encompass a wide range of -identification practices across public and private Python projects. In -practice, a single project attempting to use the full flexibility offered -by the scheme would create a situation where human users had difficulty -figuring out the relative order of versions, even though the rules above -ensure all compliant tools will order them consistently. - -The following examples illustrate a small selection of the different -approaches projects may choose to identify their releases, while still -ensuring that the "latest release" and the "latest stable release" can -be easily determined, both by human users and automated tools. - -Simple "major.minor" versioning:: - - 0.1 - 0.2 - 0.3 - 1.0 - 1.1 - ... - -Simple "major.minor.micro" versioning:: - - 1.1.0 - 1.1.1 - 1.1.2 - 1.2.0 - ... - -"major.minor" versioning with alpha, beta and release candidate -pre-releases:: - - 0.9 - 1.0a1 - 1.0a2 - 1.0b1 - 1.0c1 - 1.0 - 1.1a1 - ... - -"major.minor" versioning with developmental releases, release candidates -and post-releases for minor corrections:: - - 0.9 - 1.0.dev1 - 1.0.dev2 - 1.0.dev3 - 1.0.dev4 - 1.0rc1 - 1.0rc2 - 1.0 - 1.0.post1 - 1.1.dev1 - ... - - -Summary of permitted suffixes and relative ordering ---------------------------------------------------- - -.. note:: - - This section is intended primarily for authors of tools that - automatically process distribution metadata, rather than authors - of Python distributions deciding on a versioning scheme. - -The numeric release component of version identifiers should be sorted in -the same order as Python's tuple sorting when the release number is -parsed as follows:: - - tuple(map(int, release_number.split("."))) - -Within a numeric release (``1.0``, ``2.7.3``), the following suffixes -are permitted and are ordered as shown:: - - .devN, aN, bN, cN, rcN, <no suffix>, .postN - -Note that `rc` will always sort after `c` (regardless of the numeric -component) although they are semantically equivalent. Tools are free to -reject this case as ambiguous and remain in compliance with the PEP. - -Within an alpha (``1.0a1``), beta (``1.0b1``), or release candidate -(``1.0c1``, ``1.0rc1``), the following suffixes are permitted and are -ordered as shown:: - - .devN, <no suffix>, .postN - -Within a post-release (``1.0.post1``), the following suffixes are permitted -and are ordered as shown:: - - .devN, <no suffix> - -Note that ``devN`` and ``postN`` must always be preceded by a dot, even -when used immediately following a numeric version (e.g. ``1.0.dev456``, -``1.0.post1``). - -Within a given suffix, ordering is by the value of the numeric component. - -The following example covers many of the possible combinations:: - - 1.0.dev456 - 1.0a1 - 1.0a2.dev456 - 1.0a12.dev456 - 1.0a12 - 1.0b1.dev456 - 1.0b2 - 1.0b2.post345.dev456 - 1.0b2.post345 - 1.0c1.dev456 - 1.0c1 - 1.0 - 1.0.post456.dev34 - 1.0.post456 - 1.1.dev1 - - -Version ordering across different metadata versions ---------------------------------------------------- - -Metadata v1.0 (PEP 241) and metadata v1.1 (PEP 314) do not -specify a standard version identification or ordering scheme. This PEP does -not mandate any particular approach to handling such versions, but -acknowledges that the de facto standard for ordering them is -the scheme used by the ``pkg_resources`` component of ``setuptools``. - -Software that automatically processes distribution metadata should attempt -to normalize non-compliant version identifiers to the standard scheme, and -ignore them if normalization fails. As any normalization scheme will be -implementation specific, this means that projects using non-compliant -version identifiers may not be handled consistently across different -tools, even when correctly publishing the earlier metadata versions. - -For distributions currently using non-compliant version identifiers, these -filtering guidelines mean that it should be enough for the project to -simply switch to the use of compliant version identifiers to ensure -consistent handling by automated tools. - -Distribution users may wish to explicitly remove non-compliant versions from -any private package indexes they control. - -For metadata v1.2 (PEP 345), the version ordering described in this PEP -should be used in preference to the one defined in PEP 386. - - -Compatibility with other version schemes ----------------------------------------- - -Some projects may choose to use a version scheme which requires -translation in order to comply with the public version scheme defined in -this PEP. In such cases, the `Private-Version`__ field can be used to -record the project specific version as an arbitrary label, while the -translated public version is given in the `Version`_ field. - -__ `Private-Version (optional)`_ - -This allows automated distribution tools to provide consistently correct -ordering of published releases, while still allowing developers to use -the internal versioning scheme they prefer for their projects. - - -Semantic versioning -~~~~~~~~~~~~~~~~~~~ - -`Semantic versioning`_ is a popular version identification scheme that is -more prescriptive than this PEP regarding the significance of different -elements of a release number. Even if a project chooses not to abide by -the details of semantic versioning, the scheme is worth understanding as -it covers many of the issues that can arise when depending on other -distributions, and when publishing a distribution that others rely on. - -The "Major.Minor.Patch" (described in this PEP as "major.minor.micro") -aspects of semantic versioning (clauses 1-9 in the 2.0.0-rc-1 specification) -are fully compatible with the version scheme defined in this PEP, and abiding -by these aspects is encouraged. - -Semantic versions containing a hyphen (pre-releases - clause 10) or a -plus sign (builds - clause 11) are *not* compatible with this PEP -and are not permitted in the public `Version`_ field. - -One possible mechanism to translate such private semantic versions to -compatible public versions is to use the ``.devN`` suffix to specify the -appropriate version order. - -.. _Semantic versioning: http://semver.org/ - - -DVCS based version labels -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Many build tools integrate with distributed version control systems like -Git and Mercurial in order to add an identifying hash to the version -identifier. As hashes cannot be ordered reliably such versions are not -permitted in the public `Version`_ field. - -As with semantic versioning, the public ``.devN`` suffix may be used to -uniquely identify such releases for publication, while the private -version field is used to record the original version label. - - -Date based versions -~~~~~~~~~~~~~~~~~~~ - -As with other incompatible version schemes, date based versions can be -stored in the ``Private-Version`` field. Translating them to a compliant -public version is straightforward: the simplest approach is to subtract -the year before the first release from the major component in the release -number. - - -Version specifiers -================== - -A version specifier consists of a series of version clauses, separated by -commas. Each version clause consists of an optional comparison operator -followed by a version identifier. For example:: - - 0.9, >= 1.0, != 1.3.4, < 2.0 - -Each version identifier must be in the standard format described in -`Version scheme`_. - -The comma (",") is equivalent to a logical **and** operator. - -Whitespace between a conditional operator and the following version -identifier is optional, as is the whitespace around the commas. - - -Compatible release ------------------- - -A compatible release clause omits the comparison operator and matches any -version that is expected to be compatible with the specified version. - -For a given release identifier ``V.N``, the compatible release clause is -approximately equivalent to the pair of comparison clauses:: - - >= V.N, < V+1.dev0 - -where ``V+1`` is the next version after ``V``, as determined by -incrementing the last numeric component in ``V``. For example, -the following version clauses are approximately equivalent:: - - 2.2 - >= 2.2, < 3.dev0 - - 1.4.5 - >= 1.4.5, < 1.5.dev0 - -The difference between the two is that using a compatible release clause -does *not* count as `explicitly mentioning a pre-release`__. - -__ `Handling of pre-releases`_ - -If a pre-release, post-release or developmental release is named in a -compatible release clause as ``V.N.suffix``, then the suffix is ignored -when determining the upper limit of compatibility:: - - 2.2.post3 - >= 2.2.post3, < 3.dev0 - - 1.4.5a4 - >= 1.4.5a4, < 1.5.dev0 - - -Version comparisons -------------------- - -A version comparison clause includes a comparison operator and a version -identifier, and will match any version where the comparison is true. - -Comparison clauses are only needed to cover cases which cannot be handled -with an appropriate compatible release clause, including coping with -dependencies which do not have a robust backwards compatibility policy -and thus break the assumptions of a compatible release clause. - -The defined comparison operators are ``<``, ``>``, ``<=``, ``>=``, ``==``, -and ``!=``. - -The ordered comparison operators ``<``, ``>``, ``<=``, ``>=`` are based -on the consistent ordering defined by the standard `Version scheme`_. - -The ``==`` and ``!=`` operators are based on a kind of prefix comparison - -trailing ``0``s are implied when comparing numeric version parts of different -lengths. ``1.0 == 1 == 1.0.0``, and ``1.0a1 == ``1.0.0.0a1``. - -.. note:: - - The use of ``==`` when defining dependencies for published - distributions is strongly discouraged as it greatly complicates the - deployment of security fixes. The strict version comparison operator - is intended primarily for use when defining dependencies for repeatable - *deployments of applications* while using a shared distribution index. - - -Handling of pre-releases ------------------------- - -Pre-releases of any kind, including developmental releases, are implicitly -excluded from all version specifiers, *unless* a pre-release or developmental -release is explicitly mentioned in one of the clauses. For example, these -specifiers implicitly exclude all pre-releases and development -releases of later versions:: - - 2.2 - >= 1.0 - -While these specifiers would include at least some of them:: - - 2.2.dev0 - 2.2, != 2.3b2 - >= 1.0a1 - >= 1.0c1 - >= 1.0, != 1.0b2 - >= 1.0, < 2.0.dev123 - -Dependency resolution tools should use the above rules by default, but -should also allow users to request the following alternative behaviours: - -* accept already installed pre-releases for all version specifiers -* retrieve and install available pre-releases for all version specifiers - -Dependency resolution tools may also allow the above behaviour to be -controlled on a per-distribution basis. - -Post-releases and purely numeric releases receive no special treatment - -they are always included unless explicitly excluded. - - -Examples --------- - -* ``Requires-Dist: zope.interface (3.1)``: version 3.1 or later, but not - version 4.0 or later. Excludes pre-releases and developmental releases. -* ``Requires-Dist: zope.interface (3.1.0)``: version 3.1.0 or later, but not - version 3.2.0 or later. Excludes pre-releases and developmental releases. -* ``Requires-Dist: zope.interface (==3.1)``: any version that starts - with 3.1, excluding pre-releases and developmental releases. -* ``Requires-Dist: zope.interface (3.1.0,!=3.1.3)``: version 3.1.0 or later, - but not version 3.1.3 and not version 3.2.0 or later. Excludes pre-releases - and developmental releases. For this particular project, this means: "any - version of the 3.1 series but not 3.1.3". This is equivalent to: - ``>=3.1, !=3.1.3, <3.2``. -* ``Requires-Python: 2.6``: Any version of Python 2.6 or 2.7. It - automatically excludes Python 3 or later. -* ``Requires-Python: 3.2, < 3.3``: Specifically requires Python 3.2, - excluding pre-releases. -* ``Requires-Python: 3.3a1``: Any version of Python 3.3+, including - pre-releases like 3.4a1. - - -Depending on distributions that use non-compliant version schemes ------------------------------------------------------------------ - -A distribution using this version of the metadata standard may need to depend -on another distribution using an earlier version of the metadata standard -and a non-compliant versioning scheme. - -The normal ``Requires-Dist`` and ``Setup-Requires-Dist`` fields can be used -for such dependencies, so long as the dependency itself can be expressed -using a compliant version specifier. - -For more exotic dependencies, a metadata extension would be needed in order -to express the dependencies accurately while still obeying the restrictions -on standard version specifiers. The ``Requires-External`` field may also -be used, but would not be as amenable to automatic processing. + This field replaces the old Platform, Requires-Platform and + Requires-Python fields and has been redesigned with environment + marker based semantics that should make it possible to reliably flag, + for example, Unix specific or Windows specific distributions, as well + as Python 2 only and Python 3 only distributions. + + +Metabuild system +================ + +The ``metabuild_hooks`` field is used to define various operations that +may be invoked on a distribution in a platform independent manner. + +The metabuild system currently defines three operations as part of the +deployment of a distribution: + +* Installing to a deployment system +* Uninstalling from a deployment system +* Running the distribution's test suite on a deployment system (hence the + ``test`` runtime extra) + +Distributions may define handles for each of these operations as an +"entry point", a reference to a Python callable, with the module name +separated from the reference within the module by a colon (``:``). + +Example metabuild hooks:: + + "metabuild_hooks": { + "postinstall": "myproject.build_hooks:postinstall", + "preuininstall": "myproject.build_hooks:preuninstall", + "test_installed_dist": "some_test_harness.metabuild_hook" + } + +Build and installation tools MAY offer additional operations beyond the +core metabuild operations. These operations SHOULD be composed from the +defined metabuild operations where appropriate. + +Build and installation tools SHOULD support the legacy ``setup.py`` based +commands for metabuild operations not yet defined as metabuild hooks. + +The metabuild hooks are gathered together into a single top level +``metabuild_hooks`` field. The individual hooks are: + +* ``postinstall``: run after the distribution has been installed to a + target deployment system (or after it has been upgraded). If the hook is + not defined, it indicates no distribution specific actions are needed + following installation. +* ``preuninstall``: run before the distribution has been uninstalled from a + deployment system (or before it is upgraded). If the hook is not defined, + it indicates no distribution specific actions are needed prior to + uninstallation. +* ``test_installed_dist``: test an installed distribution is working. If the + hook is not defined, it indicates the distribution does not support + execution of the test suite after deployment. + +The expected signatures of these hooks are as follows:: + + def postinstall(current_meta, previous_meta=None): + """Run following installation or upgrade of the distribution + + *current_meta* is the distribution metadata for the version now + installed on the current system + *previous_meta* is either missing or ``None`` (indicating a fresh + install) or else the distribution metadata for the version that + was previously installed (indicating an upgrade or downgrade). + """ + + def preuninstall(current_meta, next_meta=None): + """Run prior to uninstallation or upgrade of the distribution + + *current_meta* is the distribution metadata for the version now + installed on the current system + *next_meta* is either missing or ``None`` (indicating complete + uninstallation) or else the distribution metadata for the version + that is about to be installed (indicating an upgrade or downgrade). + """ + + def test_installed_dist(current_meta): + """Check an installed distribution is working correctly + + Note that this check should always be non-destructive as it may be + invoked automatically by some tools. + + Requires that the distribution's test dependencies be installed + (indicated by the ``test`` runtime extra). + + Returns ``True`` if the check passes, ``False`` otherwise. + """ + +Metabuild hooks MUST be called with at least abbreviated metadata, and MAY +be called with full metadata. + +Where necessary, metabuild hooks check for the presence or absence of +optional dependencies defined as extras using the same techniques used +during normal operation of the distribution (for example, checking for +import failures for optional dependencies). + + +Metadata Extensions +=================== + +Extensions to the metadata may be present in a mapping under the +'extensions' key. The keys must meet the same restrictions as +distribution names, while the values may be any type natively supported +in JSON:: + + "extensions" : { + "chili" : { "type" : "Poblano", "heat" : "Mild" }, + "languages" : [ "French", "Italian", "Hebrew" ] + } + +To avoid name conflicts, it is recommended that distribution names be used +to identify metadata extensions. This practice will also make it easier to +find authoritative documentation for metadata extensions. + + +Extras (optional dependencies) +============================== + +Extras are additional dependencies that enable an optional aspect +of the distribution, generally corresponding to a ``try: import +optional_dependency ...`` block in the code. To support the use of the +distribution with or without the optional dependencies they are listed +separately from the distribution's core dependencies and must be requested +explicitly, either in the dependency specifications of another distribution, +or else when issuing a command to an installation tool. + +The names of extras MUST abide by the same restrictions as those for +distribution names. + +Example of a distribution with optional dependencies:: + + "name": "ComfyChair", + "extras": ["warmup", "c-accelerators"] + "may_require": [ + { + "dependencies": ["SoftCushions"], + "extra": "warmup" + } + ] + "build_may_require": [ + { + "dependencies": ["cython"], + "extra": "c-accelerators" + } + ] + +Other distributions require the additional dependencies by placing the +relevant extra names inside square brackets after the distribution name when +specifying the dependency. + +Extra specifications MUST support the following additional syntax: + +* Multiple features can be requested by separating them with a comma within + the brackets. +* All explicitly defined extras may be requested with the ``*`` wildcard + character. Note that this does NOT request the implicitly defined + ``test`` extra - that must always be requested explicitly when it is + desired. +* Extras may be explicitly excluded by prefixing their name with a hyphen. + +Command line based installation tools SHOULD support this same syntax to +allow extras to be requested explicitly. + +The full set of dependency requirements is then based on the top level +dependencies, along with those of any requested extras. + +Example:: + + "requires": ["ComfyChair[warmup]"] + -> requires ``ComfyChair`` and ``SoftCushions`` at run time + + "requires": ["ComfyChair[*]"] + -> requires ``ComfyChair`` and ``SoftCushions`` at run time, but + will also pick up any new optional dependencies other than those + needed solely to run the tests Environment markers =================== -An **environment marker** is a marker that can be added at the end of a -field after a semi-colon (";"), to add a condition about the execution -environment. - -Here are some example of fields using such markers:: - - Requires-Dist: pywin32 (>1.0); sys.platform == 'win32' - Requires-Dist: foo (1,!=1.3); platform.machine == 'i386' - Requires-Dist: bar; python_version == '2.4' or python_version == '2.5' - Requires-External: libxslt; 'linux' in sys.platform +An **environment marker** describes a condition about the current execution +environment. They are used to indicate when certain dependencies are only +required in particular environments, and to indicate supported platforms +for distributions with additional constraints beyond the availability of a +Python runtime. + +Here are some examples of such markers:: + + "sys_platform == 'win32'" + "platform_machine == 'i386'" + "python_version == '2.4' or python_version == '2.5'" + "'linux' in sys_platform" + +And here's an example of some conditional metadata for a distribution that +requires PyWin32 both at runtime and buildtime when using Windows:: + + "name": "ComfyChair", + "may_require": [ + { + "dependencies": ["pywin32 (>1.0)"], + "environment": "sys.platform == 'win32'" + } + ] + "build_may_require": [ + { + "dependencies": ["pywin32 (>1.0)"], + "environment": "sys.platform == 'win32'" + } + ] The micro-language behind this is a simple subset of Python: it compares only strings, with the ``==`` and ``in`` operators (and their opposites), @@ -1180,73 +1486,35 @@ MARKER: EXPR [(and|or) EXPR]* EXPR: ("(" MARKER ")") | (SUBEXPR [(in|==|!=|not in) SUBEXPR]) -where ``SUBEXPR`` belongs to any of the following (the details after the -colon in each entry define the value represented by that subexpression): - -* ``python_version``: '%s.%s' % (sys.version_info[0], sys.version_info[1]) -* ``python_full_version``: sys.version.split()[0] -* ``os.name````: os.name -* ``sys.platform````: sys.platform -* ``platform.version``: platform.version() -* ``platform.machine``: platform.machine() -* ``platform.python_implementation``: = platform.python_implementation() -* ``extra``: (name of requested feature) or None -* ``'text'``: a free string, like ``'2.4'``, or ``'win32'`` - -Notice that ``in`` and ``not in`` are restricted to strings, meaning that it -is not possible to use other sequences like tuples or lists on the right -side. - -The fields that benefit from this marker are: - -* ``Requires-Python`` -* ``Requires-External`` -* ``Requires-Dist`` -* ``Provides-Dist`` -* ``Classifier`` - - -Optional features -================= - -Distributions may use the ``Provides-Extra`` field to declare additional -features provided through "extra" dependencies, usually corresponding to a -``try: import x ...`` block. Environment markers may be used to indicate -that particular dependencies are needed only when a particular optional -feature has been requested. - -Other distributions require the feature by placing it inside -square brackets after the distribution name when declaring the -dependency. Multiple features can be requisted by separating them with -a comma within the brackets. - -The full set of dependency requirements is then the union of the sets -created by first evaluating the `Requires-Dist` fields with `extra` -set to `None` and then to the name of each requested feature. - -Example:: - - Requires-Dist: beaglevote[pdf] - -> requires beaglevote, reportlab at run time - - Setup-Requires-Dist: beaglevote[test, doc] - -> requires beaglevote, sphinx, nose at setup time - -It is an error to request a feature name that has not been declared with -`Provides-Extra` but it is legal to specify `Provides-Extra` without -referencing it in any `Requires-Dist`. - -In the example, if ``beaglevote`` grew the ability to generate PDF -without extra dependencies, it should continue to ``Provides-Extra: -pdf`` for the benefit of dependent distributions. - -The following feature names are implicitly defined for all distributions: - -- `test`: dependencies that are needed in order to run automated tests -- `doc`: dependencies that are needed in order to generate documentation - -Listing these implicit features explicitly in a ``Provides-Extra`` field is -permitted, but not required. +where ``SUBEXPR`` is either a Python string (such as ``'2.4'``, or +``'win32'``) or one of the following marker variables: + +* ``python_version``: ``'{0.major}.{0.minor}'.format(sys.version_info)`` +* ``python_full_version``: see definition below +* ``os_name````: ``os.name`` +* ``sys_platform````: ``sys.platform`` +* ``platform_version``: ``platform.version()`` +* ``platform_machine``: ``platform.machine()`` +* ``platform_python_implementation``: ``platform.python_implementation()`` + +Note that all subexpressions are restricted to strings or one of the +marker variable names, meaning that it is not possible to use other +sequences like tuples or lists on the right side of the ``in`` and +``not in`` operators. + +Unlike Python, chaining of comparison operations is NOT permitted in +environment markers. + +The ``python_full_version`` marker variable is derived from +``sys.version_info()`` in accordance with the following algorithm:: + + def format_full_version(): + info = sys.version_info + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version Updating the metadata specification @@ -1266,67 +1534,52 @@ * Metadata-Version is now 2.0, with semantics specified for handling version changes -* Most fields are now optional +* The increasingly complex ad hoc "Key: Value" format has been replaced by + a more structured JSON compatible format that is easily represented as + Python dictionaries, strings, lists. + +* Most fields are now optional and filling in dummy data for omitted fields + is explicitly disallowed * Explicit permission for in-place clarifications without releasing a new version of the specification -* General reformatting of the PEP to make it easier to read - -* Values are now expected to be UTF-8 - -* Changed the version scheme - - * added the new ``Private-Version`` field - * changed the top level sort position of the ``.devN`` suffix - * allowed single value version numbers - * explicit exclusion of leading or trailing whitespace - * explicit criterion for the exclusion of date based versions - * incorporated the version scheme directly into the PEP - -* Changed interpretation of version specifiers - - * implicitly exclude pre-releases unless explicitly requested - * treat post releases the same way as unqualified releases - -* Discuss ordering and dependencies across metadata versions - -* Clarify use of parentheses for grouping in environment marker - pseudo-grammar - -* Support for packaging, build and installation dependencies - - * the new ``Setup-Requires-Dist`` field - -* Optional feature mechanism - - * the new ``Provides-Extra`` field - * ``extra`` expression defined for environment markers - * optional feature support in ``Requires-Dist`` - -* Metadata extension mechanism - - * the new ``Extension`` field and extension specific fields +* The PEP now attempts to provide more of an explanation of *why* the fields + exist and how they are intended to be used, rather than being a simple + description of the permitted contents + +* Changed the version scheme to be based on PEP 440 rather than PEP 386 + +* Added the build label mechanism as described in PEP 440 + +* Support for different development, build, test and deployment dependencies + +* The "Extras" optional dependency mechanism + +* A well-defined metadata extension mechanism + +* Metabuild hook system + +* Clarify and simplify various aspects of environment markers: + + * allow use of parentheses for grouping in the pseudo-grammar + * consistently use underscores instead of periods in the variable names + * clarify that chained comparisons are not permitted + +* More flexible system for defining contact points and contributors + +* Defined a recommended set of project URLs + +* New system for defining supported environments * Updated obsolescence mechanism - * the new ``Obsoleted-By`` field - * the ``Obsoletes-Dist`` field has been removed - -* Simpler description format - - * the ``Description`` field is now deprecated - * A payload (containing the description) may appear after the headers. - -* Other changed fields: - - - ``Requires-Python`` (explicitly flagged as multiple use) - - ``Project-URL`` (commas permitted in labels) - -* Clarified fields: - - - ``Provides-Dist`` - - ``Keywords`` +* Added "License URL" field + +* Explicit declaration of description markup format + +* With all due respect to Charles Schulz and Peanuts, many of the examples + have been updated to be more `thematically appropriate`_ for Python ;) The rationale for major changes is given in the following sections. @@ -1342,8 +1595,8 @@ may include changes that are not compatible with existing tools. The major version number of the specification has been incremented -accordingly, as interpreting PEP 426 metadata in accordance with earlier -metadata specifications is unlikely to give the expected behaviour. +accordingly, as interpreting PEP 426 metadata obviously cannot be +interpreted in accordance with earlier metadata specifications. Whenever the major version number of the specification is incremented, it is expected that deployment will take some time, as either metadata @@ -1355,219 +1608,158 @@ Existing tools won't abide by this guideline until they're updated to support the new metadata standard, so the new semantics will first take effect for a hypothetical 2.x -> 3.0 transition. For the 1.x -> 2.0 -transition, it is recommended that tools continue to produce the +transition, we will use the approach where tools continue to produce the existing supplementary files (such as ``entry_points.txt``) in addition to any equivalents specified using the new features of the standard metadata format (including the formal extension mechanism). -Standard encoding and other format clarifications -------------------------------------------------- - -Several aspects of the file format, including the expected file encoding, -were underspecified in previous versions of the metadata standard. To -make it easier to develop interoperable tools, these details are now -explicitly specified. +Switching to a JSON compatible format +------------------------------------- + +The old "Key:Value" format was becoming increasingly limiting, with various +complexities like parsers needing to know which fields were permitted to +occur more than once, which fields supported the environment marker +syntax (with an optional ``";"`` to separate the value from the marker) and +eventually even the option to embed arbitrary JSON inside particular +subfields. + +The old serialisation format also wasn't amenable to easy conversion to +standard Python data structures for use in the new metabuild hook APIs, or +in future extensions to the importer APIs to allow them to provide +information for inclusion in the installation database. + +Accordingly, we've taken the step of switching to a JSON-compatible metadata +format. This works better for APIs and is much easier for tools to parse and +generate correctly. Changing the name of the metadata file also makes it +easy to distribute 1.x and 2.x metadata in parallel, greatly simplifying +several aspects of the migration to the new metadata format. Changing the version scheme --------------------------- -The new ``Private-Version`` field is intended to make it clearer that the -constraints on public version identifiers are there primarily to aid in -the creation of reliable automated dependency analysis tools. Projects -are free to use whatever versioning scheme they like internally, so long -as they are able to translate it to something the dependency analysis tools -will understand. - -The key change in the version scheme in this PEP relative to that in -PEP 386 is to sort top level developmental releases like ``X.Y.devN`` ahead -of alpha releases like ``X.Ya1``. This is a far more logical sort order, as -projects already using both development releases and alphas/betas/release -candidates do not want their developmental releases sorted in -between their release candidates and their full releases. There is no -rationale for using ``dev`` releases in that position rather than -merely creating additional release candidates. - -The updated sort order also means the sorting of ``dev`` versions is now -consistent between the metadata standard and the pre-existing behaviour -of ``pkg_resources`` (and hence the behaviour of current installation -tools). - -Making this change should make it easier for affected existing projects to -migrate to the latest version of the metadata standard. - -Another change to the version scheme is to allow single number -versions, similar to those used by non-Python projects like Mozilla -Firefox, Google Chrome and the Fedora Linux distribution. This is actually -expected to be more useful for version specifiers (allowing things like -the simple ``Requires-Python: 3`` rather than the more convoluted -``Requires-Python: >= 3.0, < 4``), but it is easier to allow it for both -version specifiers and release numbers, rather than splitting the -two definitions. - -The exclusion of leading and trailing whitespace was made explicit after -a couple of projects with version identifiers differing only in a -trailing ``\n`` character were found on PyPI. - -The exclusion of major release numbers that looks like dates was implied -by the overall text of PEP 386, but not clear in the definition of the -version scheme. This exclusion has been made clear in the definition of -the release component. - -Finally, as the version scheme in use is dependent on the metadata -version, it was deemed simpler to merge the scheme definition directly into -this PEP rather than continuing to maintain it as a separate PEP. - -`Appendix B` shows detailed results of an analysis of PyPI distribution -version information, as collected on 19th February, 2013. This analysis -compares the behaviour of the explicitly ordered version schemes defined in -this PEP and PEP 386 with the de facto standard defined by the behaviour -of setuptools. These metrics are useful, as the intent of both PEPs is to -follow existing setuptools behaviour as closely as is feasible, while -still throwing exceptions for unorderable versions (rather than trying -to guess an appropriate order as setuptools does). - -Overall, the percentage of compatible distributions improves from 97.7% -with PEP 386 to 98.7% with this PEP. While the number of projects affected -in practice was small, some of the affected projects are in widespread use -(such as Pinax and selenium). The surprising ordering discrepancy also -concerned developers and acted as an unnecessary barrier to adoption of -the new metadata standard. - -The data also shows that the pre-release sorting discrepancies are seen -only when analysing *all* versions from PyPI, rather than when analysing -public versions. This is largely due to the fact that PyPI normally reports -only the most recent version for each project (unless maintainers -explicitly configure their project to display additional versions). However, -installers that need to satisfy detailed version constraints often need -to look at all available versions, as they may need to retrieve an older -release. - -Even this PEP doesn't completely eliminate the sorting differences relative -to setuptools: - -* Sorts differently (after translations): 38 / 28194 (0.13 %) -* Sorts differently (no translations): 2 / 28194 (0.01 %) - -The two remaining sort order discrepancies picked up by the analysis are due -to a pair of projects which have PyPI releases ending with a carriage -return, alongside releases with the same version number, only *without* the -trailing carriage return. - -The sorting discrepancies after translation relate mainly to differences -in the handling of pre-releases where the standard mechanism is considered -to be an improvement. For example, the existing pkg_resources scheme will -sort "1.1beta1" *after* "1.1b2", whereas the suggested standard translation -for "1.1beta1" is "1.1b1", which sorts *before* "1.1b2". Similarly, the -pkg_resources scheme will sort "-dev-N" pre-releases differently from -"devN" pre-releases when they occur within the same release, while the -standard scheme will normalize both representations to ".devN" and sort -them by the numeric component. - - -A more opinionated description of the versioning scheme -------------------------------------------------------- - -As in PEP 386, the primary focus is on codifying existing practices to make -them more amenable to automation, rather than demanding that existing -projects make non-trivial changes to their workflow. However, the -standard scheme allows significantly more flexibility than is needed -for the vast majority of simple Python packages (which often don't even -need maintenance releases - many users are happy with needing to upgrade to a -new feature release to get bug fixes). - -For the benefit of novice developers, and for experienced developers -wishing to better understand the various use cases, the specification -now goes into much greater detail on the components of the defined -version scheme, including examples of how each component may be used -in practice. - -The PEP also explicitly guides developers in the direction of -semantic versioning (without requiring it), and discourages the use of -several aspects of the full versioning scheme that have largely been -included in order to cover esoteric corner cases in the practices of -existing projects and in repackaging software for Linux distributions. - - -Changing the interpretation of version specifiers -------------------------------------------------- - -The previous interpretation of version specifiers made it very easy to -accidentally download a pre-release version of a dependency. This in -turn made it difficult for developers to publish pre-release versions -of software to the Python Package Index, as even marking the package as -hidden wasn't enough to keep automated tools from downloading it, and also -made it harder for users to obtain the test release manually through the -main PyPI web interface. - -The previous interpretation also excluded post-releases from some version -specifiers for no adequately justified reason. - -The updated interpretation is intended to make it difficult to accidentally -accept a pre-release version as satisfying a dependency, while allowing -pre-release versions to be explicitly requested when needed. - -The "some forward compatibility assumed" default version constraint is -taken directly from the Ruby community's "pessimistic version constraint" -operator [4]_ to allow projects to take a cautious approach to forward -compatibility promises, while still easily setting a minimum required -version for their dependencies. It is made the default behaviour rather -than needing a separate operator in order to explicitly discourage -overspecification of dependencies by library developers. The explicit -comparison operators remain available to cope with dependencies with -unreliable or non-existent backwards compatibility policies. - - -Packaging, build and installation dependencies +See PEP 440 for a detailed rationale for the various changes made to the +versioning scheme. + + +Build labels +------------ + +See PEP 440 for the rationale behind the addition of this field. + + +Development, build and deployment dependencies ---------------------------------------------- -The new ``Setup-Requires-Dist`` field allows a distribution to indicate when -a dependency is needed to package, build or install the distribution, rather -than being needed to run the software after installation. - -This should allow distribution tools to effectively support a wider range of -distribution requirements. - - -Support for optional features of distributions ----------------------------------------------- - -The new ``Provides-Extra`` field allows distributions to declare optional -features, and to use environment markers to reduce their dependencies -when those features are not requested. Environment markers may also be -used to require a later version of Python when particular features are -requested. - -The ``Requires-Dist`` and ``Setup-Requires-Dist`` fields then allow -distributions to require optional features of other distributions. - -The ``test`` and ``doc`` features are implicitly defined for all -distributions, as one key motivation for this feature is to encourage -distributions to explicitly declare the dependencies needed to run -their automatic tests, or build their documentation, without demanding those -dependencies be present in order to merely install or use the software. +The separation of the ``requires``, ``build_requires`` and ``dev_requires`` +fields allow a distribution to indicate whether a dependency is needed +specifically to develop, build or deploy the distribution. + +As distribution metadata improves, this should allow much greater control +over where particular dependencies end up being installed . + + +Support for optional dependencies for distributions +--------------------------------------------------- + +The new extras system allows distributions to declare optional +features, and to use the ``may_require`` and ``build_may_require`` fields +to indicate when particular dependencies are needed only to support those +features. It is derived from the equivalent system that is already in +widespread use as part of ``setuptools`` and allows that aspect of the +legacy ``setuptools`` metadata to be accurately represented in the new +metadata format. + +The ``test`` extra is implicitly defined for all distributions, as it +ties in with the new metabuild hook offering a standard way to request +execution of a distribution's test suite. Identifying test suite +dependencies is already one of the most popular uses of the extras system +in ``setuptools``. Support for metadata extensions ------------------------------- -The new ``Extension`` field effectively allows sections of the metadata +The new extension effectively allows sections of the metadata namespace to be delegated to other distributions, while preserving a standard overal format metadata format for easy of processing by distribution tools that do not support a particular extension. -It also works well in combination with the new ``Setup-Requires-Dist`` field +It also works well in combination with the new ``build_requires`` field to allow a distribution to depend on tools which *do* know how to handle -the chosen extension, and the new optional features mechanism, allowing -support for particular extensions to be provided as optional features. +the chosen extension, and the new extras mechanism, allowing support for +particular extensions to be provided as optional features. + + +Support for metabuild hooks +--------------------------- + +The new metabuild system is designed to allow the wheel format to fully +replace direct installation on deployment targets, by allows projects like +Twisted to still execute code following installation from a wheel file. + +Falling back to invoking ``setup.py`` directly rather than using a +metabuild hook will remain an option when relying on version 1.x metadata, +and is also used as the interim solution for installation from source +archives. + +The ``test_installed_dist`` metabuild hook is included as a complement to +the ability to explicitly specify test dependencies. + + +Changes to environment markers +------------------------------ + +The changes to environment markers were just clarifications and +simplifications to make them easier to use. + +The arbitrariness of the choice of ``.`` and ``_`` in the different +variables was addressed by standardising on ``_`` (as these are predefined +variables rather than live references into the Python module namespace) + +The use of parentheses for grouping and the disallowance of chained +comparisons were added to address some underspecified behaviour in the +previous version of the specification. + + +Updated contact information +--------------------------- + +The switch to JSON made it possible to provide a more flexible +system for defining multiple contact points for a project, as well as +listing other contributors. + +The ``type`` concept allows for preservation of the distinction between +the original author of a project, and a lead maintainer that takes over +at a later date. + + +Changes to project URLs +----------------------- + +In addition to allow arbitrary strings as project URL labels, the new +metadata standard also defines a recommend set of four URL labels for +a distribution's home page, documentation, source control and issue tracker. + + +Changes to platform support +--------------------------- + +The new environment marker system makes it possible to define supported +platforms in a way that is actually amenable to automated processing. This +has been used to replace several older fields with poorly defined semantics. + +For the moment, the old ``Requires-External`` field has been removed +entirely. Possible replacements may be explored through the metadata +extension mechanism. Updated obsolescence mechanism ------------------------------ The marker to indicate when a project is obsolete and should be replaced -has been moved to the obsolete project (the new ``Obsoleted-By`` field), +has been moved to the obsolete project (the new ``obsoleted_by`` field), replacing the previous marker on the replacement project (the removed ``Obsoletes-Dist`` field). @@ -1578,19 +1770,228 @@ is not widely supported, and so removing it does not present any significant barrier to tools and projects adopting the new metadata format. - -Simpler description format +Explicit markup for description +------------------------------- + +Currently, PyPI attempts to detect the markup format by rendering it as +reStructuredText, and if that fails, treating it as plain text. Allowing +the intended format to be stated explicitly will allow this guessing to be +removed, and more informative error reports to be provided to users when +a rendering error occurs. + +This is especially necessary since PyPI applies additional restrictions to +the rendering process for security reasons, thus a description that renders +correctly on a developer's system may still fail to render on the server. + + +Deferred features +================= + +Several potentially useful features have been deliberately deferred in +order to better prioritise our efforts in migrating to the new metadata +standard. These all reflect information that may be nice to have in the +new metadata, but which can be readily added in metadata 2.1 without +breaking any use cases already supported by metadata 2.0. + +Once the ``pypi``, ``setuptools``, ``pip`` and ``distlib`` projects +support creation and consumption of metadata 2.0, then we may revisit +the creation of metadata 2.1 with these additional features. + +.. note:: + + Given the nature of this PEP as an interoperability specification, + this section will probably be removed before the PEP is accepted. + However, it's useful to have it here while discussion is ongoing. + + +String methods in environment markers +------------------------------------- + +Supporting at least ".startswith" and ".endswith" string methods in +environment markers would allow some conditions to be written more +naturally. For example, ``"sys.platform.startswith('win')"`` is a +somewhat more intuitive way to mark Windows specific dependencies, +since ``"'win' in sys.platform"`` is incorrect thanks to ``cygwin`` +and the fact that 64-bit Windows still shows up as ``win32`` is more +than a little strange. + + +Module listing +-------------- + +A top level ``"module"`` key, referencing a list of strings, with each +giving the fully qualified name of a public package or module provided +by the distribution. + +A flat list would be used in order to correctly accommodate namespace +packages (where a distribution may provide subpackages or submodules without +explicitly providing the parent namespace package). + +Example:: + + "modules": [ + "comfy.chair" + ] + +Explicitly providing a list of public module names will likely help +with enabling features in RPM like "Requires: python(requests)", as well +as providing richer static metadata for analysis from PyPI. + +However, this is just extra info that doesn't impact installing from wheels, +so it is a good candidate for postponing to metadata 2.1. + + +Additional metabuild hooks -------------------------- -Distribution descriptions are often quite long, sometimes including a -short guide to using the module. Moving them into the file payload allows -them to be formatted neatly as reStructuredText without needing to -carefully avoid the introduction of a blank line that would terminate -the header section. - -The ``Description`` header is deprecated rather than removed to support -easier conversion of existing tools and projects to the new metadata -format. +The following draft metabuild operations have been deferred for now: + +* Generating the metadata file on a development system +* Generating a source archive on a development system +* Generating a binary archive on a build system + +Metadata 2.0 deliberately focuses on wheel based installation, leaving +tarball and sdist based installation to use the existing ``setup.py`` +based ``distutils`` command interface. + +In the meantime, the above four operations will continue to be handled +through the ``distutils``/``setuptools`` command system: + +* ``python setup.py dist_info`` +* ``python setup.py sdist`` +* ``python setup.py bdist_wheel`` + +The following additional metabuild hooks may be added in metadata 2.1 to +cover these operations without relying on ``setup.py``: + +* ``make_dist_info``: generate the source archive's dist_info directory +* ``make_sdist``: construct a source archive +* ``build_wheel``: construct a binary wheel archive from an sdist source + archive + +Tentative signatures have been designed for those hooks, but they will +not be pursued further until 2.1 (note that the current signatures for +the hooks do *not* adequately handle the "extras" concept):: + + def make_dist_info(source_dir, info_dir): + """Generate the contents of dist_info for an sdist archive + + *source_dir* points to a source checkout or unpacked tarball + *info_dir* is the destination where the sdist metadata files should + be written + + Returns the distribution metadata as a dictionary. + """ + + def make_sdist(source_dir, contents_dir, info_dir): + """Generate the contents of an sdist archive + + *source_dir* points to a source checkout or unpacked tarball + *contents_dir* is the destination where the sdist contents should be + written (note that archiving the contents is the responsibility of + the metabuild tool rather than the hook function) + *info_dir* is the destination where the sdist metadata files should + be written + + Returns the distribution metadata as a dictionary. + """ + + def build_wheel(sdist_dir, contents_dir, info_dir, compatibility=None): + """Generate the contents of a wheel archive + + *source_dir* points to an unpacked source archive + *contents_dir* is the destination where the wheel contents should be + written (note that archiving the contents is the responsibility of + the metabuild tool rather than the hook function) + *info_dir* is the destination where the wheel metadata files should + be written + *compatibility* is an optional PEP 425 compatibility tag indicating + the desired target compatibility for the build. If the tag cannot + be satisfied, the hook should throw ``ValueError``. + + Returns the actual compatibility tag for the build + """ + + +Rejected Features +================= + +The following features have been explicitly considered and rejected as +introducing too much additional complexity for too small a gain in +expressiveness. + +.. note:: + + Given the nature of this PEP as an interoperability specification, + this section will probably be removed before the PEP is accepted. + However, it's useful to have it here while discussion is ongoing. + + +Detached metadata +----------------- + +Rather than allowing some large items (such as the description field) to +be distributed separately, this PEP instead defines two metadata subsets +that should support more reasonable caching and API designs (for example, +only the essential dependency resolution metadata would be distributed +through TUF, and it is entirely possible the updated sdist, wheel and +installation database specs will use the abbreviated metadata, leaving +the full metadata as the province of index servers). + + +Alternative dependencies +------------------------ + +An earlier draft of this PEP considered allowing lists in place of the +usual strings in dependency specifications to indicate that there aren +multiple ways to satisfy a dependency. + +If at least one of the individual dependencies was already available, then +the entire dependency would be considered satisfied, otherwise the first +entry would be added to the dependency set. + +Alternative dependency specification example:: + + ["Pillow", "PIL"] + ["mysql", "psycopg2 (>= 4)", "sqlite3"] + +However, neither of the given examples is particularly compelling, +since Pillow/PIL style forks aren't common, and the database driver use +case would arguably be better served by an SQL Alchemy defined "supported +database driver" metadata extension where a project depends on SQL Alchemy, +and then declares in the extension which database drivers are checked for +compatibility by the upstream project (similar to the advisory +``supports-platform`` field in the main metadata). + +We're also getting better support for "virtual provides" in this version of +the metadata standard, so this may end up being an installer and index +server problem to better track and publish those. + + +Compatible release comparisons in environment markers +----------------------------------------------------- + +PEP 440 defines a rich syntax for version comparisons that could +potentially be useful with ``python_version`` and ``python_full_version`` +in environment markers. However, allowing the full syntax would mean +environment markers are no longer a Python subset, while allowing +only some of the comparisons would introduce yet another special case +to handle. + +Given that environment markers are only used in cases where a higher level +"or" is implied by the metadata structure, it seems easier to require the +use of multiple comparisons against specific Python versions for the rare +cases where this would be useful. + + +Conditional provides +-------------------- + +Under the revised metadata design, conditional "provides" based on runtime +features or the environment would go in a separate "may_provide" field. +However, I'm not convinced there's a great use case for that, so the idea +is rejected unless someone can present a compelling use case (and even then +the idea wouldn't be reconsidered until metadata 2.1 at the earliest). References @@ -1607,145 +2008,14 @@ .. [1] reStructuredText markup: http://docutils.sourceforge.net/ -.. _`Python Package Index`: http://pypi.python.org/pypi/ +.. _Python Package Index: http://pypi.python.org/pypi/ .. [2] PEP 301: http://www.python.org/dev/peps/pep-0301/ -.. [3] Version compatibility analysis script: - http://hg.python.org/peps/file/default/pep-0426/pepsort.py - -.. [4] Pessimistic version constraint - http://docs.rubygems.org/read/chapter/16 - -Appendix A -========== - -The script used for this analysis is available at [3]_. - -Parsing and generating the Metadata 2.0 serialization format using -Python 3.3:: - - # Metadata 2.0 demo - from email.generator import Generator - from email import header - from email.parser import Parser - from email.policy import Compat32 - from email.utils import _has_surrogates - - class MetadataPolicy(Compat32): - max_line_length = 0 - continuation_whitespace = '\t' - - def _sanitize_header(self, name, value): - if not isinstance(value, str): - return value - if _has_surrogates(value): - raise NotImplementedError() - else: - return value - - def _fold(self, name, value, sanitize): - body = ((self.linesep+self.continuation_whitespace) - .join(value.splitlines())) - return ''.join((name, ': ', body, self.linesep)) - - if __name__ == "__main__": - import sys - import textwrap - - pkg_info = """\ - Metadata-Version: 2.0 - Name: package - Version: 0.1.0 - Summary: A package. - Description: Description - =========== - - A description of the package. - - """ - - m = Parser(policy=MetadataPolicy()).parsestr(pkg_info) - - m['License'] = 'GPL' - description = m['Description'] - description_lines = description.splitlines() - m.set_payload(description_lines[0] - + '\n' - + textwrap.dedent('\n'.join(description_lines[1:])) - + '\n') - del m['Description'] - - # Correct if sys.stdout.encoding == 'UTF-8': - Generator(sys.stdout, maxheaderlen=0).flatten(m) - -Appendix B -========== - -Metadata v2.0 guidelines versus setuptools:: - - $ ./pepsort.py - Comparing PEP 426 version sort to setuptools. - - Analysing release versions - Compatible: 24477 / 28194 (86.82 %) - Compatible with translation: 247 / 28194 (0.88 %) - Compatible with filtering: 84 / 28194 (0.30 %) - No compatible versions: 420 / 28194 (1.49 %) - Sorts differently (after translations): 0 / 28194 (0.00 %) - Sorts differently (no translations): 0 / 28194 (0.00 %) - No applicable versions: 2966 / 28194 (10.52 %) - - Analysing public versions - Compatible: 25600 / 28194 (90.80 %) - Compatible with translation: 1505 / 28194 (5.34 %) - Compatible with filtering: 13 / 28194 (0.05 %) - No compatible versions: 420 / 28194 (1.49 %) - Sorts differently (after translations): 0 / 28194 (0.00 %) - Sorts differently (no translations): 0 / 28194 (0.00 %) - No applicable versions: 656 / 28194 (2.33 %) - - Analysing all versions - Compatible: 24239 / 28194 (85.97 %) - Compatible with translation: 2833 / 28194 (10.05 %) - Compatible with filtering: 513 / 28194 (1.82 %) - No compatible versions: 320 / 28194 (1.13 %) - Sorts differently (after translations): 38 / 28194 (0.13 %) - Sorts differently (no translations): 2 / 28194 (0.01 %) - No applicable versions: 249 / 28194 (0.88 %) - -Metadata v1.2 guidelines versus setuptools:: - - $ ./pepsort.py 386 - Comparing PEP 386 version sort to setuptools. - - Analysing release versions - Compatible: 24244 / 28194 (85.99 %) - Compatible with translation: 247 / 28194 (0.88 %) - Compatible with filtering: 84 / 28194 (0.30 %) - No compatible versions: 648 / 28194 (2.30 %) - Sorts differently (after translations): 0 / 28194 (0.00 %) - Sorts differently (no translations): 0 / 28194 (0.00 %) - No applicable versions: 2971 / 28194 (10.54 %) - - Analysing public versions - Compatible: 25371 / 28194 (89.99 %) - Compatible with translation: 1507 / 28194 (5.35 %) - Compatible with filtering: 12 / 28194 (0.04 %) - No compatible versions: 648 / 28194 (2.30 %) - Sorts differently (after translations): 0 / 28194 (0.00 %) - Sorts differently (no translations): 0 / 28194 (0.00 %) - No applicable versions: 656 / 28194 (2.33 %) - - Analysing all versions - Compatible: 23969 / 28194 (85.01 %) - Compatible with translation: 2789 / 28194 (9.89 %) - Compatible with filtering: 530 / 28194 (1.88 %) - No compatible versions: 547 / 28194 (1.94 %) - Sorts differently (after translations): 96 / 28194 (0.34 %) - Sorts differently (no translations): 14 / 28194 (0.05 %) - No applicable versions: 249 / 28194 (0.88 %) +.. _thematically appropriate: https://www.youtube.com/watch?v=CSe38dzJYkY + +.. _TR39: http://www.unicode.org/reports/tr39/tr39-1.html#Confusable_Detection Copyright diff --git a/pep-0440.txt b/pep-0440.txt --- a/pep-0440.txt +++ b/pep-0440.txt @@ -9,82 +9,52 @@ Type: Standards Track Content-Type: text/x-rst Created: 18 Mar 2013 -Post-History: 30 Mar 2013 +Post-History: 30 Mar 2013, 27-May-2013 Replaces: 386 Abstract ======== -This PEP describes a scheme for identifying versions of Python -software distributions, and declaring dependencies on particular -versions. +This PEP describes a scheme for identifying versions of Python software +distributions, and declaring dependencies on particular versions. -This document addresses several limitations of the previous attempt at -a standardised approach to versioning, as described in PEP 345 and PEP -386. +This document addresses several limitations of the previous attempt at a +standardised approach to versioning, as described in PEP 345 and PEP 386. .. note:: - This PEP has been broken out of the metadata 2.0 specification in - PEP 426 and refers to some details that will be in the *next* - version of PEP 426. + This PEP was broken out of the metadata 2.0 specification in PEP 426. + + Unlike PEP 426, the notes that remain in this document are intended as + part of the final specification. Definitions =========== The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", -"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this +"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119. -"Distributions" are deployable software components published through -an index server or otherwise made available for installation. +The following terms are to be interpreted as described in PEP 426: -"Versions" are uniquely identified snapshots of a distribution. - -"Distribution archives" are the packaged files which are used to -publish and distribute the software. "Source archives" require a -build system to be available on the target system, while "binary -archives" only require that prebuilt files be moved to the correct -location on the target system. As Python is a dynamically bound -cross-platform language, many "binary" archives will contain only pure -Python source code. - -"Build tools" are automated tools intended to run on development -systems, producing source and binary distribution archives. Build -tools may also be invoked by installation tools in order to install -software distributed as source archives rather than prebuilt binary -archives. - -"Index servers" are active distribution registries which publish -version and dependency metadata and place constraints on the permitted -metadata. - -"Publication tools" are automated tools intended to run on development -systems and upload source and binary distribution archives to index -servers. - -"Installation tools" are automated tools intended to run on production -systems, consuming source and binary distribution archives from an -index server or other designated location and deploying them to the -target system. - -"Automated tools" is a collective term covering build tools, index -servers, publication tools, installation tools and any other software -that produces or consumes distribution version and dependency -metadata. - -"Projects" refers to the developers that manage the creation of a -particular distribution. +* "Distributions" +* "Versions" +* "Build tools" +* "Index servers" +* "Publication tools" +* "Installation tools" +* "Automated tools" +* "Projects" Version scheme ============== -Distribution versions are identified by both a public version -identifier, which supports all defined version comparison operations, -and a build label, which supports only strict equality comparisons. +Distribution versions are identified by both a public version identifier, +which supports all defined version comparison operations, and a build +label, which supports only strict equality comparisons. The version scheme is used both to describe the distribution version provided by a particular distribution archive, as well as to place @@ -99,14 +69,13 @@ N[.N]+[{a|b|c|rc}N][.postN][.devN] -Public version identifiers MUST NOT include leading or trailing -whitespace. +Public version identifiers MUST NOT include leading or trailing whitespace. Public version identifiers MUST be unique within a given distribution. -Installation tools SHOULD ignore any public versions which do not -comply with this scheme. Installation tools MAY warn the user when -non-compliant or ambiguous versions are detected. +Installation tools SHOULD ignore any public versions which do not comply with +this scheme. Installation tools MAY warn the user when non-compliant +or ambiguous versions are detected. Public version identifiers are separated into up to four segments: @@ -115,22 +84,19 @@ * Post-release segment: ``.postN`` * Development release segment: ``.devN`` -Any given version will be a "release", "pre-release", "post-release" -or "developmental release" as defined in the following sections. +Any given version will be a "release", "pre-release", "post-release" or +"developmental release" as defined in the following sections. .. note:: - Some hard to read version identifiers are permitted by this scheme - in order to better accommodate the wide range of versioning - practices across existing public and private Python projects, given - the constraint that the package index is not yet sophisticated - enough to allow the introduction of a simpler, - backwards-incompatible scheme. + Some hard to read version identifiers are permitted by this scheme in + order to better accommodate the wide range of versioning practices + across existing public and private Python projects. Accordingly, some of the versioning practices which are technically - permitted by the PEP are strongly discouraged for new projects. - Where this is the case, the relevant details are noted in the - following sections. + permitted by the PEP are strongly discouraged for new projects. Where + this is the case, the relevant details are noted in the following + sections. Build labels @@ -138,42 +104,44 @@ Build labels are text strings with minimal defined semantics. -To ensure build labels can be readily incorporated in file names and -URLs, they MUST be comprised of only ASCII alphanumerics, plus signs, +To ensure build labels can be readily incorporated as part of file names +and URLs, they MUST be comprised of only ASCII alphanumerics, plus signs, periods and hyphens. In addition, build labels MUST be unique within a given distribution. +As with distribution names, all comparisons of build labels MUST be case +insensitive. + Releases -------- -A version identifier that consists solely of a release segment is -termed a "release". +A version identifier that consists solely of a release segment is termed +a "release". -The release segment consists of one or more non-negative integer -values, separated by dots:: +The release segment consists of one or more non-negative integer values, +separated by dots:: N[.N]+ Releases within a project will typically be numbered in a consistently increasing fashion. -Comparison and ordering of release segments considers the numeric -value of each component of the release segment in turn. When -comparing release segments with different numbers of components, the -shorter segment is padded out with additional zeroes as necessary. +Comparison and ordering of release segments considers the numeric value +of each component of the release segment in turn. When comparing release +segments with different numbers of components, the shorter segment is +padded out with additional zeroes as necessary. -Date based release numbers are technically compatible with this -scheme, but their use is not consistent with the expected API -versioning semantics described below. Accordingly, automated tools -SHOULD at least issue a warning when encountering a leading release -component greater than or equal to ``1980`` and MAY treat this case as -an error. +Date based release numbers are declared to be incompatible with this scheme, +as their use is not consistent with the expected API versioning semantics +described below. Accordingly, automated tools SHOULD report an error +when encountering a leading release component greater than or equal +to ``1980``. -While any number of additional components after the first are -permitted under this scheme, the most common variants are to use two -components ("major.minor") or three components ("major.minor.micro"). +While any number of additional components after the first are permitted +under this scheme, the most common variants are to use two components +("major.minor") or three components ("major.minor.micro"). For example:: @@ -189,78 +157,76 @@ 2.0 2.0.1 -A release series is any set of release numbers that start with a -common prefix. For example, ``3.3.1``, ``3.3.5`` and ``3.3.9.45`` are -all part of the ``3.3`` release series. +A release series is any set of release numbers that start with a common +prefix. For example, ``3.3.1``, ``3.3.5`` and ``3.3.9.45`` are all +part of the ``3.3`` release series. .. note:: - ``X.Y`` and ``X.Y.0`` are not considered distinct release numbers, - as the release segment comparison rules implicit expand the two - component form to ``X.Y.0`` when comparing it to any release - segment that includes three components. + ``X.Y`` and ``X.Y.0`` are not considered distinct release numbers, as + the release segment comparison rules implicit expand the two component + form to ``X.Y.0`` when comparing it to any release segment that includes + three components. Pre-releases ------------ -Some projects use an "alpha, beta, release candidate" pre-release -cycle to support testing by their users prior to a full release. +Some projects use an "alpha, beta, release candidate" pre-release cycle to +support testing by their users prior to a final release. -If used as part of a project's development cycle, these pre-releases -are indicated by including a pre-release segment in the version -identifier:: +If used as part of a project's development cycle, these pre-releases are +indicated by including a pre-release segment in the version identifier:: X.YaN # Alpha release X.YbN # Beta release - X.YcN # Release candidate (alternative notation: X.YrcN) - X.Y # Full release + X.YcN # Candidate release (alternative notation: X.YrcN) + X.Y # Final release A version identifier that consists solely of a release segment and a pre-release segment is termed a "pre-release". The pre-release segment consists of an alphabetical identifier for the -pre-release phase, along with a non-negative integer value. -Pre-releases for a given release are ordered first by phase (alpha, -beta, release candidate) and then by the numerical component within -that phase. +pre-release phase, along with a non-negative integer value. Pre-releases for +a given release are ordered first by phase (alpha, beta, release candidate) +and then by the numerical component within that phase. -Build tools, publication tools and index servers SHOULD disallow the -creation of both ``c`` and ``rc`` releases for a common release -segment, but this may need to be tolerated in order to handle some -existing legacy distributions. +Installation tools MAY accept both ``c`` and ``rc`` releases for a common +release segment in order to handle some existing legacy distributions. -Installation tools SHOULD interpret all ``rc`` versions as coming -after all ``c`` versions (that is, ``rc1`` indicates a later version -than ``c2``). Installation tools MAY warn the user when such -ambiguous versions are detected, or even reject them entirely. +Installation tools SHOULD interpret all ``rc`` versions as coming after all +``c`` versions (that is, ``rc1`` indicates a later version than ``c2``). +Installation tools MAY warn the user when such ambiguous versions are +detected, or even reject them entirely. + +Build tools, publication tools and index servers SHOULD disallow the creation +of both ``c`` and ``rc`` releases for a common release segment. Post-releases ------------- -Some projects use post-releases to address minor errors in a release -that do not affect the distributed software (for example, correcting -an error in the release notes). +Some projects use post-releases to address minor errors in a release that +do not affect the distributed software (for example, correcting an error +in the release notes). -If used as part of a project's development cycle, these post-releases -are indicated by including a post-release segment in the version -identifier:: +If used as part of a project's development cycle, these post-releases are +indicated by including a post-release segment in the version identifier:: X.Y.postN # Post-release A version identifier that includes a post-release segment without a developmental release segment is termed a "post-release". -The post-release segment consists of the string ``.post``, followed by -a non-negative integer value. Post-releases are ordered by their +The post-release segment consists of the string ``.post``, followed by a +non-negative integer value. Post-releases are ordered by their numerical component, immediately following the corresponding release, and ahead of any subsequent release. .. note:: The use of post-releases to publish maintenance releases containing - actual bug fixes is strongly discouraged. In general, it is better + actual bug fixes is strongly discouraged. In general, it is better to use a longer release number and increment the final component for each maintenance release. @@ -273,22 +239,22 @@ .. note:: Creating post-releases of pre-releases is strongly discouraged, as - it makes the version identifier difficult to parse for human - readers. In general, it is substantially clearer to simply create - a new pre-release by incrementing the numeric component. + it makes the version identifier difficult to parse for human readers. + In general, it is substantially clearer to simply create a new + pre-release by incrementing the numeric component. Developmental releases ---------------------- -Some projects make regular developmental releases, and system -packagers (especially for Linux distributions) may wish to create -early releases directly from source control which do not conflict with -later project releases. +Some projects make regular developmental releases, and system packagers +(especially for Linux distributions) may wish to create early releases +directly from source control which do not conflict with later project +releases. If used as part of a project's development cycle, these developmental -releases are indicated by including a developmental release segment in -the version identifier:: +releases are indicated by including a developmental release segment in the +version identifier:: X.Y.devN # Developmental release @@ -296,11 +262,10 @@ termed a "developmental release". The developmental release segment consists of the string ``.dev``, -followed by a non-negative integer value. Developmental releases are -ordered by their numerical component, immediately before the -corresponding release (and before any pre-releases with the same -release segment), and following any previous release (including any -post-releases). +followed by a non-negative integer value. Developmental releases are ordered +by their numerical component, immediately before the corresponding release +(and before any pre-releases with the same release segment), and following +any previous release (including any post-releases). Developmental releases are also permitted for pre-releases and post-releases:: @@ -313,27 +278,24 @@ .. note:: Creating developmental releases of pre-releases is strongly - discouraged, as it makes the version identifier difficult to parse - for human readers. In general, it is substantially clearer to - simply create additional pre-releases by incrementing the numeric - component. + discouraged, as it makes the version identifier difficult to parse for + human readers. In general, it is substantially clearer to simply create + additional pre-releases by incrementing the numeric component. - Developmental releases of post-releases are also strongly - discouraged, but they may be appropriate for projects which use the - post-release notation for full maintenance releases which may - include code changes. + Developmental releases of post-releases are also strongly discouraged, + but they may be appropriate for projects which use the post-release + notation for full maintenance releases which may include code changes. Examples of compliant version schemes ------------------------------------- The standard version scheme is designed to encompass a wide range of -identification practices across public and private Python projects. -In practice, a single project attempting to use the full flexibility -offered by the scheme would create a situation where human users had -difficulty figuring out the relative order of versions, even though -the rules above ensure all compliant tools will order them -consistently. +identification practices across public and private Python projects. In +practice, a single project attempting to use the full flexibility offered +by the scheme would create a situation where human users had difficulty +figuring out the relative order of versions, even though the rules above +ensure all compliant tools will order them consistently. The following examples illustrate a small selection of the different approaches projects may choose to identify their releases, while still @@ -357,7 +319,7 @@ 1.2.0 ... -"major.minor" versioning with alpha, beta and release candidate +"major.minor" versioning with alpha, beta and candidate pre-releases:: 0.9 @@ -369,8 +331,8 @@ 1.1a1 ... -"major.minor" versioning with developmental releases, release -candidates and post-releases for minor corrections:: +"major.minor" versioning with developmental releases, release candidates +and post-releases for minor corrections:: 0.9 1.0.dev1 @@ -394,9 +356,9 @@ automatically process distribution metadata, rather than developers of Python distributions deciding on a versioning scheme. -The release segment of version identifiers MUST be sorted in the same -order as Python's tuple sorting when the release segment is parsed as -follows:: +The release segment of version identifiers MUST be sorted in +the same order as Python's tuple sorting when the release segment is +parsed as follows:: tuple(map(int, release_segment.split("."))) @@ -409,28 +371,26 @@ .devN, aN, bN, cN, rcN, <no suffix>, .postN Note that `rc` will always sort after `c` (regardless of the numeric -component) although they are semantically equivalent. Tools are free -to reject this case as ambiguous and remain in compliance with the -PEP. +component) although they are semantically equivalent. Tools are free to +reject this case as ambiguous and remain in compliance with the PEP. Within an alpha (``1.0a1``), beta (``1.0b1``), or release candidate -(``1.0c1``, ``1.0rc1``), the following suffixes are permitted and MUST -be ordered as shown:: +(``1.0c1``, ``1.0rc1``), the following suffixes are permitted and MUST be +ordered as shown:: .devN, <no suffix>, .postN -Within a post-release (``1.0.post1``), the following suffixes are -permitted and MUST be ordered as shown:: +Within a post-release (``1.0.post1``), the following suffixes are permitted +and MUST be ordered as shown:: .devN, <no suffix> -Note that ``devN`` and ``postN`` MUST always be preceded by a dot, -even when used immediately following a numeric version -(e.g. ``1.0.dev456``, ``1.0.post1``). +Note that ``devN`` and ``postN`` MUST always be preceded by a dot, even +when used immediately following a numeric version (e.g. ``1.0.dev456``, +``1.0.post1``). -Within a pre-release, post-release or development release segment with -a shared prefix, ordering MUST be by the value of the numeric -component. +Within a pre-release, post-release or development release segment with a +shared prefix, ordering MUST be by the value of the numeric component. The following example covers many of the possible combinations:: @@ -454,70 +414,67 @@ Version ordering across different metadata versions --------------------------------------------------- -Metadata v1.0 (PEP 241) and metadata v1.1 (PEP 314) do not specify a -standard version identification or ordering scheme. This PEP does not -mandate any particular approach to handling such versions, but -acknowledges that the de facto standard for ordering them is the -scheme used by the ``pkg_resources`` component of ``setuptools``. +Metadata v1.0 (PEP 241) and metadata v1.1 (PEP 314) do not +specify a standard version identification or ordering scheme. This PEP does +not mandate any particular approach to handling such versions, but +acknowledges that the de facto standard for ordering them is +the scheme used by the ``pkg_resources`` component of ``setuptools``. -Software that automatically processes distribution metadata SHOULD -attempt to normalize non-compliant version identifiers to the standard -scheme, and ignore them if normalization fails. As any normalization -scheme will be implementation specific, this means that projects using -non-compliant version identifiers may not be handled consistently -across different tools, even when correctly publishing the earlier -metadata versions. +Software that automatically processes distribution metadata SHOULD attempt +to normalize non-compliant version identifiers to the standard scheme, and +ignore them if normalization fails. As any normalization scheme will be +implementation specific, this means that projects using non-compliant +version identifiers may not be handled consistently across different +tools, even when correctly publishing the earlier metadata versions. -For distributions currently using non-compliant version identifiers, -these filtering guidelines mean that it should be enough for the -project to simply switch to the use of compliant version identifiers -to ensure consistent handling by automated tools. +For distributions currently using non-compliant version identifiers, these +filtering guidelines mean that it should be enough for the project to +simply switch to the use of compliant version identifiers to ensure +consistent handling by automated tools. -Distribution users may wish to explicitly remove non-compliant -versions from any private package indexes they control. +Distribution users may wish to explicitly remove non-compliant versions from +any private package indexes they control. -For metadata v1.2 (PEP 345), the version ordering described in this -PEP SHOULD be used in preference to the one defined in PEP 386. +For metadata v1.2 (PEP 345), the version ordering described in this PEP +SHOULD be used in preference to the one defined in PEP 386. Compatibility with other version schemes ---------------------------------------- Some projects may choose to use a version scheme which requires -translation in order to comply with the public version scheme defined -in this PEP. In such cases, the build label can be used to record the -project specific version as an arbitrary label, while the translated -public version is published in the version field. +translation in order to comply with the public version scheme defined in +this PEP. In such cases, the build label can be used to +record the project specific version as an arbitrary label, while the +translated public version is published in the version field. -This allows automated distribution tools to provide consistently -correct ordering of published releases, while still allowing -developers to use the internal versioning scheme they prefer for their -projects. +This allows automated distribution tools to provide consistently correct +ordering of published releases, while still allowing developers to use +the internal versioning scheme they prefer for their projects. Semantic versioning ~~~~~~~~~~~~~~~~~~~ -`Semantic versioning`_ is a popular version identification scheme that -is more prescriptive than this PEP regarding the significance of -different elements of a release number. Even if a project chooses not -to abide by the details of semantic versioning, the scheme is worth -understanding as it covers many of the issues that can arise when -depending on other distributions, and when publishing a distribution -that others rely on. +`Semantic versioning`_ is a popular version identification scheme that is +more prescriptive than this PEP regarding the significance of different +elements of a release number. Even if a project chooses not to abide by +the details of semantic versioning, the scheme is worth understanding as +it covers many of the issues that can arise when depending on other +distributions, and when publishing a distribution that others rely on. The "Major.Minor.Patch" (described in this PEP as "major.minor.micro") -aspects of semantic versioning (clauses 1-9 in the 2.0.0-rc-1 -specification) are fully compatible with the version scheme defined in -this PEP, and abiding by these aspects is encouraged. +aspects of semantic versioning (clauses 1-9 in the 2.0.0-rc-1 specification) +are fully compatible with the version scheme defined in this PEP, and abiding +by these aspects is encouraged. Semantic versions containing a hyphen (pre-releases - clause 10) or a -plus sign (builds - clause 11) are *not* compatible with this PEP and -are not permitted in the public version field. +plus sign (builds - clause 11) are *not* compatible with this PEP +and are not permitted in the public version field. -One possible mechanism to translate such semantic versioning based -build labels to compatible public versions is to use the ``.devN`` -suffix to specify the appropriate version order. +One possible mechanism to translate such semantic versioning based build +labels to compatible public versions is to use the ``.devN`` suffix to +specify the appropriate version order. .. _Semantic versioning: http://semver.org/ @@ -525,158 +482,167 @@ DVCS based version labels ~~~~~~~~~~~~~~~~~~~~~~~~~ -Many build tools integrate with distributed version control systems -like Git and Mercurial in order to add an identifying hash to the -version identifier. As hashes cannot be ordered reliably such -versions are not permitted in the public version field. +Many build tools integrate with distributed version control systems like +Git and Mercurial in order to add an identifying hash to the version +identifier. As hashes cannot be ordered reliably such versions are not +permitted in the public version field. -As with semantic versioning, the public ``.devN`` suffix may be used -to uniquely identify such releases for publication, while the build -label is used to record the original DVCS based version label. +As with semantic versioning, the public ``.devN`` suffix may be used to +uniquely identify such releases for publication, while the build label is +used to record the original DVCS based version label. Date based versions ~~~~~~~~~~~~~~~~~~~ As with other incompatible version schemes, date based versions can be -stored in the build label field. Translating them to a compliant -public version is straightforward: use a leading "0." prefix in the -public version label, with the date based version number as the -remaining components in the release segment. +stored in the build label field. Translating them to a compliant +public version is straightforward: use a leading ``"0."`` prefix in the +public version label, with the date based version number as the remaining +components in the release segment. This has the dual benefit of allowing subsequent migration to version -numbering based on API compatibility, as well as triggering more -appropriate version comparison semantics. +numbering based on API compatibility, as well as triggering more appropriate +version comparison semantics. Version specifiers ================== -A version specifier consists of a series of version clauses, separated -by commas. For example:: +A version specifier consists of a series of version clauses, separated by +commas. For example:: - 0.9, >= 1.0, != 1.3.4.*, < 2.0 + 0.9, ~= 0.9, >= 1.0, != 1.3.4.*, < 2.0 -The comparison operator (or lack thereof) determines the kind of -version clause: +The comparison operator (or lack thereof) determines the kind of version +clause: -* No operator: `Compatible release`_ clause +* No operator: equivalent to ``~=`` +* ``~=``: `Compatible release`_ clause * ``==``: `Version matching`_ clause * ``!=``: `Version exclusion`_ clause * ``is``: `Build reference`_ clause -* ``<``, ``>``, ``<=``, ``>=``: `Ordered comparison`_ clause +* ``<=``, ``>=``: `Inclusive ordered comparison`_ clause +* ``<``, ``>``: `Exclusive ordered comparison`_ clause -The comma (",") is equivalent to a logical **and** operator: a -candidate version must match all given version clauses in order to -match the specifier as a whole. +The comma (",") is equivalent to a logical **and** operator: a candidate +version must match all given version clauses in order to match the +specifier as a whole. Whitespace between a conditional operator and the following version identifier is optional, as is the whitespace around the commas. -When multiple candidate versions match a version specifier, the -preferred version SHOULD be the latest version as determined by the -consistent ordering defined by the standard `Version scheme`_. -Whether or not pre-releases are considered as candidate versions -SHOULD be handled as described in `Handling of pre-releases`_. +When multiple candidate versions match a version specifier, the preferred +version SHOULD be the latest version as determined by the consistent +ordering defined by the standard `Version scheme`_. Whether or not +pre-releases are considered as candidate versions SHOULD be handled as +described in `Handling of pre-releases`_. Compatible release ------------------ -A compatible release clause consists solely of a version identifier -without any comparison operator. It matches any candidate version -that is expected to be compatible with the specified version. +A compatible release clause consists of either a version identifier without +any comparison operator or else the compatible release operator ``~=`` +and a version identifier. It matches any candidate version that is expected +to be compatible with the specified version. -The specified version identifier must be in the standard format -described in `Version scheme`_. +The specified version identifier must be in the standard format described in +`Version scheme`_. -For a given release identifier ``V.N``, the compatible release clause -is approximately equivalent to the pair of comparison clauses:: +For a given release identifier ``V.N``, the compatible release clause is +approximately equivalent to the pair of comparison clauses:: >= V.N, == V.* -For example, the following version clauses are equivalent:: +For example, the following groups of version clauses are equivalent:: 2.2 + ~= 2.2 >= 2.2, == 2.* 1.4.5 + ~= 1.4.5 >= 1.4.5, == 1.4.* If a pre-release, post-release or developmental release is named in a -compatible release clause as ``V.N.suffix``, then the suffix is -ignored when determining the required prefix match:: +compatible release clause as ``V.N.suffix``, then the suffix is ignored +when determining the required prefix match:: 2.2.post3 + ~= 2.2.post3 >= 2.2.post3, == 2.* 1.4.5a4 + ~= 1.4.5a4 >= 1.4.5a4, == 1.4.* -The padding rules for release segment comparisons means that the -assumed degree of forward compatibility in a compatible release clause -can be controlled by appending additional zeroes to the version -specifier:: +The padding rules for release segment comparisons means that the assumed +degree of forward compatibility in a compatible release clause can be +controlled by appending additional zeroes to the version specifier:: 2.2.0 + ~= 2.2.0 >= 2.2.0, == 2.2.* 1.4.5.0 + ~= 1.4.5.0 >= 1.4.5.0, == 1.4.5.* Version matching ---------------- -A version matching clause includes the version matching operator -``==`` and a version identifier. +A version matching clause includes the version matching operator ``==`` +and a version identifier. -The specified version identifier must be in the standard format -described in `Version scheme`_, but a trailing ``.*`` is permitted as -described below. +The specified version identifier must be in the standard format described in +`Version scheme`_, but a trailing ``.*`` is permitted as described below. -By default, the version matching operator is based on a strict -equality comparison: the specified version must be exactly the same as -the requested version. The *only* substitution performed is the zero -padding of the release segment to ensure the release segments are -compared with the same length. +By default, the version matching operator is based on a strict equality +comparison: the specified version must be exactly the same as the requested +version. The *only* substitution performed is the zero padding of the +release segment to ensure the release segments are compared with the same +length. -Prefix matching may be requested instead of strict comparison, by -appending a trailing ``.*`` to the version identifier in the version -matching clause. This means that additional trailing segments will be -ignored when determining whether or not a version identifier matches -the clause. If the version includes only a release segment, than -trailing components in the release segment are also ignored. +Prefix matching may be requested instead of strict comparison, by appending +a trailing ``.*`` to the version identifier in the version matching clause. +This means that additional trailing segments will be ignored when +determining whether or not a version identifier matches the clause. If the +version includes only a release segment, than trailing components in the +release segment are also ignored. For example, given the version ``1.1.post1``, the following clauses would -match or not as shown: +match or not as shown:: == 1.1 # Not equal, so 1.1.post1 does not match clause == 1.1.post1 # Equal, so 1.1.post1 matches clause == 1.1.* # Same prefix, so 1.1.post1 matches clause -.. note:: +The use of ``==`` (without at least the wildcard suffix) when defining +dependencies for published distributions is strongly discouraged as it +greatly complicates the deployment of security fixes. The strict version +comparison operator is intended primarily for use when defining +dependencies for repeatable *deployments of applications* while using +a shared distribution index. - The use of ``==`` when defining dependencies for published - distributions is strongly discouraged as it greatly complicates the - deployment of security fixes. The strict version comparison - operator is intended primarily for use when defining dependencies - for repeatable *deployments of applications* while using a shared - distribution index. +Publication tools and index servers SHOULD at least emit a warning when +dependencies are pinned in this fashion and MAY refuse to allow publication +of such overly specific dependencies. Version exclusion ----------------- -A version exclusion clause includes the version matching operator -``!=`` and a version identifier. +A version exclusion clause includes the version exclusion operator ``!=`` +and a version identifier. -The allowed version identifiers and comparison semantics are the same -as those of the `Version matching`_ operator, except that the sense of -any match is inverted. +The allowed version identifiers and comparison semantics are the same as +those of the `Version matching`_ operator, except that the sense of any +match is inverted. -For example, given the version ``1.1.post1``, the following clauses -would match or not as shown: +For example, given the version ``1.1.post1``, the following clauses would +match or not as shown:: != 1.1 # Not equal, so 1.1.post1 matches clause != 1.1.post1 # Equal, so 1.1.post1 does not match clause @@ -686,58 +652,113 @@ Build reference --------------- -A build reference includes the build label matching operator ``is`` and -a build reference. - -A build reference is a direct URI reference supplied to satisfy a -dependency. The exact kinds of URIs and targets supported will be -determined by the specific installation tool used. +A build reference includes the build reference operator ``is`` and +a build label or a build URL. Publication tools and public index servers SHOULD NOT permit build references in dependency specifications. -Installation tools SHOULD support the use of build references to -identify dependencies. - -Automated tools MAY support the use of build labels in build reference -clauses. They can be clearly distinguished from URI references -without ambiguity, as ``:`` and ``/`` are not permitted in build -labels. +Installation tools SHOULD support the use of build references to identify +dependencies. Build label matching works solely on strict equality comparisons: the -candidate build label must be exactly the same as the build label in -the version clause. +candidate build label must be exactly the same as the build label in the +version clause for the clause to match the candidate distribution. +For example, a build reference could be used to depend on a ``hashdist`` +generated build of ``zlib`` with the ``hashdist`` hash used as a build +label:: -Ordered comparison ------------------- + zlib (is d4jwf2sb2g6glprsdqfdpcracwpzujwq) -An ordered comparison clause includes a comparison operator and a -version identifier, and will match any version where the comparison is -correct based on the relative position of the candidate version and -the specified version given the consistent ordering defined by the -standard `Version scheme`_. +A build URL is distinguished from a build label by the presence of +``:`` and ``/`` characters in the build reference. As these characters +are not permitted in build labels, they indicate that the reference uses +a build URL. -The supported ordered comparison operators are ``<``, ``>``, ``<=``, -``>=``. +Some appropriate targets for a build URL are a binary archive, a +source tarball, an sdist archive or a direct reference to a tag or +specific commit in an online version control system. The exact URLs and +targets supported will be installation tool specific. -As with version matching, the release segment is zero padded as -necessary to ensure the release segments are compared with the same -length. +For example, a local prebuilt wheel file may be referenced directly:: -To exclude pre-releases and post-releases correctly, the comparison -clauses ``< V`` and ``> V`` MUST be interpreted as also implying the -version matching clause ``!= V.*``. + exampledist (is file:///localbuilds/exampledist-1.0-py33-none-any.whl) + +All build URL references SHOULD either specify a local file URL, a secure +transport mechanism (such as ``https``) or else include an expected hash +value in the URL for verification purposes. If an insecure network +transport is specified without any hash information (or with hash +information that the tool doesn't understand), automated tools SHOULD +at least emit a warning and MAY refuse to rely on the URL. + +It is RECOMMENDED that only hashes which are unconditionally provided by +the latest version of the standard library's ``hashlib`` module be used +for source archive hashes. At time of writing, that list consists of +``'md5'``, ``'sha1'``, ``'sha224'``, ``'sha256'``, ``'sha384'``, and +``'sha512'``. + +For binary or source archive references, an expected hash value may be +specified by including a ``<hash-algorithm>=<expected-hash>`` as part of +the URL fragment. + +For version control references, the ``VCS+protocol`` scheme SHOULD be +used to identify both the version control system and the secure transport. + +To support version control systems that do not support including commit or +tag references directly in the URL, that information may be appended to the +end of the URL using the ``@<tag>`` notation. + +The use of ``is`` when defining dependencies for published distributions +is strongly discouraged as it greatly complicates the deployment of +security fixes. The build label matching operator is intended primarily +for use when defining dependencies for repeatable *deployments of +applications* while using a shared distribution index, as well as to +reference dependencies which are not published through an index server. + + +Inclusive ordered comparison +---------------------------- + +An inclusive ordered comparison clause includes a comparison operator and a +version identifier, and will match any version where the comparison is correct +based on the relative position of the candidate version and the specified +version given the consistent ordering defined by the standard +`Version scheme`_. + +The inclusive ordered comparison operators are ``<=`` and ``>=``. + +As with version matching, the release segment is zero padded as necessary to +ensure the release segments are compared with the same length. + + +Exclusive ordered comparison +---------------------------- + +Exclusive ordered comparisons are similar to inclusive ordered comparisons, +except that the comparison operators are ``<`` and ``>`` and the clause +MUST be effectively interpreted as implying the prefix based version +exclusion clause ``!= V.*``. + +The exclusive ordered comparison ``> V`` MUST NOT match a post-release +or maintenance release of the given version. Maintenance releases can be +permitted by using the clause ``> V.0``, while both post releases and +maintenance releases can be permitted by using the inclusive ordered +comparison ``>= V.post1``. + +The exclusive ordered comparison ``< V`` MUST NOT match a pre-release of +the given version, even if acceptance of pre-releases is enabled as +described in the section below. Handling of pre-releases ------------------------ -Pre-releases of any kind, including developmental releases, are -implicitly excluded from all version specifiers, *unless* a -pre-release or developmental release is explicitly mentioned in one of -the clauses. For example, these specifiers implicitly exclude all -pre-releases and development releases of later versions:: +Pre-releases of any kind, including developmental releases, are implicitly +excluded from all version specifiers, *unless* a pre-release or developmental +release is explicitly mentioned in one of the clauses. For example, these +specifiers implicitly exclude all pre-releases and development +releases of later versions:: 2.2 >= 1.0 @@ -751,73 +772,68 @@ >= 1.0, != 1.0b2 >= 1.0, < 2.0.dev123 -Dependency resolution tools SHOULD exclude pre-releases by default, -but SHOULD also allow users to request the following alternative -behaviours: +By default, dependency resolution tools SHOULD: * accept already installed pre-releases for all version specifiers -* retrieve and install available pre-releases for all version - specifiers +* accept remotely available pre-releases for version specifiers which + include at least one version clauses that references a pre-release +* exclude all other pre-releases from consideration + +Dependency resolution tools SHOULD also allow users to request the +following alternative behaviours: + +* accepting pre-releases for all version specifiers +* excluding pre-releases for all version specifiers (reporting an error or + warning if a pre-release is already installed locally) Dependency resolution tools MAY also allow the above behaviour to be controlled on a per-distribution basis. -Post-releases and purely numeric releases receive no special treatment -- they are always included unless explicitly excluded. +Post-releases and purely numeric releases receive no special treatment in +version specifiers - they are always included unless explicitly excluded. Examples -------- -* ``3.1``: version 3.1 or later, but not version 4.0 or - later. Excludes pre-releases and developmental releases. -* ``3.1.2``: version 3.1.2 or later, but not version 3.2.0 or - later. Excludes pre-releases and developmental releases. -* ``3.1a1``: version 3.1a1 or later, but not version 4.0 or - later. Allows pre-releases like 3.2a4 and developmental releases - like 3.2.dev1. -* ``== 3.1``: specifically version 3.1 (or 3.1.0), excludes all - pre-releases, post releases, developmental releases and any 3.1.x - maintenance releases. -* ``== 3.1.*``: any version that starts with 3.1, excluding - pre-releases and developmental releases. Equivalent to the ``3.1.0`` - compatible release clause. -* ``3.1.0, != 3.1.3``: version 3.1.0 or later, but not version 3.1.3 - and not version 3.2.0 or later. Excludes pre-releases and - developmental releases. +* ``3.1``: version 3.1 or later, but not + version 4.0 or later. Excludes pre-releases and developmental releases. +* ``3.1.2``: version 3.1.2 or later, but not + version 3.2.0 or later. Excludes pre-releases and developmental releases. +* ``3.1a1``: version 3.1a1 or later, but not + version 4.0 or later. Allows pre-releases like 3.2a4 and developmental + releases like 3.2.dev1. +* ``== 3.1``: specifically version 3.1 (or 3.1.0), excludes all pre-releases, + post releases, developmental releases and any 3.1.x maintenance releases. +* ``== 3.1.*``: any version that starts with 3.1, excluding pre-releases and + developmental releases. Equivalent to the ``3.1.0`` compatible release + clause. +* ``3.1.0, != 3.1.3``: version 3.1.0 or later, but not version 3.1.3 and + not version 3.2.0 or later. Excludes pre-releases and developmental + releases. Updating the versioning specification ===================================== -The versioning specification may be updated with clarifications -without requiring a new PEP or a change to the metadata version. +The versioning specification may be updated with clarifications without +requiring a new PEP or a change to the metadata version. -Actually changing the version comparison semantics still requires a -new versioning scheme and metadata version defined in new PEPs. +Actually changing the version comparison semantics still requires a new +versioning scheme and metadata version defined in new PEPs. Open issues =========== * The new ``is`` operator seems like a reasonable way to cleanly allow - *deployments* to bring in non-published dependencies, while heavily - discouraging the practice for published libraries. However, it's a - first draft of the idea, so feedback is definitely welcome. + installation tools to bring in non-published dependencies, while heavily + discouraging the practice for published libraries. It also makes + build labels more useful by allowing them to be used to pin dependencies + in the integration use case. -* Currently, the cleanest way to specify that a project runs on Python - 2.6+ and 3.3+ is to use a clause like:: - - Requires-Python: >= 2.6, < 4.0, != 3.0.*, != 3.1.*, != 3.2.* - - It would be better if there was a cleaner way to specify "this OR - that" in a version specifier. Perhaps something like:: - - Requires-Python: (2.6) or (3.3) - - This would be a respectable increase in the complexity of the - parsing for version specifiers though, even if it was only allowed - at the top level. + However, it's an early draft of the idea, so feedback is definitely + welcome. Summary of differences from \PEP 386 @@ -825,16 +841,16 @@ * Moved the description of version specifiers into the versioning PEP -* added the "build label" concept to better handle projects that wish - to use a non-compliant versioning scheme internally, especially - those based on DVCS hashes +* added the "build label" concept to better handle projects that wish to + use a non-compliant versioning scheme internally, especially those based + on DVCS hashes * added the "compatible release" clause * added the "build reference" clause -* separated the two kinds of "version matching" clause (strict and - prefix) +* added the trailing wildcard syntax for prefix based version matching + and exclusion * changed the top level sort position of the ``.devN`` suffix @@ -857,111 +873,106 @@ ------------------- The new build label support is intended to make it clearer that the -constraints on public version identifiers are there primarily to aid -in the creation of reliable automated dependency analysis tools. -Projects are free to use whatever versioning scheme they like -internally, so long as they are able to translate it to something the -dependency analysis tools will understand. +constraints on public version identifiers are there primarily to aid in +the creation of reliable automated dependency analysis tools. Projects +are free to use whatever versioning scheme they like internally, so long +as they are able to translate it to something the dependency analysis tools +will understand. Changing the version scheme --------------------------- The key change in the version scheme in this PEP relative to that in -PEP 386 is to sort top level developmental releases like ``X.Y.devN`` -ahead of alpha releases like ``X.Ya1``. This is a far more logical -sort order, as projects already using both development releases and -alphas/betas/release candidates do not want their developmental -releases sorted in between their release candidates and their full -releases. There is no rationale for using ``dev`` releases in that -position rather than merely creating additional release candidates. +PEP 386 is to sort top level developmental releases like ``X.Y.devN`` ahead +of alpha releases like ``X.Ya1``. This is a far more logical sort order, as +projects already using both development releases and alphas/betas/release +candidates do not want their developmental releases sorted in +between their release candidates and their final releases. There is no +rationale for using ``dev`` releases in that position rather than +merely creating additional release candidates. -The updated sort order also means the sorting of ``dev`` versions is -now consistent between the metadata standard and the pre-existing -behaviour of ``pkg_resources`` (and hence the behaviour of current -installation tools). +The updated sort order also means the sorting of ``dev`` versions is now +consistent between the metadata standard and the pre-existing behaviour +of ``pkg_resources`` (and hence the behaviour of current installation +tools). -Making this change should make it easier for affected existing -projects to migrate to the latest version of the metadata standard. +Making this change should make it easier for affected existing projects to +migrate to the latest version of the metadata standard. Another change to the version scheme is to allow single number versions, similar to those used by non-Python projects like Mozilla -Firefox, Google Chrome and the Fedora Linux distribution. This is -actually expected to be more useful for version specifiers (allowing -things like the simple ``Requires-Python: 3`` rather than the more -convoluted ``Requires-Python: >= 3.0, < 4``), but it is easier to +Firefox, Google Chrome and the Fedora Linux distribution. This is actually +expected to be more useful for version specifiers, but it is easier to allow it for both version specifiers and release numbers, rather than splitting the two definitions. -The exclusion of leading and trailing whitespace was made explicit -after a couple of projects with version identifiers differing only in -a trailing ``\n`` character were found on PyPI. +The exclusion of leading and trailing whitespace was made explicit after +a couple of projects with version identifiers differing only in a +trailing ``\n`` character were found on PyPI. -The exclusion of major release numbers that looks like dates was -implied by the overall text of PEP 386, but not clear in the -definition of the version scheme. This exclusion has been made clear -in the definition of the release component. +The exclusion of major release numbers that look like dates was implied +by the overall text of PEP 386, but not clear in the definition of the +version scheme. This exclusion has been made clear in the definition of +the release component. -`Appendix A` shows detailed results of an analysis of PyPI -distribution version information, as collected on 19th February, 2013. -This analysis compares the behaviour of the explicitly ordered version -schemes defined in this PEP and PEP 386 with the de facto standard -defined by the behaviour of setuptools. These metrics are useful, as -the intent of both PEPs is to follow existing setuptools behaviour as -closely as is feasible, while still throwing exceptions for -unorderable versions (rather than trying to guess an appropriate order -as setuptools does). +`Appendix A` shows detailed results of an analysis of PyPI distribution +version information, as collected on 19th February, 2013. This analysis +compares the behaviour of the explicitly ordered version schemes defined in +this PEP and PEP 386 with the de facto standard defined by the behaviour +of setuptools. These metrics are useful, as the intent of both PEPs is to +follow existing setuptools behaviour as closely as is feasible, while +still throwing exceptions for unorderable versions (rather than trying +to guess an appropriate order as setuptools does). -Overall, the percentage of compatible distributions improves from -97.7% with PEP 386 to 98.7% with this PEP. While the number of -projects affected in practice was small, some of the affected projects -are in widespread use (such as Pinax and selenium). The surprising -ordering discrepancy also concerned developers and acted as an -unnecessary barrier to adoption of the new metadata standard, even for -projects that weren't directly affected. +Overall, the percentage of compatible distributions improves from 97.7% +with PEP 386 to 98.7% with this PEP. While the number of projects affected +in practice was small, some of the affected projects are in widespread use +(such as Pinax and selenium). The surprising ordering discrepancy also +concerned developers and acted as an unnecessary barrier to adoption of +the new metadata standard, even for projects that weren't directly affected. -The data also shows that the pre-release sorting discrepancies are -seen only when analysing *all* versions from PyPI, rather than when -analysing public versions. This is largely due to the fact that PyPI -normally reports only the most recent version for each project (unless -maintainers explicitly configure their project to display additional -versions). However, installers that need to satisfy detailed version -constraints often need to look at all available versions, as they may -need to retrieve an older release. +The data also shows that the pre-release sorting discrepancies are seen +only when analysing *all* versions from PyPI, rather than when analysing +public versions. This is largely due to the fact that PyPI normally reports +only the most recent version for each project (unless maintainers +explicitly configure their project to display additional versions). However, +installers that need to satisfy detailed version constraints often need +to look at all available versions, as they may need to retrieve an older +release. -Even this PEP doesn't completely eliminate the sorting differences -relative to setuptools: +Even this PEP doesn't completely eliminate the sorting differences relative +to setuptools: * Sorts differently (after translations): 38 / 28194 (0.13 %) * Sorts differently (no translations): 2 / 28194 (0.01 %) -The two remaining sort order discrepancies picked up by the analysis -are due to a pair of projects which have PyPI releases ending with a -carriage return, alongside releases with the same version number, only -*without* the trailing carriage return. +The two remaining sort order discrepancies picked up by the analysis are due +to a pair of projects which have PyPI releases ending with a carriage +return, alongside releases with the same version number, only *without* the +trailing carriage return. -The sorting discrepancies after translation relate mainly to -differences in the handling of pre-releases where the standard -mechanism is considered to be an improvement. For example, the -existing pkg_resources scheme will sort "1.1beta1" *after* "1.1b2", -whereas the suggested standard translation for "1.1beta1" is "1.1b1", -which sorts *before* "1.1b2". Similarly, the pkg_resources scheme -will sort "-dev-N" pre-releases differently from "devN" pre-releases -when they occur within the same release, while the scheme in this PEP -requires normalizing both representations to ".devN" and sorting them -by the numeric component. +The sorting discrepancies after translation relate mainly to differences +in the handling of pre-releases where the standard mechanism is considered +to be an improvement. For example, the existing pkg_resources scheme will +sort "1.1beta1" *after* "1.1b2", whereas the suggested standard translation +for "1.1beta1" is "1.1b1", which sorts *before* "1.1b2". Similarly, the +pkg_resources scheme will sort "-dev-N" pre-releases differently from +"devN" pre-releases when they occur within the same release, while the +scheme in this PEP requires normalizing both representations to ".devN" and +sorting them by the numeric component. A more opinionated description of the versioning scheme ------------------------------------------------------- -As in PEP 386, the primary focus is on codifying existing practices to -make them more amenable to automation, rather than demanding that -existing projects make non-trivial changes to their workflow. -However, the standard scheme allows significantly more flexibility -than is needed for the vast majority of simple Python packages (which -often don't even need maintenance releases - many users are happy with -needing to upgrade to a new feature release to get bug fixes). +As in PEP 386, the primary focus is on codifying existing practices to make +them more amenable to automation, rather than demanding that existing +projects make non-trivial changes to their workflow. However, the +standard scheme allows significantly more flexibility than is needed +for the vast majority of simple Python packages (which often don't even +need maintenance releases - many users are happy with needing to upgrade to a +new feature release to get bug fixes). For the benefit of novice developers, and for experienced developers wishing to better understand the various use cases, the specification @@ -969,57 +980,74 @@ version scheme, including examples of how each component may be used in practice. -The PEP also explicitly guides developers in the direction of semantic -versioning (without requiring it), and discourages the use of several -aspects of the full versioning scheme that have largely been included -in order to cover esoteric corner cases in the practices of existing -projects and in repackaging software for Linux distributions. +The PEP also explicitly guides developers in the direction of +semantic versioning (without requiring it), and discourages the use of +several aspects of the full versioning scheme that have largely been +included in order to cover esoteric corner cases in the practices of +existing projects and in repackaging software for Linux distributions. Describing version specifiers alongside the versioning scheme ------------------------------------------------------------- -The main reason to even have a standardised version scheme in the -first place is to make it easier to do reliable automated dependency -analysis. It makes more sense to describe the primary use case for -version identifiers alongside their definition. +The main reason to even have a standardised version scheme in the first place +is to make it easier to do reliable automated dependency analysis. It makes +more sense to describe the primary use case for version identifiers alongside +their definition. Changing the interpretation of version specifiers ------------------------------------------------- The previous interpretation of version specifiers made it very easy to -accidentally download a pre-release version of a dependency. This in +accidentally download a pre-release version of a dependency. This in turn made it difficult for developers to publish pre-release versions -of software to the Python Package Index, as even marking the package -as hidden wasn't enough to keep automated tools from downloading it, -and also made it harder for users to obtain the test release manually -through the main PyPI web interface. +of software to the Python Package Index, as even marking the package as +hidden wasn't enough to keep automated tools from downloading it, and also +made it harder for users to obtain the test release manually through the +main PyPI web interface. -The previous interpretation also excluded post-releases from some -version specifiers for no adequately justified reason. +The previous interpretation also excluded post-releases from some version +specifiers for no adequately justified reason. -The updated interpretation is intended to make it difficult to -accidentally accept a pre-release version as satisfying a dependency, -while allowing pre-release versions to be explicitly requested when -needed. +The updated interpretation is intended to make it difficult to accidentally +accept a pre-release version as satisfying a dependency, while allowing +pre-release versions to be explicitly requested when needed. The "some forward compatibility assumed" default version constraint is -taken directly from the Ruby community's "pessimistic version -constraint" operator [2]_ to allow projects to take a cautious -approach to forward compatibility promises, while still easily setting -a minimum required version for their dependencies. It is made the -default behaviour rather than needing a separate operator in order to -explicitly discourage overspecification of dependencies by library -developers. The explicit comparison operators remain available to cope -with dependencies with unreliable or non-existent backwards -compatibility policies, as well as for legitimate use cases related to -deployment of integrated applications. +taken directly from the Ruby community's "pessimistic version constraint" +operator [2]_ to allow projects to take a cautious approach to forward +compatibility promises, while still easily setting a minimum required +version for their dependencies. It is made the default behaviour rather +than needing a separate operator in order to explicitly discourage +overspecification of dependencies by library developers. The explicit +comparison operators remain available to cope with dependencies with +unreliable or non-existent backwards compatibility policies, as well +as for legitimate use cases related to deployment of integrated applications. -The two kinds of version matching (strict and prefix based) were -separated to make it possible to sensibly define the compatible -release clauses and the desired pre-release handling semantics for -``<`` and ``>`` ordered comparison clauses. +The optional explicit spelling of the compatible release clause (``~=``) is +inspired by the Ruby (``~>``) and PHP (``~``) equivalents. It is defined +in order to allow easier conversion to the legacy ``pkg_resources`` version +specifier format (which omits the parentheses, but requires a comparison +operator). + +Further improvements are also planned to the handling of parallel +installation of multiple versions of the same library, but these will +depend on updates to the installation database definition along with +improved tools for dynamic path manipulation. + +The trailing wildcard syntax to request prefix based version matching was +added to make it possible to sensibly define both compatible release clauses +and the desired pre-release handling semantics for ``<`` and ``>`` ordered +comparison clauses. + +Build references are added for two purposes. In conjunction with build +labels, they allow hash based references, such as those employed by +`hashdist <http://hashdist.readthedocs.org/en/latest/build_spec.html>`__, +or generated from version control. In conjunction with build URLs, they +allow the new metadata standard to natively support an existing feature of +``pip``, which allows arbitrary URLs like +``file:///localbuilds/exampledist-1.0-py33-none-any.whl``. References @@ -1028,18 +1056,18 @@ The initial attempt at a standardised version scheme, along with the justifications for needing such a standard can be found in PEP 386. -.. [1] Version compatibility analysis script - (http://hg.python.org/peps/file/default/pep-0426/pepsort.py) +.. [1] Version compatibility analysis script: + http://hg.python.org/peps/file/default/pep-0426/pepsort.py .. [2] Pessimistic version constraint - (http://docs.rubygems.org/read/chapter/16) + http://docs.rubygems.org/read/chapter/16 Appendix A ========== -Metadata v2.0 guidelines versus setuptools (note that this analysis -was run when this PEP was still embedded as part of PEP 426):: +Metadata v2.0 guidelines versus setuptools (note that this analysis was +run when this PEP was still embedded as part of PEP 426):: $ ./pepsort.py Comparing PEP 426 version sort to setuptools. @@ -1110,7 +1138,6 @@ This document has been placed in the public domain. - .. Local Variables: mode: indented-text -- Repository URL: http://hg.python.org/peps

1 0