[Python-checkins] cpython: Factor out code used by packaging commands for HTTP requests (#12169).
eric.araujo
python-checkins at python.org
Fri Jul 8 16:28:30 CEST 2011
http://hg.python.org/cpython/rev/c2785ed52ed4
changeset: 71253:c2785ed52ed4
user: Éric Araujo <merwok at netwok.org>
date: Fri Jul 08 16:27:12 2011 +0200
summary:
Factor out code used by packaging commands for HTTP requests (#12169).
We now have one function to prepare multipart POST requests, and we use
CRLF, as recommended by the HTTP spec (#10150). Initial patch by John
Edmonds.
files:
Lib/packaging/command/register.py | 24 +----
Lib/packaging/command/upload.py | 50 +--------
Lib/packaging/command/upload_docs.py | 46 +--------
Lib/packaging/tests/test_command_register.py | 10 +-
Lib/packaging/tests/test_command_upload_docs.py | 27 +-----
Lib/packaging/tests/test_util.py | 26 +++++-
Lib/packaging/util.py | 47 +++++++++
Misc/ACKS | 1 +
Misc/NEWS | 3 +
9 files changed, 96 insertions(+), 138 deletions(-)
diff --git a/Lib/packaging/command/register.py b/Lib/packaging/command/register.py
--- a/Lib/packaging/command/register.py
+++ b/Lib/packaging/command/register.py
@@ -10,7 +10,7 @@
from packaging import logger
from packaging.util import (read_pypirc, generate_pypirc, DEFAULT_REPOSITORY,
- DEFAULT_REALM, get_pypirc_path)
+ DEFAULT_REALM, get_pypirc_path, encode_multipart)
from packaging.command.cmd import Command
class register(Command):
@@ -231,29 +231,11 @@
if 'name' in data:
logger.info('Registering %s to %s', data['name'], self.repository)
# Build up the MIME payload for the urllib2 POST data
- boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
- sep_boundary = '\n--' + boundary
- end_boundary = sep_boundary + '--'
- body = io.StringIO()
- for key, value in data.items():
- # handle multiple entries for the same name
- if not isinstance(value, (tuple, list)):
- value = [value]
-
- for value in value:
- body.write(sep_boundary)
- body.write('\nContent-Disposition: form-data; name="%s"'%key)
- body.write("\n\n")
- body.write(value)
- if value and value[-1] == '\r':
- body.write('\n') # write an extra newline (lurve Macs)
- body.write(end_boundary)
- body.write("\n")
- body = body.getvalue()
+ content_type, body = encode_multipart(data.items(), [])
# build the Request
headers = {
- 'Content-type': 'multipart/form-data; boundary=%s; charset=utf-8'%boundary,
+ 'Content-type': content_type,
'Content-length': str(len(body))
}
req = urllib.request.Request(self.repository, body, headers)
diff --git a/Lib/packaging/command/upload.py b/Lib/packaging/command/upload.py
--- a/Lib/packaging/command/upload.py
+++ b/Lib/packaging/command/upload.py
@@ -14,7 +14,7 @@
from packaging import logger
from packaging.errors import PackagingOptionError
from packaging.util import (spawn, read_pypirc, DEFAULT_REPOSITORY,
- DEFAULT_REALM)
+ DEFAULT_REALM, encode_multipart)
from packaging.command.cmd import Command
@@ -131,54 +131,22 @@
auth = b"Basic " + standard_b64encode(user_pass)
# Build up the MIME payload for the POST data
- boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
- sep_boundary = b'\n--' + boundary
- end_boundary = sep_boundary + b'--'
- body = BytesIO()
+ files = []
+ for key in ('content', 'gpg_signature'):
+ if key in data:
+ filename_, value = data.pop(key)
+ files.append((key, filename_, value))
- file_fields = ('content', 'gpg_signature')
-
- for key, value in data.items():
- # handle multiple entries for the same name
- if not isinstance(value, tuple):
- value = [value]
-
- content_dispo = '\nContent-Disposition: form-data; name="%s"' % key
-
- if key in file_fields:
- filename_, content = value
- filename_ = ';filename="%s"' % filename_
- body.write(sep_boundary)
- body.write(content_dispo.encode('utf-8'))
- body.write(filename_.encode('utf-8'))
- body.write(b"\n\n")
- body.write(content)
- else:
- for value in value:
- value = str(value).encode('utf-8')
- body.write(sep_boundary)
- body.write(content_dispo.encode('utf-8'))
- body.write(b"\n\n")
- body.write(value)
- if value and value.endswith(b'\r'):
- # write an extra newline (lurve Macs)
- body.write(b'\n')
-
- body.write(end_boundary)
- body.write(b"\n")
- body = body.getvalue()
+ content_type, body = encode_multipart(data.items(), files)
logger.info("Submitting %s to %s", filename, self.repository)
# build the Request
- headers = {'Content-type':
- 'multipart/form-data; boundary=%s' %
- boundary.decode('ascii'),
+ headers = {'Content-type': content_type,
'Content-length': str(len(body)),
'Authorization': auth}
- request = Request(self.repository, data=body,
- headers=headers)
+ request = Request(self.repository, body, headers)
# send the data
try:
result = urlopen(request)
diff --git a/Lib/packaging/command/upload_docs.py b/Lib/packaging/command/upload_docs.py
--- a/Lib/packaging/command/upload_docs.py
+++ b/Lib/packaging/command/upload_docs.py
@@ -10,7 +10,8 @@
from io import BytesIO
from packaging import logger
-from packaging.util import read_pypirc, DEFAULT_REPOSITORY, DEFAULT_REALM
+from packaging.util import (read_pypirc, DEFAULT_REPOSITORY, DEFAULT_REALM,
+ encode_multipart)
from packaging.errors import PackagingFileError
from packaging.command.cmd import Command
@@ -28,49 +29,6 @@
return destination
-# grabbed from
-# http://code.activestate.com/recipes/
-# 146306-http-client-to-post-using-multipartform-data/
-# TODO factor this out for use by install and command/upload
-
-def encode_multipart(fields, files, boundary=None):
- """
- *fields* is a sequence of (name: str, value: str) elements for regular
- form fields, *files* is a sequence of (name: str, filename: str, value:
- bytes) elements for data to be uploaded as files.
-
- Returns (content_type: bytes, body: bytes) ready for http.client.HTTP.
- """
- if boundary is None:
- boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
- elif not isinstance(boundary, bytes):
- raise TypeError('boundary is not bytes but %r' % type(boundary))
-
- l = []
- for key, value in fields:
- l.extend((
- b'--' + boundary,
- ('Content-Disposition: form-data; name="%s"' %
- key).encode('utf-8'),
- b'',
- value.encode('utf-8')))
-
- for key, filename, value in files:
- l.extend((
- b'--' + boundary,
- ('Content-Disposition: form-data; name="%s"; filename="%s"' %
- (key, filename)).encode('utf-8'),
- b'',
- value))
- l.append(b'--' + boundary + b'--')
- l.append(b'')
-
- body = b'\r\n'.join(l)
-
- content_type = b'multipart/form-data; boundary=' + boundary
- return content_type, body
-
-
class upload_docs(Command):
description = "upload HTML documentation to PyPI"
diff --git a/Lib/packaging/tests/test_command_register.py b/Lib/packaging/tests/test_command_register.py
--- a/Lib/packaging/tests/test_command_register.py
+++ b/Lib/packaging/tests/test_command_register.py
@@ -152,7 +152,7 @@
req1 = dict(self.conn.reqs[0].headers)
req2 = dict(self.conn.reqs[1].headers)
self.assertEqual(req2['Content-length'], req1['Content-length'])
- self.assertIn('xxx', self.conn.reqs[1].data)
+ self.assertIn(b'xxx', self.conn.reqs[1].data)
def test_password_not_in_file(self):
@@ -180,8 +180,8 @@
self.assertEqual(len(self.conn.reqs), 1)
req = self.conn.reqs[0]
headers = dict(req.headers)
- self.assertEqual(headers['Content-length'], '608')
- self.assertIn('tarek', req.data)
+ self.assertEqual(headers['Content-length'], '628')
+ self.assertIn(b'tarek', req.data)
def test_password_reset(self):
# this test runs choice 3
@@ -195,8 +195,8 @@
self.assertEqual(len(self.conn.reqs), 1)
req = self.conn.reqs[0]
headers = dict(req.headers)
- self.assertEqual(headers['Content-length'], '290')
- self.assertIn('tarek', req.data)
+ self.assertEqual(headers['Content-length'], '298')
+ self.assertIn(b'tarek', req.data)
@unittest.skipUnless(DOCUTILS_SUPPORT, 'needs docutils')
def test_strict(self):
diff --git a/Lib/packaging/tests/test_command_upload_docs.py b/Lib/packaging/tests/test_command_upload_docs.py
--- a/Lib/packaging/tests/test_command_upload_docs.py
+++ b/Lib/packaging/tests/test_command_upload_docs.py
@@ -9,8 +9,7 @@
_ssl = None
from packaging.command import upload_docs as upload_docs_mod
-from packaging.command.upload_docs import (upload_docs, zip_dir,
- encode_multipart)
+from packaging.command.upload_docs import upload_docs, zip_dir
from packaging.dist import Distribution
from packaging.errors import PackagingFileError, PackagingOptionError
@@ -23,23 +22,6 @@
PyPIServerTestCase = object
-EXPECTED_MULTIPART_OUTPUT = [
- b'---x',
- b'Content-Disposition: form-data; name="username"',
- b'',
- b'wok',
- b'---x',
- b'Content-Disposition: form-data; name="password"',
- b'',
- b'secret',
- b'---x',
- b'Content-Disposition: form-data; name="picture"; filename="wok.png"',
- b'',
- b'PNG89',
- b'---x--',
- b'',
-]
-
PYPIRC = """\
[distutils]
index-servers = server1
@@ -108,13 +90,6 @@
zip_f = zipfile.ZipFile(compressed)
self.assertEqual(zip_f.namelist(), ['index.html', 'docs/index.html'])
- def test_encode_multipart(self):
- fields = [('username', 'wok'), ('password', 'secret')]
- files = [('picture', 'wok.png', b'PNG89')]
- content_type, body = encode_multipart(fields, files, b'-x')
- self.assertEqual(b'multipart/form-data; boundary=-x', content_type)
- self.assertEqual(EXPECTED_MULTIPART_OUTPUT, body.split(b'\r\n'))
-
def prepare_command(self):
self.cmd.upload_dir = self.prepare_sample_dir()
self.cmd.ensure_finalized()
diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py
--- a/Lib/packaging/tests/test_util.py
+++ b/Lib/packaging/tests/test_util.py
@@ -19,7 +19,7 @@
get_compiler_versions, _MAC_OS_X_LD_VERSION, byte_compile, find_packages,
spawn, get_pypirc_path, generate_pypirc, read_pypirc, resolve_name, iglob,
RICH_GLOB, egginfo_to_distinfo, is_setuptools, is_distutils, is_packaging,
- get_install_method, cfg_to_args)
+ get_install_method, cfg_to_args, encode_multipart)
PYPIRC = """\
@@ -54,6 +54,23 @@
password:xxx
"""
+EXPECTED_MULTIPART_OUTPUT = [
+ b'---x',
+ b'Content-Disposition: form-data; name="username"',
+ b'',
+ b'wok',
+ b'---x',
+ b'Content-Disposition: form-data; name="password"',
+ b'',
+ b'secret',
+ b'---x',
+ b'Content-Disposition: form-data; name="picture"; filename="wok.png"',
+ b'',
+ b'PNG89',
+ b'---x--',
+ b'',
+]
+
class FakePopen:
test_class = None
@@ -525,6 +542,13 @@
self.assertEqual(args['scripts'], dist.scripts)
self.assertEqual(args['py_modules'], dist.py_modules)
+ def test_encode_multipart(self):
+ fields = [('username', 'wok'), ('password', 'secret')]
+ files = [('picture', 'wok.png', b'PNG89')]
+ content_type, body = encode_multipart(fields, files, b'-x')
+ self.assertEqual(b'multipart/form-data; boundary=-x', content_type)
+ self.assertEqual(EXPECTED_MULTIPART_OUTPUT, body.split(b'\r\n'))
+
class GlobTestCaseBase(support.TempdirManager,
support.LoggingCatcher,
diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py
--- a/Lib/packaging/util.py
+++ b/Lib/packaging/util.py
@@ -1487,3 +1487,50 @@
_path_created.add(abs_head)
return created_dirs
+
+
+def encode_multipart(fields, files, boundary=None):
+ """Prepare a multipart HTTP request.
+
+ *fields* is a sequence of (name: str, value: str) elements for regular
+ form fields, *files* is a sequence of (name: str, filename: str, value:
+ bytes) elements for data to be uploaded as files.
+
+ Returns (content_type: bytes, body: bytes) ready for http.client.HTTP.
+ """
+ # Taken from
+ # http://code.activestate.com/recipes/146306-http-client-to-post-using-multipartform-data/
+
+ if boundary is None:
+ boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
+ elif not isinstance(boundary, bytes):
+ raise TypeError('boundary must be bytes, not %r' % type(boundary))
+
+ l = []
+ for key, values in fields:
+ # handle multiple entries for the same name
+ if not isinstance(values, (tuple, list)):
+ values=[values]
+
+ for value in values:
+ l.extend((
+ b'--' + boundary,
+ ('Content-Disposition: form-data; name="%s"' %
+ key).encode('utf-8'),
+ b'',
+ value.encode('utf-8')))
+
+ for key, filename, value in files:
+ l.extend((
+ b'--' + boundary,
+ ('Content-Disposition: form-data; name="%s"; filename="%s"' %
+ (key, filename)).encode('utf-8'),
+ b'',
+ value))
+
+ l.append(b'--' + boundary + b'--')
+ l.append(b'')
+
+ body = b'\r\n'.join(l)
+ content_type = b'multipart/form-data; boundary=' + boundary
+ return content_type, body
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -263,6 +263,7 @@
Walter Dörwald
Hans Eckardt
Rodolpho Eckhardt
+John Edmonds
Grant Edwards
John Ehresman
Eric Eisner
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -219,6 +219,9 @@
Library
-------
+- Issues #12169 and #10510: Factor out code used by various packaging commands
+ to make HTTP POST requests, and make sure it uses CRLF.
+
- Issue #12016: Multibyte CJK decoders now resynchronize faster. They only
ignore the first byte of an invalid byte sequence. For example,
b'\xff\n'.decode('gb2312', 'replace') gives '\ufffd\n' instead of '\ufffd'.
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list