[pypy-commit] pypy 32ptr-on-64bit: hg merge default
arigo
noreply at buildbot.pypy.org
Sun Jan 22 20:28:04 CET 2012
Author: Armin Rigo <arigo at tunes.org>
Branch: 32ptr-on-64bit
Changeset: r51659:35da68422201
Date: 2012-01-22 20:12 +0100
http://bitbucket.org/pypy/pypy/changeset/35da68422201/
Log: hg merge default
diff too long, truncating to 10000 out of 192957 lines
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,6 +1,10 @@
syntax: glob
*.py[co]
*~
+.*.swp
+.idea
+.project
+.pydevproject
syntax: regexp
^testresult$
@@ -24,6 +28,8 @@
^pypy/translator/c/src/libffi_msvc/.+\.dll$
^pypy/translator/c/src/libffi_msvc/.+\.lib$
^pypy/translator/c/src/libffi_msvc/.+\.exp$
+^pypy/translator/c/src/cjkcodecs/.+\.o$
+^pypy/translator/c/src/cjkcodecs/.+\.obj$
^pypy/translator/jvm/\.project$
^pypy/translator/jvm/\.classpath$
^pypy/translator/jvm/eclipse-bin$
@@ -36,6 +42,8 @@
^pypy/translator/benchmark/shootout_benchmarks$
^pypy/translator/goal/pypy-translation-snapshot$
^pypy/translator/goal/pypy-c
+^pypy/translator/goal/pypy-jvm
+^pypy/translator/goal/pypy-jvm.jar
^pypy/translator/goal/.+\.exe$
^pypy/translator/goal/.+\.dll$
^pypy/translator/goal/target.+-c$
@@ -62,6 +70,7 @@
^pypy/doc/image/lattice3\.png$
^pypy/doc/image/stackless_informal\.png$
^pypy/doc/image/parsing_example.+\.png$
+^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
^compiled
^.git/
^release/
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -1,1 +1,4 @@
b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
+b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
+d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
+ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -27,7 +27,7 @@
DEALINGS IN THE SOFTWARE.
-PyPy Copyright holders 2003-2011
+PyPy Copyright holders 2003-2012
-----------------------------------
Except when otherwise stated (look for LICENSE files or information at
@@ -42,34 +42,39 @@
Samuele Pedroni
Michael Hudson
Holger Krekel
+ Alex Gaynor
Christian Tismer
+ Hakan Ardo
Benjamin Peterson
+ David Schneider
Eric van Riet Paap
- Anders Chrigström
- Håkan Ardö
+ Anders Chrigstrom
Richard Emslie
Dan Villiom Podlaski Christiansen
Alexander Schremmer
- Alex Gaynor
- David Schneider
- Aurelién Campeas
+ Lukas Diekmann
+ Aurelien Campeas
Anders Lehmann
Camillo Bruni
Niklaus Haldimann
+ Sven Hager
Leonardo Santagada
Toon Verwaest
Seo Sanghyeon
+ Justin Peel
Lawrence Oluyede
Bartosz Skowron
Jakub Gustak
Guido Wesdorp
+ Daniel Roberts
+ Laura Creighton
Adrien Di Mascio
- Laura Creighton
Ludovic Aubry
Niko Matsakis
- Daniel Roberts
+ Wim Lavrijsen
+ Matti Picus
Jason Creighton
- Jacob Hallén
+ Jacob Hallen
Alex Martelli
Anders Hammarquist
Jan de Mooij
@@ -83,29 +88,36 @@
Alexandre Fayolle
Marius Gedminas
Simon Burton
+ David Edelsohn
Jean-Paul Calderone
John Witulski
- Wim Lavrijsen
+ Timo Paulssen
+ holger krekel
+ Dario Bertini
+ Mark Pearse
Andreas Stührk
Jean-Philippe St. Pierre
Guido van Rossum
Pavel Vinogradov
Valentino Volonghi
Paul deGrandis
+ Ilya Osadchiy
+ Ronny Pfannschmidt
Adrian Kuhn
tav
Georg Brandl
+ Philip Jenvey
Gerald Klix
Wanja Saatkamp
Boris Feigin
Oscar Nierstrasz
- Dario Bertini
David Malcolm
Eugene Oden
Henry Mason
+ Jeff Terrace
Lukas Renggli
Guenter Jantzen
- Ronny Pfannschmidt
+ Ned Batchelder
Bert Freudenberg
Amit Regmi
Ben Young
@@ -122,8 +134,8 @@
Jared Grubb
Karl Bartel
Gabriel Lavoie
+ Victor Stinner
Brian Dorsey
- Victor Stinner
Stuart Williams
Toby Watson
Antoine Pitrou
@@ -134,19 +146,22 @@
Jonathan David Riehl
Elmo Mäntynen
Anders Qvist
- Beatrice Düring
+ Beatrice During
Alexander Sedov
+ Corbin Simpson
Vincent Legoll
+ Romain Guillebert
Alan McIntyre
- Romain Guillebert
Alex Perry
Jens-Uwe Mager
+ Simon Cross
Dan Stromberg
- Lukas Diekmann
+ Guillebert Romain
Carl Meyer
Pieter Zieschang
Alejandro J. Cura
Sylvain Thenault
+ Christoph Gerum
Travis Francis Athougies
Henrik Vendelbo
Lutz Paelike
@@ -155,8 +170,10 @@
Lucio Torre
Lene Wagner
Miguel de Val Borro
+ Artur Lisiecki
+ Bruno Gola
Ignas Mikalajunas
- Artur Lisiecki
+ Stefano Rivera
Joshua Gilbert
Godefroid Chappelle
Yusei Tahara
@@ -165,26 +182,31 @@
Gustavo Niemeyer
William Leslie
Akira Li
- Kristján Valur Jónsson
+ Kristjan Valur Jonsson
Bobby Impollonia
+ Michael Hudson-Doyle
+ Laurence Tratt
+ Yasir Suhail
Andrew Thompson
Anders Sigfridsson
+ Floris Bruynooghe
Jacek Generowicz
Dan Colish
- Sven Hager
Zooko Wilcox-O Hearn
- Anders Hammarquist
+ Dan Loewenherz
+ Chris Lambacher
Dinu Gherman
- Dan Colish
+ Brett Cannon
Daniel Neuhäuser
Michael Chermside
Konrad Delong
Anna Ravencroft
Greg Price
Armin Ronacher
+ Christian Muirhead
Jim Baker
- Philip Jenvey
Rodrigo Araújo
+ Romain Guillebert
Heinrich-Heine University, Germany
Open End AB (formerly AB Strakt), Sweden
diff --git a/README b/README
--- a/README
+++ b/README
@@ -15,10 +15,10 @@
The getting-started document will help guide you:
- http://codespeak.net/pypy/dist/pypy/doc/getting-started.html
+ http://doc.pypy.org/en/latest/getting-started.html
It will also point you to the rest of the documentation which is generated
from files in the pypy/doc directory within the source repositories. Enjoy
and send us feedback!
- the pypy-dev team <pypy-dev at codespeak.net>
+ the pypy-dev team <pypy-dev at python.org>
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
#
-__version__ = '2.0.3'
+__version__ = '2.1.0.dev4'
diff --git a/_pytest/assertion.py b/_pytest/assertion.py
deleted file mode 100644
--- a/_pytest/assertion.py
+++ /dev/null
@@ -1,177 +0,0 @@
-"""
-support for presented detailed information in failing assertions.
-"""
-import py
-import sys
-from _pytest.monkeypatch import monkeypatch
-
-def pytest_addoption(parser):
- group = parser.getgroup("debugconfig")
- group._addoption('--no-assert', action="store_true", default=False,
- dest="noassert",
- help="disable python assert expression reinterpretation."),
-
-def pytest_configure(config):
- # The _reprcompare attribute on the py.code module is used by
- # py._code._assertionnew to detect this plugin was loaded and in
- # turn call the hooks defined here as part of the
- # DebugInterpreter.
- m = monkeypatch()
- config._cleanup.append(m.undo)
- warn_about_missing_assertion()
- if not config.getvalue("noassert") and not config.getvalue("nomagic"):
- def callbinrepr(op, left, right):
- hook_result = config.hook.pytest_assertrepr_compare(
- config=config, op=op, left=left, right=right)
- for new_expl in hook_result:
- if new_expl:
- return '\n~'.join(new_expl)
- m.setattr(py.builtin.builtins,
- 'AssertionError', py.code._AssertionError)
- m.setattr(py.code, '_reprcompare', callbinrepr)
-
-def warn_about_missing_assertion():
- try:
- assert False
- except AssertionError:
- pass
- else:
- sys.stderr.write("WARNING: failing tests may report as passing because "
- "assertions are turned off! (are you using python -O?)\n")
-
-# Provide basestring in python3
-try:
- basestring = basestring
-except NameError:
- basestring = str
-
-
-def pytest_assertrepr_compare(op, left, right):
- """return specialised explanations for some operators/operands"""
- width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
- left_repr = py.io.saferepr(left, maxsize=int(width/2))
- right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
- summary = '%s %s %s' % (left_repr, op, right_repr)
-
- issequence = lambda x: isinstance(x, (list, tuple))
- istext = lambda x: isinstance(x, basestring)
- isdict = lambda x: isinstance(x, dict)
- isset = lambda x: isinstance(x, set)
-
- explanation = None
- try:
- if op == '==':
- if istext(left) and istext(right):
- explanation = _diff_text(left, right)
- elif issequence(left) and issequence(right):
- explanation = _compare_eq_sequence(left, right)
- elif isset(left) and isset(right):
- explanation = _compare_eq_set(left, right)
- elif isdict(left) and isdict(right):
- explanation = _diff_text(py.std.pprint.pformat(left),
- py.std.pprint.pformat(right))
- elif op == 'not in':
- if istext(left) and istext(right):
- explanation = _notin_text(left, right)
- except py.builtin._sysex:
- raise
- except:
- excinfo = py.code.ExceptionInfo()
- explanation = ['(pytest_assertion plugin: representation of '
- 'details failed. Probably an object has a faulty __repr__.)',
- str(excinfo)
- ]
-
-
- if not explanation:
- return None
-
- # Don't include pageloads of data, should be configurable
- if len(''.join(explanation)) > 80*8:
- explanation = ['Detailed information too verbose, truncated']
-
- return [summary] + explanation
-
-
-def _diff_text(left, right):
- """Return the explanation for the diff between text
-
- This will skip leading and trailing characters which are
- identical to keep the diff minimal.
- """
- explanation = []
- i = 0 # just in case left or right has zero length
- for i in range(min(len(left), len(right))):
- if left[i] != right[i]:
- break
- if i > 42:
- i -= 10 # Provide some context
- explanation = ['Skipping %s identical '
- 'leading characters in diff' % i]
- left = left[i:]
- right = right[i:]
- if len(left) == len(right):
- for i in range(len(left)):
- if left[-i] != right[-i]:
- break
- if i > 42:
- i -= 10 # Provide some context
- explanation += ['Skipping %s identical '
- 'trailing characters in diff' % i]
- left = left[:-i]
- right = right[:-i]
- explanation += [line.strip('\n')
- for line in py.std.difflib.ndiff(left.splitlines(),
- right.splitlines())]
- return explanation
-
-
-def _compare_eq_sequence(left, right):
- explanation = []
- for i in range(min(len(left), len(right))):
- if left[i] != right[i]:
- explanation += ['At index %s diff: %r != %r' %
- (i, left[i], right[i])]
- break
- if len(left) > len(right):
- explanation += ['Left contains more items, '
- 'first extra item: %s' % py.io.saferepr(left[len(right)],)]
- elif len(left) < len(right):
- explanation += ['Right contains more items, '
- 'first extra item: %s' % py.io.saferepr(right[len(left)],)]
- return explanation # + _diff_text(py.std.pprint.pformat(left),
- # py.std.pprint.pformat(right))
-
-
-def _compare_eq_set(left, right):
- explanation = []
- diff_left = left - right
- diff_right = right - left
- if diff_left:
- explanation.append('Extra items in the left set:')
- for item in diff_left:
- explanation.append(py.io.saferepr(item))
- if diff_right:
- explanation.append('Extra items in the right set:')
- for item in diff_right:
- explanation.append(py.io.saferepr(item))
- return explanation
-
-
-def _notin_text(term, text):
- index = text.find(term)
- head = text[:index]
- tail = text[index+len(term):]
- correct_text = head + tail
- diff = _diff_text(correct_text, text)
- newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
- for line in diff:
- if line.startswith('Skipping'):
- continue
- if line.startswith('- '):
- continue
- if line.startswith('+ '):
- newdiff.append(' ' + line[2:])
- else:
- newdiff.append(line)
- return newdiff
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/__init__.py
@@ -0,0 +1,128 @@
+"""
+support for presenting detailed information in failing assertions.
+"""
+import py
+import imp
+import marshal
+import struct
+import sys
+import pytest
+from _pytest.monkeypatch import monkeypatch
+from _pytest.assertion import reinterpret, util
+
+try:
+ from _pytest.assertion.rewrite import rewrite_asserts
+except ImportError:
+ rewrite_asserts = None
+else:
+ import ast
+
+def pytest_addoption(parser):
+ group = parser.getgroup("debugconfig")
+ group.addoption('--assertmode', action="store", dest="assertmode",
+ choices=("on", "old", "off", "default"), default="default",
+ metavar="on|old|off",
+ help="""control assertion debugging tools.
+'off' performs no assertion debugging.
+'old' reinterprets the expressions in asserts to glean information.
+'on' (the default) rewrites the assert statements in test modules to provide
+sub-expression results.""")
+ group.addoption('--no-assert', action="store_true", default=False,
+ dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+ group.addoption('--nomagic', action="store_true", default=False,
+ dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+
+class AssertionState:
+ """State for the assertion plugin."""
+
+ def __init__(self, config, mode):
+ self.mode = mode
+ self.trace = config.trace.root.get("assertion")
+
+def pytest_configure(config):
+ warn_about_missing_assertion()
+ mode = config.getvalue("assertmode")
+ if config.getvalue("noassert") or config.getvalue("nomagic"):
+ if mode not in ("off", "default"):
+ raise pytest.UsageError("assertion options conflict")
+ mode = "off"
+ elif mode == "default":
+ mode = "on"
+ if mode != "off":
+ def callbinrepr(op, left, right):
+ hook_result = config.hook.pytest_assertrepr_compare(
+ config=config, op=op, left=left, right=right)
+ for new_expl in hook_result:
+ if new_expl:
+ return '\n~'.join(new_expl)
+ m = monkeypatch()
+ config._cleanup.append(m.undo)
+ m.setattr(py.builtin.builtins, 'AssertionError',
+ reinterpret.AssertionError)
+ m.setattr(util, '_reprcompare', callbinrepr)
+ if mode == "on" and rewrite_asserts is None:
+ mode = "old"
+ config._assertstate = AssertionState(config, mode)
+ config._assertstate.trace("configured with mode set to %r" % (mode,))
+
+def _write_pyc(co, source_path):
+ if hasattr(imp, "cache_from_source"):
+ # Handle PEP 3147 pycs.
+ pyc = py.path.local(imp.cache_from_source(str(source_path)))
+ pyc.ensure()
+ else:
+ pyc = source_path + "c"
+ mtime = int(source_path.mtime())
+ fp = pyc.open("wb")
+ try:
+ fp.write(imp.get_magic())
+ fp.write(struct.pack("<l", mtime))
+ marshal.dump(co, fp)
+ finally:
+ fp.close()
+ return pyc
+
+def before_module_import(mod):
+ if mod.config._assertstate.mode != "on":
+ return
+ # Some deep magic: load the source, rewrite the asserts, and write a
+ # fake pyc, so that it'll be loaded when the module is imported.
+ source = mod.fspath.read()
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ # Let this pop up again in the real import.
+ mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
+ return
+ rewrite_asserts(tree)
+ try:
+ co = compile(tree, str(mod.fspath), "exec")
+ except SyntaxError:
+ # It's possible that this error is from some bug in the assertion
+ # rewriting, but I don't know of a fast way to tell.
+ mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
+ return
+ mod._pyc = _write_pyc(co, mod.fspath)
+ mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+
+def after_module_import(mod):
+ if not hasattr(mod, "_pyc"):
+ return
+ state = mod.config._assertstate
+ try:
+ mod._pyc.remove()
+ except py.error.ENOENT:
+ state.trace("couldn't find pyc: %r" % (mod._pyc,))
+ else:
+ state.trace("removed pyc: %r" % (mod._pyc,))
+
+def warn_about_missing_assertion():
+ try:
+ assert False
+ except AssertionError:
+ pass
+ else:
+ sys.stderr.write("WARNING: failing tests may report as passing because "
+ "assertions are turned off! (are you using python -O?)\n")
+
+pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/newinterpret.py
@@ -0,0 +1,333 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace oldinterpret.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from _pytest.assertion import util
+from _pytest.assertion.reinterpret import BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+ # See http://bugs.jython.org/issue1497
+ _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+ "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+ "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+ "List", "Tuple")
+ _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+ "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+ "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+ "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+ _expr_nodes = set(getattr(ast, name) for name in _exprs)
+ _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+ def _is_ast_expr(node):
+ return node.__class__ in _expr_nodes
+ def _is_ast_stmt(node):
+ return node.__class__ in _stmt_nodes
+else:
+ def _is_ast_expr(node):
+ return isinstance(node, ast.expr)
+ def _is_ast_stmt(node):
+ return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+ """Error found while interpreting AST."""
+
+ def __init__(self, explanation=""):
+ self.cause = sys.exc_info()
+ self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+ mod = ast.parse(source)
+ visitor = DebugInterpreter(frame)
+ try:
+ visitor.visit(mod)
+ except Failure:
+ failure = sys.exc_info()[1]
+ return getfailure(failure)
+ if should_fail:
+ return ("(assertion failed, but when it was re-run for "
+ "printing intermediate values, it did not fail. Suggestions: "
+ "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+ if frame is None:
+ frame = py.code.Frame(sys._getframe(1))
+ return interpret(offending_line, frame)
+
+def getfailure(e):
+ explanation = util.format_explanation(e.explanation)
+ value = e.cause[1]
+ if str(value):
+ lines = explanation.split('\n')
+ lines[0] += " << %s" % (value,)
+ explanation = '\n'.join(lines)
+ text = "%s: %s" % (e.cause[0].__name__, explanation)
+ if text.startswith('AssertionError: assert '):
+ text = text[16:]
+ return text
+
+operator_map = {
+ ast.BitOr : "|",
+ ast.BitXor : "^",
+ ast.BitAnd : "&",
+ ast.LShift : "<<",
+ ast.RShift : ">>",
+ ast.Add : "+",
+ ast.Sub : "-",
+ ast.Mult : "*",
+ ast.Div : "/",
+ ast.FloorDiv : "//",
+ ast.Mod : "%",
+ ast.Eq : "==",
+ ast.NotEq : "!=",
+ ast.Lt : "<",
+ ast.LtE : "<=",
+ ast.Gt : ">",
+ ast.GtE : ">=",
+ ast.Pow : "**",
+ ast.Is : "is",
+ ast.IsNot : "is not",
+ ast.In : "in",
+ ast.NotIn : "not in"
+}
+
+unary_map = {
+ ast.Not : "not %s",
+ ast.Invert : "~%s",
+ ast.USub : "-%s",
+ ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+ """Interpret AST nodes to gleam useful debugging information. """
+
+ def __init__(self, frame):
+ self.frame = frame
+
+ def generic_visit(self, node):
+ # Fallback when we don't have a special implementation.
+ if _is_ast_expr(node):
+ mod = ast.Expression(node)
+ co = self._compile(mod)
+ try:
+ result = self.frame.eval(co)
+ except Exception:
+ raise Failure()
+ explanation = self.frame.repr(result)
+ return explanation, result
+ elif _is_ast_stmt(node):
+ mod = ast.Module([node])
+ co = self._compile(mod, "exec")
+ try:
+ self.frame.exec_(co)
+ except Exception:
+ raise Failure()
+ return None, None
+ else:
+ raise AssertionError("can't handle %s" %(node,))
+
+ def _compile(self, source, mode="eval"):
+ return compile(source, "<assertion interpretation>", mode)
+
+ def visit_Expr(self, expr):
+ return self.visit(expr.value)
+
+ def visit_Module(self, mod):
+ for stmt in mod.body:
+ self.visit(stmt)
+
+ def visit_Name(self, name):
+ explanation, result = self.generic_visit(name)
+ # See if the name is local.
+ source = "%r in locals() is not globals()" % (name.id,)
+ co = self._compile(source)
+ try:
+ local = self.frame.eval(co)
+ except Exception:
+ # have to assume it isn't
+ local = None
+ if local is None or not self.frame.is_true(local):
+ return name.id, result
+ return explanation, result
+
+ def visit_Compare(self, comp):
+ left = comp.left
+ left_explanation, left_result = self.visit(left)
+ for op, next_op in zip(comp.ops, comp.comparators):
+ next_explanation, next_result = self.visit(next_op)
+ op_symbol = operator_map[op.__class__]
+ explanation = "%s %s %s" % (left_explanation, op_symbol,
+ next_explanation)
+ source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, __exprinfo_left=left_result,
+ __exprinfo_right=next_result)
+ except Exception:
+ raise Failure(explanation)
+ try:
+ if not self.frame.is_true(result):
+ break
+ except KeyboardInterrupt:
+ raise
+ except:
+ break
+ left_explanation, left_result = next_explanation, next_result
+
+ if util._reprcompare is not None:
+ res = util._reprcompare(op_symbol, left_result, next_result)
+ if res:
+ explanation = res
+ return explanation, result
+
+ def visit_BoolOp(self, boolop):
+ is_or = isinstance(boolop.op, ast.Or)
+ explanations = []
+ for operand in boolop.values:
+ explanation, result = self.visit(operand)
+ explanations.append(explanation)
+ if result == is_or:
+ break
+ name = is_or and " or " or " and "
+ explanation = "(" + name.join(explanations) + ")"
+ return explanation, result
+
+ def visit_UnaryOp(self, unary):
+ pattern = unary_map[unary.op.__class__]
+ operand_explanation, operand_result = self.visit(unary.operand)
+ explanation = pattern % (operand_explanation,)
+ co = self._compile(pattern % ("__exprinfo_expr",))
+ try:
+ result = self.frame.eval(co, __exprinfo_expr=operand_result)
+ except Exception:
+ raise Failure(explanation)
+ return explanation, result
+
+ def visit_BinOp(self, binop):
+ left_explanation, left_result = self.visit(binop.left)
+ right_explanation, right_result = self.visit(binop.right)
+ symbol = operator_map[binop.op.__class__]
+ explanation = "(%s %s %s)" % (left_explanation, symbol,
+ right_explanation)
+ source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, __exprinfo_left=left_result,
+ __exprinfo_right=right_result)
+ except Exception:
+ raise Failure(explanation)
+ return explanation, result
+
+ def visit_Call(self, call):
+ func_explanation, func = self.visit(call.func)
+ arg_explanations = []
+ ns = {"__exprinfo_func" : func}
+ arguments = []
+ for arg in call.args:
+ arg_explanation, arg_result = self.visit(arg)
+ arg_name = "__exprinfo_%s" % (len(ns),)
+ ns[arg_name] = arg_result
+ arguments.append(arg_name)
+ arg_explanations.append(arg_explanation)
+ for keyword in call.keywords:
+ arg_explanation, arg_result = self.visit(keyword.value)
+ arg_name = "__exprinfo_%s" % (len(ns),)
+ ns[arg_name] = arg_result
+ keyword_source = "%s=%%s" % (keyword.arg)
+ arguments.append(keyword_source % (arg_name,))
+ arg_explanations.append(keyword_source % (arg_explanation,))
+ if call.starargs:
+ arg_explanation, arg_result = self.visit(call.starargs)
+ arg_name = "__exprinfo_star"
+ ns[arg_name] = arg_result
+ arguments.append("*%s" % (arg_name,))
+ arg_explanations.append("*%s" % (arg_explanation,))
+ if call.kwargs:
+ arg_explanation, arg_result = self.visit(call.kwargs)
+ arg_name = "__exprinfo_kwds"
+ ns[arg_name] = arg_result
+ arguments.append("**%s" % (arg_name,))
+ arg_explanations.append("**%s" % (arg_explanation,))
+ args_explained = ", ".join(arg_explanations)
+ explanation = "%s(%s)" % (func_explanation, args_explained)
+ args = ", ".join(arguments)
+ source = "__exprinfo_func(%s)" % (args,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, **ns)
+ except Exception:
+ raise Failure(explanation)
+ pattern = "%s\n{%s = %s\n}"
+ rep = self.frame.repr(result)
+ explanation = pattern % (rep, rep, explanation)
+ return explanation, result
+
+ def _is_builtin_name(self, name):
+ pattern = "%r not in globals() and %r not in locals()"
+ source = pattern % (name.id, name.id)
+ co = self._compile(source)
+ try:
+ return self.frame.eval(co)
+ except Exception:
+ return False
+
+ def visit_Attribute(self, attr):
+ if not isinstance(attr.ctx, ast.Load):
+ return self.generic_visit(attr)
+ source_explanation, source_result = self.visit(attr.value)
+ explanation = "%s.%s" % (source_explanation, attr.attr)
+ source = "__exprinfo_expr.%s" % (attr.attr,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, __exprinfo_expr=source_result)
+ except Exception:
+ raise Failure(explanation)
+ explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+ self.frame.repr(result),
+ source_explanation, attr.attr)
+ # Check if the attr is from an instance.
+ source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+ source = source % (attr.attr,)
+ co = self._compile(source)
+ try:
+ from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+ except Exception:
+ from_instance = None
+ if from_instance is None or self.frame.is_true(from_instance):
+ rep = self.frame.repr(result)
+ pattern = "%s\n{%s = %s\n}"
+ explanation = pattern % (rep, rep, explanation)
+ return explanation, result
+
+ def visit_Assert(self, assrt):
+ test_explanation, test_result = self.visit(assrt.test)
+ explanation = "assert %s" % (test_explanation,)
+ if not self.frame.is_true(test_result):
+ try:
+ raise BuiltinAssertionError
+ except Exception:
+ raise Failure(explanation)
+ return explanation, test_result
+
+ def visit_Assign(self, assign):
+ value_explanation, value_result = self.visit(assign.value)
+ explanation = "... = %s" % (value_explanation,)
+ name = ast.Name("__exprinfo_expr", ast.Load(),
+ lineno=assign.value.lineno,
+ col_offset=assign.value.col_offset)
+ new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+ col_offset=assign.col_offset)
+ mod = ast.Module([new_assign])
+ co = self._compile(mod, "exec")
+ try:
+ self.frame.exec_(co, __exprinfo_expr=value_result)
+ except Exception:
+ raise Failure(explanation)
+ return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/oldinterpret.py
@@ -0,0 +1,552 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from _pytest.assertion.util import format_explanation
+from _pytest.assertion.reinterpret import BuiltinAssertionError
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+ def __init__(self, node):
+ self.exc, self.value, self.tb = sys.exc_info()
+ self.node = node
+
+class View(object):
+ """View base class.
+
+ If C is a subclass of View, then C(x) creates a proxy object around
+ the object x. The actual class of the proxy is not C in general,
+ but a *subclass* of C determined by the rules below. To avoid confusion
+ we call view class the class of the proxy (a subclass of C, so of View)
+ and object class the class of x.
+
+ Attributes and methods not found in the proxy are automatically read on x.
+ Other operations like setting attributes are performed on the proxy, as
+ determined by its view class. The object x is available from the proxy
+ as its __obj__ attribute.
+
+ The view class selection is determined by the __view__ tuples and the
+ optional __viewkey__ method. By default, the selected view class is the
+ most specific subclass of C whose __view__ mentions the class of x.
+ If no such subclass is found, the search proceeds with the parent
+ object classes. For example, C(True) will first look for a subclass
+ of C with __view__ = (..., bool, ...) and only if it doesn't find any
+ look for one with __view__ = (..., int, ...), and then ..., object,...
+ If everything fails the class C itself is considered to be the default.
+
+ Alternatively, the view class selection can be driven by another aspect
+ of the object x, instead of the class of x, by overriding __viewkey__.
+ See last example at the end of this module.
+ """
+
+ _viewcache = {}
+ __view__ = ()
+
+ def __new__(rootclass, obj, *args, **kwds):
+ self = object.__new__(rootclass)
+ self.__obj__ = obj
+ self.__rootclass__ = rootclass
+ key = self.__viewkey__()
+ try:
+ self.__class__ = self._viewcache[key]
+ except KeyError:
+ self.__class__ = self._selectsubclass(key)
+ return self
+
+ def __getattr__(self, attr):
+ # attributes not found in the normal hierarchy rooted on View
+ # are looked up in the object's real class
+ return getattr(self.__obj__, attr)
+
+ def __viewkey__(self):
+ return self.__obj__.__class__
+
+ def __matchkey__(self, key, subclasses):
+ if inspect.isclass(key):
+ keys = inspect.getmro(key)
+ else:
+ keys = [key]
+ for key in keys:
+ result = [C for C in subclasses if key in C.__view__]
+ if result:
+ return result
+ return []
+
+ def _selectsubclass(self, key):
+ subclasses = list(enumsubclasses(self.__rootclass__))
+ for C in subclasses:
+ if not isinstance(C.__view__, tuple):
+ C.__view__ = (C.__view__,)
+ choices = self.__matchkey__(key, subclasses)
+ if not choices:
+ return self.__rootclass__
+ elif len(choices) == 1:
+ return choices[0]
+ else:
+ # combine the multiple choices
+ return type('?', tuple(choices), {})
+
+ def __repr__(self):
+ return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+ for subcls in cls.__subclasses__():
+ for subsubclass in enumsubclasses(subcls):
+ yield subsubclass
+ yield cls
+
+
+class Interpretable(View):
+ """A parse tree node with a few extra methods."""
+ explanation = None
+
+ def is_builtin(self, frame):
+ return False
+
+ def eval(self, frame):
+ # fall-back for unknown expression nodes
+ try:
+ expr = ast.Expression(self.__obj__)
+ expr.filename = '<eval>'
+ self.__obj__.filename = '<eval>'
+ co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+ result = frame.eval(co)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ self.result = result
+ self.explanation = self.explanation or frame.repr(self.result)
+
+ def run(self, frame):
+ # fall-back for unknown statement nodes
+ try:
+ expr = ast.Module(None, ast.Stmt([self.__obj__]))
+ expr.filename = '<run>'
+ co = pycodegen.ModuleCodeGenerator(expr).getCode()
+ frame.exec_(co)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+ def nice_explanation(self):
+ return format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+ __view__ = ast.Name
+
+ def is_local(self, frame):
+ source = '%r in locals() is not globals()' % self.name
+ try:
+ return frame.is_true(frame.eval(source))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def is_global(self, frame):
+ source = '%r in globals()' % self.name
+ try:
+ return frame.is_true(frame.eval(source))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def is_builtin(self, frame):
+ source = '%r not in locals() and %r not in globals()' % (
+ self.name, self.name)
+ try:
+ return frame.is_true(frame.eval(source))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def eval(self, frame):
+ super(Name, self).eval(frame)
+ if not self.is_local(frame):
+ self.explanation = self.name
+
+class Compare(Interpretable):
+ __view__ = ast.Compare
+
+ def eval(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ for operation, expr2 in self.ops:
+ if hasattr(self, 'result'):
+ # shortcutting in chained expressions
+ if not frame.is_true(self.result):
+ break
+ expr2 = Interpretable(expr2)
+ expr2.eval(frame)
+ self.explanation = "%s %s %s" % (
+ expr.explanation, operation, expr2.explanation)
+ source = "__exprinfo_left %s __exprinfo_right" % operation
+ try:
+ self.result = frame.eval(source,
+ __exprinfo_left=expr.result,
+ __exprinfo_right=expr2.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ expr = expr2
+
+class And(Interpretable):
+ __view__ = ast.And
+
+ def eval(self, frame):
+ explanations = []
+ for expr in self.nodes:
+ expr = Interpretable(expr)
+ expr.eval(frame)
+ explanations.append(expr.explanation)
+ self.result = expr.result
+ if not frame.is_true(expr.result):
+ break
+ self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+ __view__ = ast.Or
+
+ def eval(self, frame):
+ explanations = []
+ for expr in self.nodes:
+ expr = Interpretable(expr)
+ expr.eval(frame)
+ explanations.append(expr.explanation)
+ self.result = expr.result
+ if frame.is_true(expr.result):
+ break
+ self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+ ast.Not : 'not __exprinfo_expr',
+ ast.Invert : '(~__exprinfo_expr)',
+ }.items():
+
+ class UnaryArith(Interpretable):
+ __view__ = astclass
+
+ def eval(self, frame, astpattern=astpattern):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ self.explanation = astpattern.replace('__exprinfo_expr',
+ expr.explanation)
+ try:
+ self.result = frame.eval(astpattern,
+ __exprinfo_expr=expr.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+ keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+ ast.Add : '(__exprinfo_left + __exprinfo_right)',
+ ast.Sub : '(__exprinfo_left - __exprinfo_right)',
+ ast.Mul : '(__exprinfo_left * __exprinfo_right)',
+ ast.Div : '(__exprinfo_left / __exprinfo_right)',
+ ast.Mod : '(__exprinfo_left % __exprinfo_right)',
+ ast.Power : '(__exprinfo_left ** __exprinfo_right)',
+ }.items():
+
+ class BinaryArith(Interpretable):
+ __view__ = astclass
+
+ def eval(self, frame, astpattern=astpattern):
+ left = Interpretable(self.left)
+ left.eval(frame)
+ right = Interpretable(self.right)
+ right.eval(frame)
+ self.explanation = (astpattern
+ .replace('__exprinfo_left', left .explanation)
+ .replace('__exprinfo_right', right.explanation))
+ try:
+ self.result = frame.eval(astpattern,
+ __exprinfo_left=left.result,
+ __exprinfo_right=right.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+ keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+ __view__ = ast.CallFunc
+
+ def is_bool(self, frame):
+ source = 'isinstance(__exprinfo_value, bool)'
+ try:
+ return frame.is_true(frame.eval(source,
+ __exprinfo_value=self.result))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def eval(self, frame):
+ node = Interpretable(self.node)
+ node.eval(frame)
+ explanations = []
+ vars = {'__exprinfo_fn': node.result}
+ source = '__exprinfo_fn('
+ for a in self.args:
+ if isinstance(a, ast.Keyword):
+ keyword = a.name
+ a = a.expr
+ else:
+ keyword = None
+ a = Interpretable(a)
+ a.eval(frame)
+ argname = '__exprinfo_%d' % len(vars)
+ vars[argname] = a.result
+ if keyword is None:
+ source += argname + ','
+ explanations.append(a.explanation)
+ else:
+ source += '%s=%s,' % (keyword, argname)
+ explanations.append('%s=%s' % (keyword, a.explanation))
+ if self.star_args:
+ star_args = Interpretable(self.star_args)
+ star_args.eval(frame)
+ argname = '__exprinfo_star'
+ vars[argname] = star_args.result
+ source += '*' + argname + ','
+ explanations.append('*' + star_args.explanation)
+ if self.dstar_args:
+ dstar_args = Interpretable(self.dstar_args)
+ dstar_args.eval(frame)
+ argname = '__exprinfo_kwds'
+ vars[argname] = dstar_args.result
+ source += '**' + argname + ','
+ explanations.append('**' + dstar_args.explanation)
+ self.explanation = "%s(%s)" % (
+ node.explanation, ', '.join(explanations))
+ if source.endswith(','):
+ source = source[:-1]
+ source += ')'
+ try:
+ self.result = frame.eval(source, **vars)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ if not node.is_builtin(frame) or not self.is_bool(frame):
+ r = frame.repr(self.result)
+ self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+ __view__ = ast.Getattr
+
+ def eval(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ source = '__exprinfo_expr.%s' % self.attrname
+ try:
+ self.result = frame.eval(source, __exprinfo_expr=expr.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+ # if the attribute comes from the instance, its value is interesting
+ source = ('hasattr(__exprinfo_expr, "__dict__") and '
+ '%r in __exprinfo_expr.__dict__' % self.attrname)
+ try:
+ from_instance = frame.is_true(
+ frame.eval(source, __exprinfo_expr=expr.result))
+ except passthroughex:
+ raise
+ except:
+ from_instance = True
+ if from_instance:
+ r = frame.repr(self.result)
+ self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+ __view__ = ast.Assert
+
+ def run(self, frame):
+ test = Interpretable(self.test)
+ test.eval(frame)
+ # print the result as 'assert <explanation>'
+ self.result = test.result
+ self.explanation = 'assert ' + test.explanation
+ if not frame.is_true(test.result):
+ try:
+ raise BuiltinAssertionError
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+class Assign(Interpretable):
+ __view__ = ast.Assign
+
+ def run(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ self.result = expr.result
+ self.explanation = '... = ' + expr.explanation
+ # fall-back-run the rest of the assignment
+ ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+ mod = ast.Module(None, ast.Stmt([ass]))
+ mod.filename = '<run>'
+ co = pycodegen.ModuleCodeGenerator(mod).getCode()
+ try:
+ frame.exec_(co, __exprinfo_expr=expr.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+class Discard(Interpretable):
+ __view__ = ast.Discard
+
+ def run(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ self.result = expr.result
+ self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+ __view__ = ast.Stmt
+
+ def run(self, frame):
+ for stmt in self.nodes:
+ stmt = Interpretable(stmt)
+ stmt.run(frame)
+
+
+def report_failure(e):
+ explanation = e.node.nice_explanation()
+ if explanation:
+ explanation = ", in: " + explanation
+ else:
+ explanation = ""
+ sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+ if frame is None:
+ frame = sys._getframe(1)
+ frame = py.code.Frame(frame)
+ expr = parse(s, 'eval')
+ assert isinstance(expr, ast.Expression)
+ node = Interpretable(expr.node)
+ try:
+ node.eval(frame)
+ except passthroughex:
+ raise
+ except Failure:
+ e = sys.exc_info()[1]
+ report_failure(e)
+ else:
+ if not frame.is_true(node.result):
+ sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+ module = Interpretable(parse(source, 'exec').node)
+ #print "got module", module
+ if isinstance(frame, py.std.types.FrameType):
+ frame = py.code.Frame(frame)
+ try:
+ module.run(frame)
+ except Failure:
+ e = sys.exc_info()[1]
+ return getfailure(e)
+ except passthroughex:
+ raise
+ except:
+ import traceback
+ traceback.print_exc()
+ if should_fail:
+ return ("(assertion failed, but when it was re-run for "
+ "printing intermediate values, it did not fail. Suggestions: "
+ "compute assert expression before the assert or use --nomagic)")
+ else:
+ return None
+
+def getmsg(excinfo):
+ if isinstance(excinfo, tuple):
+ excinfo = py.code.ExceptionInfo(excinfo)
+ #frame, line = gettbline(tb)
+ #frame = py.code.Frame(frame)
+ #return interpret(line, frame)
+
+ tb = excinfo.traceback[-1]
+ source = str(tb.statement).strip()
+ x = interpret(source, tb.frame, should_fail=True)
+ if not isinstance(x, str):
+ raise TypeError("interpret returned non-string %r" % (x,))
+ return x
+
+def getfailure(e):
+ explanation = e.node.nice_explanation()
+ if str(e.value):
+ lines = explanation.split('\n')
+ lines[0] += " << %s" % (e.value,)
+ explanation = '\n'.join(lines)
+ text = "%s: %s" % (e.exc.__name__, explanation)
+ if text.startswith('AssertionError: assert '):
+ text = text[16:]
+ return text
+
+def run(s, frame=None):
+ if frame is None:
+ frame = sys._getframe(1)
+ frame = py.code.Frame(frame)
+ module = Interpretable(parse(s, 'exec').node)
+ try:
+ module.run(frame)
+ except Failure:
+ e = sys.exc_info()[1]
+ report_failure(e)
+
+
+if __name__ == '__main__':
+ # example:
+ def f():
+ return 5
+ def g():
+ return 3
+ def h(x):
+ return 'never'
+ check("f() * g() == 5")
+ check("not f()")
+ check("not (f() and g() or 0)")
+ check("f() == g()")
+ i = 4
+ check("i == f()")
+ check("len(f()) == 0")
+ check("isinstance(2+3+4, float)")
+
+ run("x = i")
+ check("x == 5")
+
+ run("assert not f(), 'oops'")
+ run("a, b, c = 1, 2")
+ run("a, b, c = f()")
+
+ check("max([f(),g()]) == 4")
+ check("'hello'[g()] == 'h'")
+ run("'guk%d' % h(f())")
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/reinterpret.py
@@ -0,0 +1,48 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+class AssertionError(BuiltinAssertionError):
+ def __init__(self, *args):
+ BuiltinAssertionError.__init__(self, *args)
+ if args:
+ try:
+ self.msg = str(args[0])
+ except py.builtin._sysex:
+ raise
+ except:
+ self.msg = "<[broken __repr__] %s at %0xd>" %(
+ args[0].__class__, id(args[0]))
+ else:
+ f = py.code.Frame(sys._getframe(1))
+ try:
+ source = f.code.fullsource
+ if source is not None:
+ try:
+ source = source.getstatement(f.lineno, assertion=True)
+ except IndexError:
+ source = None
+ else:
+ source = str(source.deindent()).strip()
+ except py.error.ENOENT:
+ source = None
+ # this can also occur during reinterpretation, when the
+ # co_filename is set to "<run>".
+ if source:
+ self.msg = reinterpret(source, f, should_fail=True)
+ else:
+ self.msg = "<could not determine information>"
+ if not self.args:
+ self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+ AssertionError.__module__ = "builtins"
+ reinterpret_old = "old reinterpretation not available for py3"
+else:
+ from _pytest.assertion.oldinterpret import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+ from _pytest.assertion.newinterpret import interpret as reinterpret
+else:
+ reinterpret = reinterpret_old
+
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/rewrite.py
@@ -0,0 +1,340 @@
+"""Rewrite assertion AST to produce nice error messages"""
+
+import ast
+import collections
+import itertools
+import sys
+
+import py
+from _pytest.assertion import util
+
+
+def rewrite_asserts(mod):
+ """Rewrite the assert statements in mod."""
+ AssertionRewriter().run(mod)
+
+
+_saferepr = py.io.saferepr
+from _pytest.assertion.util import format_explanation as _format_explanation
+
+def _format_boolop(operands, explanations, is_or):
+ show_explanations = []
+ for operand, expl in zip(operands, explanations):
+ show_explanations.append(expl)
+ if operand == is_or:
+ break
+ return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+
+def _call_reprcompare(ops, results, expls, each_obj):
+ for i, res, expl in zip(range(len(ops)), results, expls):
+ try:
+ done = not res
+ except Exception:
+ done = True
+ if done:
+ break
+ if util._reprcompare is not None:
+ custom = util._reprcompare(ops[i], each_obj[i], each_obj[i + 1])
+ if custom is not None:
+ return custom
+ return expl
+
+
+unary_map = {
+ ast.Not : "not %s",
+ ast.Invert : "~%s",
+ ast.USub : "-%s",
+ ast.UAdd : "+%s"
+}
+
+binop_map = {
+ ast.BitOr : "|",
+ ast.BitXor : "^",
+ ast.BitAnd : "&",
+ ast.LShift : "<<",
+ ast.RShift : ">>",
+ ast.Add : "+",
+ ast.Sub : "-",
+ ast.Mult : "*",
+ ast.Div : "/",
+ ast.FloorDiv : "//",
+ ast.Mod : "%",
+ ast.Eq : "==",
+ ast.NotEq : "!=",
+ ast.Lt : "<",
+ ast.LtE : "<=",
+ ast.Gt : ">",
+ ast.GtE : ">=",
+ ast.Pow : "**",
+ ast.Is : "is",
+ ast.IsNot : "is not",
+ ast.In : "in",
+ ast.NotIn : "not in"
+}
+
+
+def set_location(node, lineno, col_offset):
+ """Set node location information recursively."""
+ def _fix(node, lineno, col_offset):
+ if "lineno" in node._attributes:
+ node.lineno = lineno
+ if "col_offset" in node._attributes:
+ node.col_offset = col_offset
+ for child in ast.iter_child_nodes(node):
+ _fix(child, lineno, col_offset)
+ _fix(node, lineno, col_offset)
+ return node
+
+
+class AssertionRewriter(ast.NodeVisitor):
+
+ def run(self, mod):
+ """Find all assert statements in *mod* and rewrite them."""
+ if not mod.body:
+ # Nothing to do.
+ return
+ # Insert some special imports at the top of the module but after any
+ # docstrings and __future__ imports.
+ aliases = [ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
+ ast.alias("_pytest.assertion.rewrite", "@pytest_ar")]
+ expect_docstring = True
+ pos = 0
+ lineno = 0
+ for item in mod.body:
+ if (expect_docstring and isinstance(item, ast.Expr) and
+ isinstance(item.value, ast.Str)):
+ doc = item.value.s
+ if "PYTEST_DONT_REWRITE" in doc:
+ # The module has disabled assertion rewriting.
+ return
+ lineno += len(doc) - 1
+ expect_docstring = False
+ elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
+ item.identifier != "__future__"):
+ lineno = item.lineno
+ break
+ pos += 1
+ imports = [ast.Import([alias], lineno=lineno, col_offset=0)
+ for alias in aliases]
+ mod.body[pos:pos] = imports
+ # Collect asserts.
+ nodes = collections.deque([mod])
+ while nodes:
+ node = nodes.popleft()
+ for name, field in ast.iter_fields(node):
+ if isinstance(field, list):
+ new = []
+ for i, child in enumerate(field):
+ if isinstance(child, ast.Assert):
+ # Transform assert.
+ new.extend(self.visit(child))
+ else:
+ new.append(child)
+ if isinstance(child, ast.AST):
+ nodes.append(child)
+ setattr(node, name, new)
+ elif (isinstance(field, ast.AST) and
+ # Don't recurse into expressions as they can't contain
+ # asserts.
+ not isinstance(field, ast.expr)):
+ nodes.append(field)
+
+ def variable(self):
+ """Get a new variable."""
+ # Use a character invalid in python identifiers to avoid clashing.
+ name = "@py_assert" + str(next(self.variable_counter))
+ self.variables.add(name)
+ return name
+
+ def assign(self, expr):
+ """Give *expr* a name."""
+ name = self.variable()
+ self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr))
+ return ast.Name(name, ast.Load())
+
+ def display(self, expr):
+ """Call py.io.saferepr on the expression."""
+ return self.helper("saferepr", expr)
+
+ def helper(self, name, *args):
+ """Call a helper in this module."""
+ py_name = ast.Name("@pytest_ar", ast.Load())
+ attr = ast.Attribute(py_name, "_" + name, ast.Load())
+ return ast.Call(attr, list(args), [], None, None)
+
+ def builtin(self, name):
+ """Return the builtin called *name*."""
+ builtin_name = ast.Name("@py_builtins", ast.Load())
+ return ast.Attribute(builtin_name, name, ast.Load())
+
+ def explanation_param(self, expr):
+ specifier = "py" + str(next(self.variable_counter))
+ self.explanation_specifiers[specifier] = expr
+ return "%(" + specifier + ")s"
+
+ def push_format_context(self):
+ self.explanation_specifiers = {}
+ self.stack.append(self.explanation_specifiers)
+
+ def pop_format_context(self, expl_expr):
+ current = self.stack.pop()
+ if self.stack:
+ self.explanation_specifiers = self.stack[-1]
+ keys = [ast.Str(key) for key in current.keys()]
+ format_dict = ast.Dict(keys, list(current.values()))
+ form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
+ name = "@py_format" + str(next(self.variable_counter))
+ self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
+ return ast.Name(name, ast.Load())
+
+ def generic_visit(self, node):
+ """Handle expressions we don't have custom code for."""
+ assert isinstance(node, ast.expr)
+ res = self.assign(node)
+ return res, self.explanation_param(self.display(res))
+
+ def visit_Assert(self, assert_):
+ if assert_.msg:
+ # There's already a message. Don't mess with it.
+ return [assert_]
+ self.statements = []
+ self.variables = set()
+ self.variable_counter = itertools.count()
+ self.stack = []
+ self.on_failure = []
+ self.push_format_context()
+ # Rewrite assert into a bunch of statements.
+ top_condition, explanation = self.visit(assert_.test)
+ # Create failure message.
+ body = self.on_failure
+ negation = ast.UnaryOp(ast.Not(), top_condition)
+ self.statements.append(ast.If(negation, body, []))
+ explanation = "assert " + explanation
+ template = ast.Str(explanation)
+ msg = self.pop_format_context(template)
+ fmt = self.helper("format_explanation", msg)
+ err_name = ast.Name("AssertionError", ast.Load())
+ exc = ast.Call(err_name, [fmt], [], None, None)
+ if sys.version_info[0] >= 3:
+ raise_ = ast.Raise(exc, None)
+ else:
+ raise_ = ast.Raise(exc, None, None)
+ body.append(raise_)
+ # Delete temporary variables.
+ names = [ast.Name(name, ast.Del()) for name in self.variables]
+ if names:
+ delete = ast.Delete(names)
+ self.statements.append(delete)
+ # Fix line numbers.
+ for stmt in self.statements:
+ set_location(stmt, assert_.lineno, assert_.col_offset)
+ return self.statements
+
+ def visit_Name(self, name):
+ # Check if the name is local or not.
+ locs = ast.Call(self.builtin("locals"), [], [], None, None)
+ globs = ast.Call(self.builtin("globals"), [], [], None, None)
+ ops = [ast.In(), ast.IsNot()]
+ test = ast.Compare(ast.Str(name.id), ops, [locs, globs])
+ expr = ast.IfExp(test, self.display(name), ast.Str(name.id))
+ return name, self.explanation_param(expr)
+
+ def visit_BoolOp(self, boolop):
+ operands = []
+ explanations = []
+ self.push_format_context()
+ for operand in boolop.values:
+ res, explanation = self.visit(operand)
+ operands.append(res)
+ explanations.append(explanation)
+ expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
+ is_or = ast.Num(isinstance(boolop.op, ast.Or))
+ expl_template = self.helper("format_boolop",
+ ast.Tuple(operands, ast.Load()), expls,
+ is_or)
+ expl = self.pop_format_context(expl_template)
+ res = self.assign(ast.BoolOp(boolop.op, operands))
+ return res, self.explanation_param(expl)
+
+ def visit_UnaryOp(self, unary):
+ pattern = unary_map[unary.op.__class__]
+ operand_res, operand_expl = self.visit(unary.operand)
+ res = self.assign(ast.UnaryOp(unary.op, operand_res))
+ return res, pattern % (operand_expl,)
+
+ def visit_BinOp(self, binop):
+ symbol = binop_map[binop.op.__class__]
+ left_expr, left_expl = self.visit(binop.left)
+ right_expr, right_expl = self.visit(binop.right)
+ explanation = "(%s %s %s)" % (left_expl, symbol, right_expl)
+ res = self.assign(ast.BinOp(left_expr, binop.op, right_expr))
+ return res, explanation
+
+ def visit_Call(self, call):
+ new_func, func_expl = self.visit(call.func)
+ arg_expls = []
+ new_args = []
+ new_kwargs = []
+ new_star = new_kwarg = None
+ for arg in call.args:
+ res, expl = self.visit(arg)
+ new_args.append(res)
+ arg_expls.append(expl)
+ for keyword in call.keywords:
+ res, expl = self.visit(keyword.value)
+ new_kwargs.append(ast.keyword(keyword.arg, res))
+ arg_expls.append(keyword.arg + "=" + expl)
+ if call.starargs:
+ new_star, expl = self.visit(call.starargs)
+ arg_expls.append("*" + expl)
+ if call.kwargs:
+ new_kwarg, expl = self.visit(call.kwarg)
+ arg_expls.append("**" + expl)
+ expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
+ new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
+ res = self.assign(new_call)
+ res_expl = self.explanation_param(self.display(res))
+ outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
+ return res, outer_expl
+
+ def visit_Attribute(self, attr):
+ if not isinstance(attr.ctx, ast.Load):
+ return self.generic_visit(attr)
+ value, value_expl = self.visit(attr.value)
+ res = self.assign(ast.Attribute(value, attr.attr, ast.Load()))
+ res_expl = self.explanation_param(self.display(res))
+ pat = "%s\n{%s = %s.%s\n}"
+ expl = pat % (res_expl, res_expl, value_expl, attr.attr)
+ return res, expl
+
+ def visit_Compare(self, comp):
+ self.push_format_context()
+ left_res, left_expl = self.visit(comp.left)
+ res_variables = [self.variable() for i in range(len(comp.ops))]
+ load_names = [ast.Name(v, ast.Load()) for v in res_variables]
+ store_names = [ast.Name(v, ast.Store()) for v in res_variables]
+ it = zip(range(len(comp.ops)), comp.ops, comp.comparators)
+ expls = []
+ syms = []
+ results = [left_res]
+ for i, op, next_operand in it:
+ next_res, next_expl = self.visit(next_operand)
+ results.append(next_res)
+ sym = binop_map[op.__class__]
+ syms.append(ast.Str(sym))
+ expl = "%s %s %s" % (left_expl, sym, next_expl)
+ expls.append(ast.Str(expl))
+ res_expr = ast.Compare(left_res, [op], [next_res])
+ self.statements.append(ast.Assign([store_names[i]], res_expr))
+ left_res, left_expl = next_res, next_expl
+ # Use py.code._reprcompare if that's available.
+ expl_call = self.helper("call_reprcompare",
+ ast.Tuple(syms, ast.Load()),
+ ast.Tuple(load_names, ast.Load()),
+ ast.Tuple(expls, ast.Load()),
+ ast.Tuple(results, ast.Load()))
+ if len(comp.ops) > 1:
+ res = ast.BoolOp(ast.And(), load_names)
+ else:
+ res = load_names[0]
+ return res, self.explanation_param(self.pop_format_context(expl_call))
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/util.py
@@ -0,0 +1,213 @@
+"""Utilities for assertion debugging"""
+
+import py
+
+
+# The _reprcompare attribute on the util module is used by the new assertion
+# interpretation code and assertion rewriter to detect this plugin was
+# loaded and in turn call the hooks defined here as part of the
+# DebugInterpreter.
+_reprcompare = None
+
+def format_explanation(explanation):
+ """This formats an explanation
+
+ Normally all embedded newlines are escaped, however there are
+ three exceptions: \n{, \n} and \n~. The first two are intended
+ cover nested explanations, see function and attribute explanations
+ for examples (.visit_Call(), visit_Attribute()). The last one is
+ for when one explanation needs to span multiple lines, e.g. when
+ displaying diffs.
+ """
+ # simplify 'assert False where False = ...'
+ where = 0
+ while True:
+ start = where = explanation.find("False\n{False = ", where)
+ if where == -1:
+ break
+ level = 0
+ for i, c in enumerate(explanation[start:]):
+ if c == "{":
+ level += 1
+ elif c == "}":
+ level -= 1
+ if not level:
+ break
+ else:
+ raise AssertionError("unbalanced braces: %r" % (explanation,))
+ end = start + i
+ where = end
+ if explanation[end - 1] == '\n':
+ explanation = (explanation[:start] + explanation[start+15:end-1] +
+ explanation[end+1:])
+ where -= 17
+ raw_lines = (explanation or '').split('\n')
+ # escape newlines not followed by {, } and ~
+ lines = [raw_lines[0]]
+ for l in raw_lines[1:]:
+ if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+ lines.append(l)
+ else:
+ lines[-1] += '\\n' + l
+
+ result = lines[:1]
+ stack = [0]
+ stackcnt = [0]
+ for line in lines[1:]:
+ if line.startswith('{'):
+ if stackcnt[-1]:
+ s = 'and '
+ else:
+ s = 'where '
+ stack.append(len(result))
+ stackcnt[-1] += 1
+ stackcnt.append(0)
+ result.append(' +' + ' '*(len(stack)-1) + s + line[1:])
+ elif line.startswith('}'):
+ assert line.startswith('}')
+ stack.pop()
+ stackcnt.pop()
+ result[stack[-1]] += line[1:]
+ else:
+ assert line.startswith('~')
+ result.append(' '*len(stack) + line[1:])
+ assert len(stack) == 1
+ return '\n'.join(result)
+
+
+# Provide basestring in python3
+try:
+ basestring = basestring
+except NameError:
+ basestring = str
+
+
+def assertrepr_compare(op, left, right):
+ """return specialised explanations for some operators/operands"""
+ width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
+ left_repr = py.io.saferepr(left, maxsize=int(width/2))
+ right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
+ summary = '%s %s %s' % (left_repr, op, right_repr)
+
+ issequence = lambda x: isinstance(x, (list, tuple))
+ istext = lambda x: isinstance(x, basestring)
+ isdict = lambda x: isinstance(x, dict)
+ isset = lambda x: isinstance(x, set)
+
+ explanation = None
+ try:
+ if op == '==':
+ if istext(left) and istext(right):
+ explanation = _diff_text(left, right)
+ elif issequence(left) and issequence(right):
+ explanation = _compare_eq_sequence(left, right)
+ elif isset(left) and isset(right):
+ explanation = _compare_eq_set(left, right)
+ elif isdict(left) and isdict(right):
+ explanation = _diff_text(py.std.pprint.pformat(left),
+ py.std.pprint.pformat(right))
+ elif op == 'not in':
+ if istext(left) and istext(right):
+ explanation = _notin_text(left, right)
+ except py.builtin._sysex:
+ raise
+ except:
+ excinfo = py.code.ExceptionInfo()
+ explanation = ['(pytest_assertion plugin: representation of '
+ 'details failed. Probably an object has a faulty __repr__.)',
+ str(excinfo)
+ ]
+
+
+ if not explanation:
+ return None
+
+ # Don't include pageloads of data, should be configurable
+ if len(''.join(explanation)) > 80*8:
+ explanation = ['Detailed information too verbose, truncated']
+
+ return [summary] + explanation
+
+
+def _diff_text(left, right):
+ """Return the explanation for the diff between text
+
+ This will skip leading and trailing characters which are
+ identical to keep the diff minimal.
+ """
+ explanation = []
+ i = 0 # just in case left or right has zero length
+ for i in range(min(len(left), len(right))):
+ if left[i] != right[i]:
+ break
+ if i > 42:
+ i -= 10 # Provide some context
+ explanation = ['Skipping %s identical '
+ 'leading characters in diff' % i]
+ left = left[i:]
+ right = right[i:]
+ if len(left) == len(right):
+ for i in range(len(left)):
+ if left[-i] != right[-i]:
+ break
+ if i > 42:
+ i -= 10 # Provide some context
+ explanation += ['Skipping %s identical '
+ 'trailing characters in diff' % i]
+ left = left[:-i]
+ right = right[:-i]
+ explanation += [line.strip('\n')
+ for line in py.std.difflib.ndiff(left.splitlines(),
+ right.splitlines())]
+ return explanation
+
+
+def _compare_eq_sequence(left, right):
+ explanation = []
+ for i in range(min(len(left), len(right))):
+ if left[i] != right[i]:
+ explanation += ['At index %s diff: %r != %r' %
+ (i, left[i], right[i])]
+ break
+ if len(left) > len(right):
+ explanation += ['Left contains more items, '
+ 'first extra item: %s' % py.io.saferepr(left[len(right)],)]
+ elif len(left) < len(right):
+ explanation += ['Right contains more items, '
+ 'first extra item: %s' % py.io.saferepr(right[len(left)],)]
+ return explanation # + _diff_text(py.std.pprint.pformat(left),
+ # py.std.pprint.pformat(right))
+
+
+def _compare_eq_set(left, right):
+ explanation = []
+ diff_left = left - right
+ diff_right = right - left
+ if diff_left:
+ explanation.append('Extra items in the left set:')
+ for item in diff_left:
+ explanation.append(py.io.saferepr(item))
+ if diff_right:
+ explanation.append('Extra items in the right set:')
+ for item in diff_right:
+ explanation.append(py.io.saferepr(item))
+ return explanation
+
+
+def _notin_text(term, text):
+ index = text.find(term)
+ head = text[:index]
+ tail = text[index+len(term):]
+ correct_text = head + tail
+ diff = _diff_text(correct_text, text)
+ newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
+ for line in diff:
+ if line.startswith('Skipping'):
+ continue
+ if line.startswith('- '):
+ continue
+ if line.startswith('+ '):
+ newdiff.append(' ' + line[2:])
+ else:
+ newdiff.append(line)
+ return newdiff
diff --git a/_pytest/doctest.py b/_pytest/doctest.py
--- a/_pytest/doctest.py
+++ b/_pytest/doctest.py
@@ -59,7 +59,7 @@
inner_excinfo = py.code.ExceptionInfo(excinfo.value.exc_info)
lines += ["UNEXPECTED EXCEPTION: %s" %
repr(inner_excinfo.value)]
-
+ lines += py.std.traceback.format_exception(*excinfo.value.exc_info)
return ReprFailDoctest(reprlocation, lines)
else:
return super(DoctestItem, self).repr_failure(excinfo)
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -16,9 +16,6 @@
group.addoption('--traceconfig',
action="store_true", dest="traceconfig", default=False,
help="trace considerations of conftest.py files."),
- group._addoption('--nomagic',
- action="store_true", dest="nomagic", default=False,
- help="don't reinterpret asserts, no traceback cutting. ")
group.addoption('--debug',
action="store_true", dest="debug", default=False,
help="generate and show internal debugging information.")
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -65,7 +65,8 @@
class LogXML(object):
def __init__(self, logfile, prefix):
- self.logfile = logfile
+ logfile = os.path.expanduser(os.path.expandvars(logfile))
+ self.logfile = os.path.normpath(logfile)
self.prefix = prefix
self.test_logs = []
self.passed = self.skipped = 0
@@ -76,7 +77,7 @@
names = report.nodeid.split("::")
names[0] = names[0].replace("/", '.')
names = tuple(names)
- d = {'time': self._durations.pop(names, "0")}
+ d = {'time': self._durations.pop(report.nodeid, "0")}
names = [x.replace(".py", "") for x in names if x != "()"]
classnames = names[:-1]
if self.prefix:
@@ -170,12 +171,11 @@
self.append_skipped(report)
def pytest_runtest_call(self, item, __multicall__):
- names = tuple(item.listnames())
start = time.time()
try:
return __multicall__.execute()
finally:
- self._durations[names] = time.time() - start
+ self._durations[item.nodeid] = time.time() - start
def pytest_collectreport(self, report):
if not report.passed:
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -46,23 +46,25 @@
def pytest_namespace():
- return dict(collect=dict(Item=Item, Collector=Collector, File=File))
+ collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
+ return dict(collect=collect)
def pytest_configure(config):
py.test.config = config # compatibiltiy
if config.option.exitfirst:
config.option.maxfail = 1
-def pytest_cmdline_main(config):
- """ default command line protocol for initialization, session,
- running tests and reporting. """
+def wrap_session(config, doit):
+ """Skeleton command line program"""
session = Session(config)
session.exitstatus = EXIT_OK
+ initstate = 0
try:
config.pluginmanager.do_configure(config)
+ initstate = 1
config.hook.pytest_sessionstart(session=session)
- config.hook.pytest_collection(session=session)
- config.hook.pytest_runtestloop(session=session)
+ initstate = 2
+ doit(config, session)
except pytest.UsageError:
raise
except KeyboardInterrupt:
@@ -77,18 +79,24 @@
sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
if not session.exitstatus and session._testsfailed:
session.exitstatus = EXIT_TESTSFAILED
- config.hook.pytest_sessionfinish(session=session,
- exitstatus=session.exitstatus)
- config.pluginmanager.do_unconfigure(config)
+ if initstate >= 2:
+ config.hook.pytest_sessionfinish(session=session,
+ exitstatus=session.exitstatus)
+ if initstate >= 1:
+ config.pluginmanager.do_unconfigure(config)
return session.exitstatus
+def pytest_cmdline_main(config):
+ return wrap_session(config, _main)
+
+def _main(config, session):
+ """ default command line protocol for initialization, session,
+ running tests and reporting. """
+ config.hook.pytest_collection(session=session)
+ config.hook.pytest_runtestloop(session=session)
+
def pytest_collection(session):
- session.perform_collect()
- hook = session.config.hook
- hook.pytest_collection_modifyitems(session=session,
- config=session.config, items=session.items)
- hook.pytest_collection_finish(session=session)
- return True
+ return session.perform_collect()
def pytest_runtestloop(session):
if session.config.option.collectonly:
@@ -374,6 +382,16 @@
return HookProxy(fspath, self.config)
def perform_collect(self, args=None, genitems=True):
+ hook = self.config.hook
+ try:
+ items = self._perform_collect(args, genitems)
+ hook.pytest_collection_modifyitems(session=self,
+ config=self.config, items=items)
+ finally:
+ hook.pytest_collection_finish(session=self)
+ return items
+
+ def _perform_collect(self, args, genitems):
if args is None:
args = self.config.args
self.trace("perform_collect", self, args)
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -153,7 +153,7 @@
def __repr__(self):
return "<MarkInfo %r args=%r kwargs=%r>" % (
- self._name, self.args, self.kwargs)
+ self.name, self.args, self.kwargs)
def pytest_itemcollected(item):
if not isinstance(item, pytest.Function):
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -6,7 +6,7 @@
import inspect
import time
from fnmatch import fnmatch
-from _pytest.main import Session
+from _pytest.main import Session, EXIT_OK
from py.builtin import print_
from _pytest.core import HookRelay
@@ -292,13 +292,19 @@
assert '::' not in str(arg)
p = py.path.local(arg)
x = session.fspath.bestrelpath(p)
- return session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionstart(session=session)
+ res = session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+ return res
def getpathnode(self, path):
- config = self.parseconfig(path)
+ config = self.parseconfigure(path)
session = Session(config)
x = session.fspath.bestrelpath(path)
- return session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionstart(session=session)
+ res = session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+ return res
def genitems(self, colitems):
session = colitems[0].session
@@ -312,7 +318,9 @@
config = self.parseconfigure(*args)
rec = self.getreportrecorder(config)
session = Session(config)
+ config.hook.pytest_sessionstart(session=session)
session.perform_collect()
+ config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
return session.items, rec
def runitem(self, source):
@@ -382,6 +390,8 @@
c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
keep=0, rootdir=self.tmpdir, lock_timeout=None)
c.parse(args)
+ c.pluginmanager.do_configure(c)
+ self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
return c
finally:
py.test.config = oldconfig
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -226,8 +226,13 @@
def _importtestmodule(self):
# we assume we are only called once per module
+ from _pytest import assertion
+ assertion.before_module_import(self)
try:
- mod = self.fspath.pyimport(ensuresyspath=True)
+ try:
+ mod = self.fspath.pyimport(ensuresyspath=True)
+ finally:
+ assertion.after_module_import(self)
except SyntaxError:
excinfo = py.code.ExceptionInfo()
raise self.CollectError(excinfo.getrepr(style="short"))
@@ -374,7 +379,7 @@
# test generators are seen as collectors but they also
# invoke setup/teardown on popular request
# (induced by the common "test_*" naming shared with normal tests)
- self.config._setupstate.prepare(self)
+ self.session._setupstate.prepare(self)
# see FunctionMixin.setup and test_setupstate_is_preserved_134
self._preservedparent = self.parent.obj
l = []
@@ -721,7 +726,7 @@
def _addfinalizer(self, finalizer, scope):
colitem = self._getscopeitem(scope)
- self.config._setupstate.addfinalizer(
+ self._pyfuncitem.session._setupstate.addfinalizer(
finalizer=finalizer, colitem=colitem)
def __repr__(self):
@@ -742,8 +747,10 @@
raise self.LookupError(msg)
def showfuncargs(config):
- from _pytest.main import Session
- session = Session(config)
+ from _pytest.main import wrap_session
+ return wrap_session(config, _showfuncargs_main)
+
+def _showfuncargs_main(config, session):
session.perform_collect()
if session.items:
plugins = session.items[0].getplugins()
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -14,17 +14,15 @@
#
# pytest plugin hooks
-# XXX move to pytest_sessionstart and fix py.test owns tests
-def pytest_configure(config):
- config._setupstate = SetupState()
+def pytest_sessionstart(session):
+ session._setupstate = SetupState()
def pytest_sessionfinish(session, exitstatus):
- if hasattr(session.config, '_setupstate'):
- hook = session.config.hook
- rep = hook.pytest__teardown_final(session=session)
- if rep:
- hook.pytest__teardown_final_logerror(session=session, report=rep)
- session.exitstatus = 1
+ hook = session.config.hook
+ rep = hook.pytest__teardown_final(session=session)
+ if rep:
+ hook.pytest__teardown_final_logerror(session=session, report=rep)
+ session.exitstatus = 1
class NodeInfo:
def __init__(self, location):
@@ -46,16 +44,16 @@
return reports
def pytest_runtest_setup(item):
- item.config._setupstate.prepare(item)
+ item.session._setupstate.prepare(item)
def pytest_runtest_call(item):
item.runtest()
def pytest_runtest_teardown(item):
- item.config._setupstate.teardown_exact(item)
+ item.session._setupstate.teardown_exact(item)
def pytest__teardown_final(session):
- call = CallInfo(session.config._setupstate.teardown_all, when="teardown")
+ call = CallInfo(session._setupstate.teardown_all, when="teardown")
if call.excinfo:
ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
call.excinfo.traceback = ntraceback.filter()
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
--- a/ctypes_configure/configure.py
+++ b/ctypes_configure/configure.py
@@ -559,7 +559,9 @@
C_HEADER = """
#include <stdio.h>
#include <stddef.h> /* for offsetof() */
-#include <stdint.h> /* FreeBSD: for uint64_t */
+#ifndef _WIN32
+# include <stdint.h> /* FreeBSD: for uint64_t */
+#endif
void dump(char* key, int value) {
printf("%s: %d\\n", key, value);
diff --git a/ctypes_configure/stdoutcapture.py b/ctypes_configure/stdoutcapture.py
--- a/ctypes_configure/stdoutcapture.py
+++ b/ctypes_configure/stdoutcapture.py
@@ -15,6 +15,15 @@
not hasattr(os, 'fdopen')):
self.dummy = 1
else:
+ try:
+ self.tmpout = os.tmpfile()
+ if mixed_out_err:
+ self.tmperr = self.tmpout
+ else:
+ self.tmperr = os.tmpfile()
+ except OSError: # bah? on at least one Windows box
+ self.dummy = 1
+ return
self.dummy = 0
# make new stdout/stderr files if needed
self.localoutfd = os.dup(1)
@@ -29,11 +38,6 @@
sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
else:
self.saved_stderr = None
- self.tmpout = os.tmpfile()
- if mixed_out_err:
- self.tmperr = self.tmpout
- else:
- self.tmperr = os.tmpfile()
os.dup2(self.tmpout.fileno(), 1)
os.dup2(self.tmperr.fileno(), 2)
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
return 'unknown'
-def dot2plain(content, contenttype, use_codespeak=False):
- if contenttype == 'plain':
- # already a .plain file
- return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+ if contenttype != 'neato':
+ cmdline = 'dot -Tplain'
+ else:
+ cmdline = 'neato -Tplain'
+ #print >> sys.stderr, '* running:', cmdline
+ close_fds = sys.platform != 'win32'
+ p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ (child_in, child_out) = (p.stdin, p.stdout)
+ try:
+ import thread
+ except ImportError:
+ bkgndwrite(child_in, content)
+ else:
+ thread.start_new_thread(bkgndwrite, (child_in, content))
+ plaincontent = child_out.read()
+ child_out.close()
+ if not plaincontent: # 'dot' is likely not installed
+ raise PlainParseError("no result from running 'dot'")
+ return plaincontent
- if not use_codespeak:
- if contenttype != 'neato':
- cmdline = 'dot -Tplain'
- else:
- cmdline = 'neato -Tplain'
- #print >> sys.stderr, '* running:', cmdline
- close_fds = sys.platform != 'win32'
- p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
- (child_in, child_out) = (p.stdin, p.stdout)
- try:
- import thread
- except ImportError:
- bkgndwrite(child_in, content)
- else:
- thread.start_new_thread(bkgndwrite, (child_in, content))
- plaincontent = child_out.read()
- child_out.close()
- if not plaincontent: # 'dot' is likely not installed
- raise PlainParseError("no result from running 'dot'")
- else:
- import urllib
- request = urllib.urlencode({'dot': content})
- url = 'http://codespeak.net/pypy/convertdot.cgi'
- print >> sys.stderr, '* posting:', url
- g = urllib.urlopen(url, data=request)
- result = []
- while True:
- data = g.read(16384)
- if not data:
- break
- result.append(data)
- g.close()
- plaincontent = ''.join(result)
- # very simple-minded way to give a somewhat better error message
- if plaincontent.startswith('<body'):
- raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+ import urllib
+ request = urllib.urlencode({'dot': content})
+ url = 'http://codespeak.net/pypy/convertdot.cgi'
+ print >> sys.stderr, '* posting:', url
+ g = urllib.urlopen(url, data=request)
+ result = []
+ while True:
+ data = g.read(16384)
+ if not data:
+ break
+ result.append(data)
+ g.close()
+ plaincontent = ''.join(result)
+ # very simple-minded way to give a somewhat better error message
+ if plaincontent.startswith('<body'):
+ raise Exception("the dot on codespeak has very likely crashed")
return plaincontent
def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
def parse_dot(graph_id, content, links={}, fixedfont=False):
contenttype = guess_type(content)
- try:
- plaincontent = dot2plain(content, contenttype, use_codespeak=False)
- return list(parse_plain(graph_id, plaincontent, links, fixedfont))
- except PlainParseError:
- # failed, retry via codespeak
- plaincontent = dot2plain(content, contenttype, use_codespeak=True)
- return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+ if contenttype == 'plain':
+ plaincontent = content
+ else:
+ try:
+ plaincontent = dot2plain_graphviz(content, contenttype)
+ except PlainParseError, e:
+ print e
+ # failed, retry via codespeak
+ plaincontent = dot2plain_codespeak(content, contenttype)
+ return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -62,7 +62,6 @@
from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
from _ssl import SSLError
from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
from _ssl import RAND_status, RAND_egd, RAND_add
from _ssl import \
SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
SSL_ERROR_WANT_CONNECT, \
SSL_ERROR_EOF, \
SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+ PROTOCOL_TLSv1: "TLSv1",
+ PROTOCOL_SSLv23: "SSLv23",
+ PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+ from _ssl import PROTOCOL_SSLv2
+except ImportError:
+ pass
+else:
+ _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
from socket import socket, _fileobject, _delegate_methods, error as socket_error
from socket import getnameinfo as _getnameinfo
@@ -408,16 +419,7 @@
return DER_cert_to_PEM_cert(dercert)
def get_protocol_name(protocol_code):
- if protocol_code == PROTOCOL_TLSv1:
- return "TLSv1"
- elif protocol_code == PROTOCOL_SSLv23:
- return "SSLv23"
- elif protocol_code == PROTOCOL_SSLv2:
- return "SSLv2"
- elif protocol_code == PROTOCOL_SSLv3:
- return "SSLv3"
- else:
- return "<unknown>"
+ return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
# a replacement for the old socket.ssl function
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
--- a/lib-python/2.7/test/test_os.py
+++ b/lib-python/2.7/test/test_os.py
@@ -74,7 +74,8 @@
self.assertFalse(os.path.exists(name),
"file already exists for temporary file")
# make sure we can create the file
- open(name, "w")
+ f = open(name, "w")
+ f.close()
self.files.append(name)
def test_tempnam(self):
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
def skip_if_broken_ubuntu_ssl(func):
- # We need to access the lower-level wrapper in order to create an
- # implicit SSL context without trying to connect or listen.
- try:
- import _ssl
- except ImportError:
- # The returned function won't get executed, just ignore the error
- pass
- @functools.wraps(func)
- def f(*args, **kwargs):
+ if hasattr(ssl, 'PROTOCOL_SSLv2'):
+ # We need to access the lower-level wrapper in order to create an
+ # implicit SSL context without trying to connect or listen.
try:
- s = socket.socket(socket.AF_INET)
- _ssl.sslwrap(s._sock, 0, None, None,
- ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
- except ssl.SSLError as e:
- if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
- platform.linux_distribution() == ('debian', 'squeeze/sid', '')
- and 'Invalid SSL protocol variant specified' in str(e)):
- raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
- return func(*args, **kwargs)
- return f
+ import _ssl
+ except ImportError:
+ # The returned function won't get executed, just ignore the error
+ pass
+ @functools.wraps(func)
+ def f(*args, **kwargs):
+ try:
+ s = socket.socket(socket.AF_INET)
+ _ssl.sslwrap(s._sock, 0, None, None,
+ ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+ except ssl.SSLError as e:
+ if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+ platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+ and 'Invalid SSL protocol variant specified' in str(e)):
+ raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+ return func(*args, **kwargs)
+ return f
+ else:
+ return func
class BasicSocketTests(unittest.TestCase):
def test_constants(self):
- ssl.PROTOCOL_SSLv2
+ #ssl.PROTOCOL_SSLv2
ssl.PROTOCOL_SSLv23
ssl.PROTOCOL_SSLv3
ssl.PROTOCOL_TLSv1
@@ -964,7 +967,8 @@
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
- try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+ if hasattr(ssl, 'PROTOCOL_SSLv2'):
+ try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
@@ -976,7 +980,8 @@
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
- try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+ if hasattr(ssl, 'PROTOCOL_SSLv2'):
+ try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
diff --git a/lib-python/TODO b/lib-python/TODO
deleted file mode 100644
--- a/lib-python/TODO
+++ /dev/null
@@ -1,100 +0,0 @@
-TODO list for 2.7.0
-===================
-
-You can find the results of the most recent buildbot run at:
-http://buildbot.pypy.org/
-
-
-Probably easy tasks
--------------------
-
-- (unicode|bytearray).(index|find) should accept None as indices (see
- test_unicode.py)
-
-- missing posix.confstr and posix.confstr_names
-
-- remove code duplication: bit_length() and _count_bits() in rlib/rbigint.py,
- objspace/std/longobject.py and objspace/std/longtype.py.
-
-- missing module pyexpat.errors
-
-- support for PYTHONIOENCODING, this needs a way to update file.encoding
-
-- implement format__Complex_ANY() in pypy/objspace/std/complexobject.py
-
-- Code like this does not work, for two reasons::
-
- \
- from __future__ import (with_statement,
- unicode_literals)
- assert type("") is unicode
-
-- Code like::
-
- assert(x is not None, "error message")
-
- should emit a SyntaxWarning when compiled (the tuple is always true)
-
-
-Medium tasks
-------------
-
-- socket module has a couple of changes (including AF_TIPC packet range)
-
-Longer tasks
-------------
-
-- Fix usage of __cmp__ in subclasses::
-
- class badint(int):
- def __cmp__(self, other):
- raise RuntimeError
- raises(RuntimeError, cmp, 0, badint(1))
-
-- Fix comparison of objects layout: if two classes have the same __slots__, it
- should be possible to change the instances __class__::
-
- class A(object): __slots__ = ('a', 'b')
- class B(object): __slots__ = ('b', 'a')
- a = A()
- a.__class__ = B
-
-- Show a ResourceWarning when a file/socket is not explicitely closed, like
- CPython did for 3.2: http://svn.python.org/view?view=rev&revision=85920
- in PyPy this should be enabled by default
-
-Won't do for this release
--------------------------
-
-Note: when you give up with a missing feature, please mention it here, as well
-as the various skips added to the test suite.
-
-- py3k warnings
-
- * the -3 flag is accepted on the command line, but displays a warning (see
- `translator/goal/app_main.py`)
-
-- CJK codecs.
-
- * In `./conftest.py`, skipped all `test_codecencodings_*.py` and
- `test_codecmaps_*.py`.
-
- * In test_codecs, commented out various items in `all_unicode_encodings`.
-
-- Error messages about ill-formed calls (like "argument after ** must be a
- mapping") don't always show the function name. That's hard to fix for
- the case of errors raised when the Argument object is created (as opposed
- to when parsing for a given target function, which occurs later).
-
- * Some "..." were added to doctests in test_extcall.py
-
-- CPython's builtin methods are both functions and unbound methods (for
- example, `str.upper is dict(str.__dict__)['upper']`). This is not the case
- in pypy, and assertions like `object.__str__ is object.__str__` are False
- with pypy. Use the `==` operator instead.
-
- * pprint.py, _threading_local.py
-
-- When importing a nested module fails, the ImportError message mentions the
- name of the package up to the component that could not be imported (CPython
- prefers to display the names starting with the failing part).
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -61,7 +61,7 @@
usemodules = '',
skip=None):
self.basename = basename
- self._usemodules = usemodules.split()
+ self._usemodules = usemodules.split() + ['signal']
self._compiler = compiler
self.core = core
self.skip = skip
@@ -154,18 +154,18 @@
RegrTest('test_cmd.py'),
RegrTest('test_cmd_line_script.py'),
RegrTest('test_codeccallbacks.py', core=True),
- RegrTest('test_codecencodings_cn.py', skip="encodings not available"),
- RegrTest('test_codecencodings_hk.py', skip="encodings not available"),
- RegrTest('test_codecencodings_jp.py', skip="encodings not available"),
- RegrTest('test_codecencodings_kr.py', skip="encodings not available"),
- RegrTest('test_codecencodings_tw.py', skip="encodings not available"),
+ RegrTest('test_codecencodings_cn.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecencodings_hk.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecencodings_jp.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecencodings_kr.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecencodings_tw.py', usemodules='_multibytecodec'),
- RegrTest('test_codecmaps_cn.py', skip="encodings not available"),
- RegrTest('test_codecmaps_hk.py', skip="encodings not available"),
- RegrTest('test_codecmaps_jp.py', skip="encodings not available"),
- RegrTest('test_codecmaps_kr.py', skip="encodings not available"),
- RegrTest('test_codecmaps_tw.py', skip="encodings not available"),
- RegrTest('test_codecs.py', core=True),
+ RegrTest('test_codecmaps_cn.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecmaps_hk.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecmaps_kr.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'),
+ RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'),
RegrTest('test_codeop.py', core=True),
RegrTest('test_coercion.py', core=True),
RegrTest('test_collections.py'),
@@ -201,7 +201,7 @@
RegrTest('test_difflib.py'),
RegrTest('test_dircache.py', core=True),
RegrTest('test_dis.py'),
- RegrTest('test_distutils.py'),
+ RegrTest('test_distutils.py', skip=True),
RegrTest('test_dl.py', skip=True),
RegrTest('test_doctest.py', usemodules="thread"),
RegrTest('test_doctest2.py'),
@@ -314,10 +314,10 @@
RegrTest('test_mmap.py'),
RegrTest('test_module.py', core=True),
RegrTest('test_modulefinder.py'),
- RegrTest('test_multibytecodec.py', skip="unsupported codecs"),
+ RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
RegrTest('test_multibytecodec_support.py', skip="not a test"),
RegrTest('test_multifile.py'),
- RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+ RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
RegrTest('test_mutants.py', core="possibly"),
RegrTest('test_mutex.py'),
RegrTest('test_netrc.py'),
@@ -359,7 +359,7 @@
RegrTest('test_property.py', core=True),
RegrTest('test_pstats.py'),
RegrTest('test_pty.py', skip="unsupported extension module"),
- RegrTest('test_pwd.py', skip=skip_win32),
+ RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32),
RegrTest('test_py3kwarn.py'),
RegrTest('test_pyclbr.py'),
RegrTest('test_pydoc.py'),
@@ -400,7 +400,7 @@
RegrTest('test_softspace.py', core=True),
RegrTest('test_sort.py', core=True),
- RegrTest('test_ssl.py'),
+ RegrTest('test_ssl.py', usemodules='_ssl _socket select'),
RegrTest('test_str.py', core=True),
RegrTest('test_strftime.py'),
@@ -569,7 +569,6 @@
#
import os
import time
-import socket
import getpass
class ReallyRunFileExternal(py.test.collect.Item):
diff --git a/lib-python/modified-2.7/ctypes/__init__.py b/lib-python/modified-2.7/ctypes/__init__.py
--- a/lib-python/modified-2.7/ctypes/__init__.py
+++ b/lib-python/modified-2.7/ctypes/__init__.py
@@ -7,6 +7,7 @@
__version__ = "1.1.0"
+import _ffi
from _ctypes import Union, Structure, Array
from _ctypes import _Pointer
from _ctypes import CFuncPtr as _CFuncPtr
@@ -350,7 +351,7 @@
self._FuncPtr = _FuncPtr
if handle is None:
- self._handle = _dlopen(self._name, mode)
+ self._handle = _ffi.CDLL(name, mode)
else:
self._handle = handle
@@ -488,9 +489,12 @@
_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
return CFunctionType
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
def cast(obj, typ):
- return _cast(obj, obj, typ)
+ try:
+ c_void_p.from_param(obj)
+ except TypeError, e:
+ raise ArgumentError(str(e))
+ return _cast_addr(obj, obj, typ)
_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
def string_at(ptr, size=-1):
diff --git a/lib-python/modified-2.7/ctypes/test/test_callbacks.py b/lib-python/modified-2.7/ctypes/test/test_callbacks.py
--- a/lib-python/modified-2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/modified-2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
import unittest
from ctypes import *
+from ctypes.test import xfail
import _ctypes_test
class Callbacks(unittest.TestCase):
@@ -98,6 +99,7 @@
## self.check_type(c_char_p, "abc")
## self.check_type(c_char_p, "def")
+ @xfail
def test_pyobject(self):
o = ()
from sys import getrefcount as grc
diff --git a/lib-python/modified-2.7/ctypes/test/test_cfuncs.py b/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
@@ -3,8 +3,8 @@
import unittest
from ctypes import *
-
import _ctypes_test
+from test.test_support import impl_detail
class CFunctions(unittest.TestCase):
_dll = CDLL(_ctypes_test.__file__)
@@ -158,12 +158,14 @@
self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
self.assertEqual(self.S(), 42)
+ @impl_detail('long double not supported by PyPy', pypy=False)
def test_longdouble(self):
self._dll.tf_D.restype = c_longdouble
self._dll.tf_D.argtypes = (c_longdouble,)
self.assertEqual(self._dll.tf_D(42.), 14.)
self.assertEqual(self.S(), 42)
-
+
+ @impl_detail('long double not supported by PyPy', pypy=False)
def test_longdouble_plus(self):
self._dll.tf_bD.restype = c_longdouble
self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/lib-python/modified-2.7/ctypes/test/test_functions.py b/lib-python/modified-2.7/ctypes/test/test_functions.py
--- a/lib-python/modified-2.7/ctypes/test/test_functions.py
+++ b/lib-python/modified-2.7/ctypes/test/test_functions.py
@@ -8,6 +8,7 @@
from ctypes import *
import sys, unittest
from ctypes.test import xfail
+from test.test_support import impl_detail
try:
WINFUNCTYPE
@@ -144,6 +145,7 @@
self.assertEqual(result, -21)
self.assertEqual(type(result), float)
+ @impl_detail('long double not supported by PyPy', pypy=False)
def test_longdoubleresult(self):
f = dll._testfunc_D_bhilfD
f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
diff --git a/lib-python/modified-2.7/ctypes/test/test_libc.py b/lib-python/modified-2.7/ctypes/test/test_libc.py
--- a/lib-python/modified-2.7/ctypes/test/test_libc.py
+++ b/lib-python/modified-2.7/ctypes/test/test_libc.py
@@ -25,7 +25,11 @@
lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort))
self.assertEqual(chars.raw, " ,,aaaadmmmnpppsss\x00")
- def test_no_more_xfail(self):
+ def SKIPPED_test_no_more_xfail(self):
+ # We decided to not explicitly support the whole ctypes-2.7
+ # and instead go for a case-by-case, demand-driven approach.
+ # So this test is skipped instead of failing.
+ import socket
import ctypes.test
self.assertTrue(not hasattr(ctypes.test, 'xfail'),
"You should incrementally grep for '@xfail' and remove them, they are real failures")
diff --git a/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py b/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py
--- a/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py
+++ b/lib-python/modified-2.7/ctypes/test/test_simplesubclasses.py
@@ -1,6 +1,5 @@
import unittest
from ctypes import *
-from ctypes.test import xfail
class MyInt(c_int):
def __cmp__(self, other):
@@ -27,7 +26,6 @@
self.assertEqual(None, cb())
- @xfail
def test_int_callback(self):
args = []
def func(arg):
diff --git a/lib-python/modified-2.7/ctypes/util.py b/lib-python/modified-2.7/ctypes/util.py
--- a/lib-python/modified-2.7/ctypes/util.py
+++ b/lib-python/modified-2.7/ctypes/util.py
@@ -72,8 +72,8 @@
return name
if os.name == "posix" and sys.platform == "darwin":
- from ctypes.macholib.dyld import dyld_find as _dyld_find
def find_library(name):
+ from ctypes.macholib.dyld import dyld_find as _dyld_find
possible = ['lib%s.dylib' % name,
'%s.dylib' % name,
'%s.framework/%s' % (name, name)]
diff --git a/lib-python/modified-2.7/distutils/cygwinccompiler.py b/lib-python/modified-2.7/distutils/cygwinccompiler.py
--- a/lib-python/modified-2.7/distutils/cygwinccompiler.py
+++ b/lib-python/modified-2.7/distutils/cygwinccompiler.py
@@ -75,6 +75,9 @@
elif msc_ver == '1500':
# VS2008 / MSVC 9.0
return ['msvcr90']
+ elif msc_ver == '1600':
+ # VS2010 / MSVC 10.0
+ return ['msvcr100']
else:
raise ValueError("Unknown MS Compiler version %s " % msc_ver)
diff --git a/lib-python/modified-2.7/distutils/sysconfig.py b/lib-python/modified-2.7/distutils/sysconfig.py
--- a/lib-python/modified-2.7/distutils/sysconfig.py
+++ b/lib-python/modified-2.7/distutils/sysconfig.py
@@ -20,8 +20,10 @@
if '__pypy__' in sys.builtin_module_names:
from distutils.sysconfig_pypy import *
from distutils.sysconfig_pypy import _config_vars # needed by setuptools
+ from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
else:
from distutils.sysconfig_cpython import *
from distutils.sysconfig_cpython import _config_vars # needed by setuptools
+ from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
--- a/lib-python/modified-2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
@@ -116,3 +116,13 @@
if compiler.compiler_type == "unix":
compiler.compiler_so.extend(['-fPIC', '-Wimplicit'])
compiler.shared_lib_extension = get_config_var('SO')
+ if "CFLAGS" in os.environ:
+ cflags = os.environ["CFLAGS"]
+ compiler.compiler.append(cflags)
+ compiler.compiler_so.append(cflags)
+ compiler.linker_so.append(cflags)
+
+
+from sysconfig_cpython import (
+ parse_makefile, _variable_rx, expand_makefile_vars)
+
diff --git a/lib-python/modified-2.7/distutils/unixccompiler.py b/lib-python/modified-2.7/distutils/unixccompiler.py
--- a/lib-python/modified-2.7/distutils/unixccompiler.py
+++ b/lib-python/modified-2.7/distutils/unixccompiler.py
@@ -324,7 +324,7 @@
# On OSX users can specify an alternate SDK using
# '-isysroot', calculate the SDK root if it is specified
# (and use it further on)
- cflags = sysconfig.get_config_var('CFLAGS')
+ cflags = sysconfig.get_config_var('CFLAGS') or ''
m = re.search(r'-isysroot\s+(\S+)', cflags)
if m is None:
sysroot = '/'
diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/heapq.py
@@ -0,0 +1,442 @@
+# -*- coding: latin-1 -*-
+
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0. For the sake of comparison,
+non-existing elements are considered to be infinite. The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = [] # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0] # smallest item on the heap without popping it
+heapify(x) # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+ # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing. This makes the relationship between the
+ index for a node and the indexes for its children slightly less
+ obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by Fran�ois Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0. For the sake of comparison,
+non-existing elements are considered to be infinite. The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament. The numbers below are `k', not a[k]:
+
+ 0
+
+ 1 2
+
+ 3 4 5 6
+
+ 7 8 9 10 11 12 13 14
+
+ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In
+an usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had. However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner. The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted. This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time. When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap. So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case. However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts. You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1]. It is very important that the initial
+sort produces the longest runs possible. Tournaments are a good way
+to that. If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases. The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting. When the first heap completely
+vanishes, you switch heaps and start a new run. Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know. I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks. On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge). Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+ 'nlargest', 'nsmallest', 'heappushpop']
+
+from itertools import islice, repeat, count, imap, izip, tee, chain
+from operator import itemgetter
+import bisect
+
+def heappush(heap, item):
+ """Push item onto heap, maintaining the heap invariant."""
+ heap.append(item)
+ _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+ """Pop the smallest item off the heap, maintaining the heap invariant."""
+ lastelt = heap.pop() # raises appropriate IndexError if heap is empty
+ if heap:
+ returnitem = heap[0]
+ heap[0] = lastelt
+ _siftup(heap, 0)
+ else:
+ returnitem = lastelt
+ return returnitem
+
+def heapreplace(heap, item):
+ """Pop and return the current smallest value, and add the new item.
+
+ This is more efficient than heappop() followed by heappush(), and can be
+ more appropriate when using a fixed-size heap. Note that the value
+ returned may be larger than item! That constrains reasonable uses of
+ this routine unless written as part of a conditional replacement:
+
+ if item > heap[0]:
+ item = heapreplace(heap, item)
+ """
+ returnitem = heap[0] # raises appropriate IndexError if heap is empty
+ heap[0] = item
+ _siftup(heap, 0)
+ return returnitem
+
+def heappushpop(heap, item):
+ """Fast version of a heappush followed by a heappop."""
+ if heap and heap[0] < item:
+ item, heap[0] = heap[0], item
+ _siftup(heap, 0)
+ return item
+
+def heapify(x):
+ """Transform list into a heap, in-place, in O(len(heap)) time."""
+ n = len(x)
+ # Transform bottom-up. The largest index there's any point to looking at
+ # is the largest with a child index in-range, so must have 2*i + 1 < n,
+ # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+ # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is
+ # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+ for i in reversed(xrange(n//2)):
+ _siftup(x, i)
+
+def nlargest(n, iterable):
+ """Find the n largest elements in a dataset.
+
+ Equivalent to: sorted(iterable, reverse=True)[:n]
+ """
+ if n < 0: # for consistency with the c impl
+ return []
+ it = iter(iterable)
+ result = list(islice(it, n))
+ if not result:
+ return result
+ heapify(result)
+ _heappushpop = heappushpop
+ for elem in it:
+ _heappushpop(result, elem)
+ result.sort(reverse=True)
+ return result
+
+def nsmallest(n, iterable):
+ """Find the n smallest elements in a dataset.
+
+ Equivalent to: sorted(iterable)[:n]
+ """
+ if n < 0: # for consistency with the c impl
+ return []
+ if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
+ # For smaller values of n, the bisect method is faster than a minheap.
+ # It is also memory efficient, consuming only n elements of space.
+ it = iter(iterable)
+ result = sorted(islice(it, 0, n))
+ if not result:
+ return result
+ insort = bisect.insort
+ pop = result.pop
+ los = result[-1] # los --> Largest of the nsmallest
+ for elem in it:
+ if los <= elem:
+ continue
+ insort(result, elem)
+ pop()
+ los = result[-1]
+ return result
+ # An alternative approach manifests the whole iterable in memory but
+ # saves comparisons by heapifying all at once. Also, saves time
+ # over bisect.insort() which has O(n) data movement time for every
+ # insertion. Finding the n smallest of an m length iterable requires
+ # O(m) + O(n log m) comparisons.
+ h = list(iterable)
+ heapify(h)
+ return map(heappop, repeat(h, min(n, len(h))))
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos
+# is the index of a leaf with a possibly out-of-order value. Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+ newitem = heap[pos]
+ # Follow the path to the root, moving parents down until finding a place
+ # newitem fits.
+ while pos > startpos:
+ parentpos = (pos - 1) >> 1
+ parent = heap[parentpos]
+ if newitem < parent:
+ heap[pos] = parent
+ pos = parentpos
+ continue
+ break
+ heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too. We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way. During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early). See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom __cmp__ methods, or in array elements
+# storing (priority, record) tuples. Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory. Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify Compares needed by 1000 heappops
+# -------------------------- --------------------------------
+# 1837 cut to 1663 14996 cut to 8680
+# 1855 cut to 1659 14966 cut to 8678
+# 1847 cut to 1660 15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares: heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares): list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+ endpos = len(heap)
+ startpos = pos
+ newitem = heap[pos]
+ # Bubble up the smaller child until hitting a leaf.
+ childpos = 2*pos + 1 # leftmost child position
+ while childpos < endpos:
+ # Set childpos to index of smaller child.
+ rightpos = childpos + 1
+ if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+ childpos = rightpos
+ # Move the smaller child up.
+ heap[pos] = heap[childpos]
+ pos = childpos
+ childpos = 2*pos + 1
+ # The leaf at pos is empty now. Put newitem there, and bubble it up
+ # to its final resting place (by sifting its parents down).
+ heap[pos] = newitem
+ _siftdown(heap, startpos, pos)
+
+# If available, use C implementation
+try:
+ from _heapq import *
+except ImportError:
+ pass
+
+def merge(*iterables):
+ '''Merge multiple sorted inputs into a single sorted output.
+
+ Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+ does not pull the data into memory all at once, and assumes that each of
+ the input streams is already sorted (smallest to largest).
+
+ >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+ [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+ '''
+ _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
+
+ h = []
+ h_append = h.append
+ for itnum, it in enumerate(map(iter, iterables)):
+ try:
+ next = it.next
+ h_append([next(), itnum, next])
+ except _StopIteration:
+ pass
+ heapify(h)
+
+ while 1:
+ try:
+ while 1:
+ v, itnum, next = s = h[0] # raises IndexError when h is empty
+ yield v
+ s[0] = next() # raises StopIteration when exhausted
+ _heapreplace(h, s) # restore heap condition
+ except _StopIteration:
+ _heappop(h) # remove empty iterator
+ except IndexError:
+ return
+
+# Extend the implementations of nsmallest and nlargest to use a key= argument
+_nsmallest = nsmallest
+def nsmallest(n, iterable, key=None):
+ """Find the n smallest elements in a dataset.
+
+ Equivalent to: sorted(iterable, key=key)[:n]
+ """
+ # Short-cut for n==1 is to use min() when len(iterable)>0
+ if n == 1:
+ it = iter(iterable)
+ head = list(islice(it, 1))
+ if not head:
+ return []
+ if key is None:
+ return [min(chain(head, it))]
+ return [min(chain(head, it), key=key)]
+
+ # When n>=size, it's faster to use sort()
+ try:
+ size = len(iterable)
+ except (TypeError, AttributeError):
+ pass
+ else:
+ if n >= size:
+ return sorted(iterable, key=key)[:n]
+
+ # When key is none, use simpler decoration
+ if key is None:
+ it = izip(iterable, count()) # decorate
+ result = _nsmallest(n, it)
+ return map(itemgetter(0), result) # undecorate
+
+ # General case, slowest method
+ in1, in2 = tee(iterable)
+ it = izip(imap(key, in1), count(), in2) # decorate
+ result = _nsmallest(n, it)
+ return map(itemgetter(2), result) # undecorate
+
+_nlargest = nlargest
+def nlargest(n, iterable, key=None):
+ """Find the n largest elements in a dataset.
+
+ Equivalent to: sorted(iterable, key=key, reverse=True)[:n]
+ """
+
+ # Short-cut for n==1 is to use max() when len(iterable)>0
+ if n == 1:
+ it = iter(iterable)
+ head = list(islice(it, 1))
+ if not head:
+ return []
+ if key is None:
+ return [max(chain(head, it))]
+ return [max(chain(head, it), key=key)]
+
+ # When n>=size, it's faster to use sort()
+ try:
+ size = len(iterable)
+ except (TypeError, AttributeError):
+ pass
+ else:
+ if n >= size:
+ return sorted(iterable, key=key, reverse=True)[:n]
+
+ # When key is none, use simpler decoration
+ if key is None:
+ it = izip(iterable, count(0,-1)) # decorate
+ result = _nlargest(n, it)
+ return map(itemgetter(0), result) # undecorate
+
+ # General case, slowest method
+ in1, in2 = tee(iterable)
+ it = izip(imap(key, in1), count(0,-1), in2) # decorate
+ result = _nlargest(n, it)
+ return map(itemgetter(2), result) # undecorate
+
+if __name__ == "__main__":
+ # Simple sanity test
+ heap = []
+ data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
+ for item in data:
+ heappush(heap, item)
+ sort = []
+ while heap:
+ sort.append(heappop(heap))
+ print sort
+
+ import doctest
+ doctest.testmod()
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+ (null)
+ |
+ | HTTPConnection()
+ v
+ Idle
+ |
+ | putrequest()
+ v
+ Request-started
+ |
+ | ( putheader() )* endheaders()
+ v
+ Request-sent
+ |
+ | response = getresponse()
+ v
+ Unread-response [Response-headers-read]
+ |\____________________
+ | |
+ | response.read() | putrequest()
+ v v
+ Idle Req-started-unread-response
+ ______/|
+ / |
+ response.read() | | ( putheader() )* endheaders()
+ v v
+ Request-started Req-sent-unread-response
+ |
+ | response.read()
+ v
+ Request-sent
+
+This diagram presents the following rules:
+ -- a second request may not be started until {response-headers-read}
+ -- a response [object] cannot be retrieved until {request-sent}
+ -- there is no differentiation between an unread response body and a
+ partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+ HTTPResponse class does not enforce this state machine, which
+ implies sophisticated clients may accelerate the request/response
+ pipeline. Caution should be taken, though: accelerating the states
+ beyond the above pattern may imply knowledge of the server's
+ connection-close behavior for certain requests. For example, it
+ is impossible to tell whether the server will close the connection
+ UNTIL the response headers have been read; this means that further
+ requests cannot be placed into the pipeline until it is known that
+ the server will NOT be closing the connection.
+
+Logical State __state __response
+------------- ------- ----------
+Idle _CS_IDLE None
+Request-started _CS_REQ_STARTED None
+Request-sent _CS_REQ_SENT None
+Unread-response _CS_IDLE <response_class>
+Req-started-unread-response _CS_REQ_STARTED <response_class>
+Req-sent-unread-response _CS_REQ_SENT <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+ if py3kwarning:
+ warnings.filterwarnings("ignore", ".*mimetools has been removed",
+ DeprecationWarning)
+ import mimetools
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+ "HTTPException", "NotConnected", "UnknownProtocol",
+ "UnknownTransferEncoding", "UnimplementedFileMode",
+ "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+ "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+ "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+ 100: 'Continue',
+ 101: 'Switching Protocols',
+
+ 200: 'OK',
+ 201: 'Created',
+ 202: 'Accepted',
+ 203: 'Non-Authoritative Information',
+ 204: 'No Content',
+ 205: 'Reset Content',
+ 206: 'Partial Content',
+
+ 300: 'Multiple Choices',
+ 301: 'Moved Permanently',
+ 302: 'Found',
+ 303: 'See Other',
+ 304: 'Not Modified',
+ 305: 'Use Proxy',
+ 306: '(Unused)',
+ 307: 'Temporary Redirect',
+
+ 400: 'Bad Request',
+ 401: 'Unauthorized',
+ 402: 'Payment Required',
+ 403: 'Forbidden',
+ 404: 'Not Found',
+ 405: 'Method Not Allowed',
+ 406: 'Not Acceptable',
+ 407: 'Proxy Authentication Required',
+ 408: 'Request Timeout',
+ 409: 'Conflict',
+ 410: 'Gone',
+ 411: 'Length Required',
+ 412: 'Precondition Failed',
+ 413: 'Request Entity Too Large',
+ 414: 'Request-URI Too Long',
+ 415: 'Unsupported Media Type',
+ 416: 'Requested Range Not Satisfiable',
+ 417: 'Expectation Failed',
+
+ 500: 'Internal Server Error',
+ 501: 'Not Implemented',
+ 502: 'Bad Gateway',
+ 503: 'Service Unavailable',
+ 504: 'Gateway Timeout',
+ 505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+ def addheader(self, key, value):
+ """Add header for field key handling repeats."""
+ prev = self.dict.get(key)
+ if prev is None:
+ self.dict[key] = value
+ else:
+ combined = ", ".join((prev, value))
+ self.dict[key] = combined
+
+ def addcontinue(self, key, more):
+ """Add more field data from a continuation line."""
+ prev = self.dict[key]
+ self.dict[key] = prev + "\n " + more
+
+ def readheaders(self):
+ """Read header lines.
+
+ Read header lines up to the entirely blank line that terminates them.
+ The (normally blank) line that ends the headers is skipped, but not
+ included in the returned list. If a non-header line ends the headers,
+ (which is an error), an attempt is made to backspace over it; it is
+ never included in the returned list.
+
+ The variable self.status is set to the empty string if all went well,
+ otherwise it is an error message. The variable self.headers is a
+ completely uninterpreted list of lines contained in the header (so
+ printing them will reproduce the header exactly as it appears in the
+ file).
+
+ If multiple header fields with the same name occur, they are combined
+ according to the rules in RFC 2616 sec 4.2:
+
+ Appending each subsequent field-value to the first, each separated
+ by a comma. The order in which header fields with the same field-name
+ are received is significant to the interpretation of the combined
+ field value.
+ """
+ # XXX The implementation overrides the readheaders() method of
+ # rfc822.Message. The base class design isn't amenable to
+ # customized behavior here so the method here is a copy of the
+ # base class code with a few small changes.
+
+ self.dict = {}
+ self.unixfrom = ''
+ self.headers = hlist = []
+ self.status = ''
+ headerseen = ""
+ firstline = 1
+ startofline = unread = tell = None
+ if hasattr(self.fp, 'unread'):
+ unread = self.fp.unread
+ elif self.seekable:
+ tell = self.fp.tell
+ while True:
+ if tell:
+ try:
+ startofline = tell()
+ except IOError:
+ startofline = tell = None
+ self.seekable = 0
+ line = self.fp.readline()
+ if not line:
+ self.status = 'EOF in headers'
+ break
+ # Skip unix From name time lines
+ if firstline and line.startswith('From '):
+ self.unixfrom = self.unixfrom + line
+ continue
+ firstline = 0
+ if headerseen and line[0] in ' \t':
+ # XXX Not sure if continuation lines are handled properly
+ # for http and/or for repeating headers
+ # It's a continuation line.
+ hlist.append(line)
+ self.addcontinue(headerseen, line.strip())
+ continue
+ elif self.iscomment(line):
+ # It's a comment. Ignore it.
+ continue
+ elif self.islast(line):
+ # Note! No pushback here! The delimiter line gets eaten.
+ break
+ headerseen = self.isheader(line)
+ if headerseen:
+ # It's a legal header line, save it.
+ hlist.append(line)
+ self.addheader(headerseen, line[len(headerseen)+1:].strip())
+ continue
+ else:
+ # It's not a header line; throw it back and stop here.
+ if not self.dict:
+ self.status = 'No headers'
+ else:
+ self.status = 'Non-header line where header expected'
+ # Try to undo the read.
+ if unread:
+ unread(line)
+ elif tell:
+ self.fp.seek(startofline)
+ else:
+ self.status = self.status + '; bad seek'
+ break
+
+class HTTPResponse:
+
+ # strict: If true, raise BadStatusLine if the status line can't be
+ # parsed as a valid HTTP/1.0 or 1.1 status line. By default it is
+ # false because it prevents clients from talking to HTTP/0.9
+ # servers. Note that a response with a sufficiently corrupted
+ # status line will look like an HTTP/0.9 response.
+
+ # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+ def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+ if buffering:
+ # The caller won't be using any sock.recv() calls, so buffering
+ # is fine and recommended for performance.
+ self.fp = sock.makefile('rb')
+ else:
+ # The buffer size is specified as zero, because the headers of
+ # the response are read with readline(). If the reads were
+ # buffered the readline() calls could consume some of the
+ # response, which make be read via a recv() on the underlying
+ # socket.
+ self.fp = sock.makefile('rb', 0)
+ self.debuglevel = debuglevel
+ self.strict = strict
+ self._method = method
+
+ self.msg = None
+
+ # from the Status-Line of the response
+ self.version = _UNKNOWN # HTTP-Version
+ self.status = _UNKNOWN # Status-Code
+ self.reason = _UNKNOWN # Reason-Phrase
+
+ self.chunked = _UNKNOWN # is "chunked" being used?
+ self.chunk_left = _UNKNOWN # bytes left to read in current chunk
+ self.length = _UNKNOWN # number of bytes left in response
+ self.will_close = _UNKNOWN # conn will close at end of response
+
+ def _read_status(self):
+ # Initialize with Simple-Response defaults
+ line = self.fp.readline()
+ if self.debuglevel > 0:
+ print "reply:", repr(line)
+ if not line:
+ # Presumably, the server closed the connection before
+ # sending a valid response.
+ raise BadStatusLine(line)
+ try:
+ [version, status, reason] = line.split(None, 2)
+ except ValueError:
+ try:
+ [version, status] = line.split(None, 1)
+ reason = ""
+ except ValueError:
+ # empty version will cause next test to fail and status
+ # will be treated as 0.9 response.
+ version = ""
+ if not version.startswith('HTTP/'):
+ if self.strict:
+ self.close()
+ raise BadStatusLine(line)
+ else:
+ # assume it's a Simple-Response from an 0.9 server
+ self.fp = LineAndFileWrapper(line, self.fp)
+ return "HTTP/0.9", 200, ""
+
+ # The status code is a three-digit number
+ try:
+ status = int(status)
+ if status < 100 or status > 999:
+ raise BadStatusLine(line)
+ except ValueError:
+ raise BadStatusLine(line)
+ return version, status, reason
+
+ def begin(self):
+ if self.msg is not None:
+ # we've already started reading the response
+ return
+
+ # read until we get a non-100 response
+ while True:
+ version, status, reason = self._read_status()
+ if status != CONTINUE:
+ break
+ # skip the header from the 100 response
+ while True:
+ skip = self.fp.readline().strip()
+ if not skip:
+ break
+ if self.debuglevel > 0:
+ print "header:", skip
+
+ self.status = status
+ self.reason = reason.strip()
+ if version == 'HTTP/1.0':
+ self.version = 10
+ elif version.startswith('HTTP/1.'):
+ self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
+ elif version == 'HTTP/0.9':
+ self.version = 9
+ else:
+ raise UnknownProtocol(version)
+
+ if self.version == 9:
+ self.length = None
+ self.chunked = 0
+ self.will_close = 1
+ self.msg = HTTPMessage(StringIO())
+ return
+
+ self.msg = HTTPMessage(self.fp, 0)
+ if self.debuglevel > 0:
+ for hdr in self.msg.headers:
+ print "header:", hdr,
+
+ # don't let the msg keep an fp
+ self.msg.fp = None
+
+ # are we using the chunked-style of transfer encoding?
+ tr_enc = self.msg.getheader('transfer-encoding')
+ if tr_enc and tr_enc.lower() == "chunked":
+ self.chunked = 1
+ self.chunk_left = None
+ else:
+ self.chunked = 0
+
+ # will the connection close at the end of the response?
+ self.will_close = self._check_close()
+
+ # do we have a Content-Length?
+ # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+ length = self.msg.getheader('content-length')
+ if length and not self.chunked:
+ try:
+ self.length = int(length)
+ except ValueError:
+ self.length = None
+ else:
+ if self.length < 0: # ignore nonsensical negative lengths
+ self.length = None
+ else:
+ self.length = None
+
+ # does the body have a fixed length? (of zero)
+ if (status == NO_CONTENT or status == NOT_MODIFIED or
+ 100 <= status < 200 or # 1xx codes
+ self._method == 'HEAD'):
+ self.length = 0
+
+ # if the connection remains open, and we aren't using chunked, and
+ # a content-length was not provided, then assume that the connection
+ # WILL close.
+ if not self.will_close and \
+ not self.chunked and \
+ self.length is None:
+ self.will_close = 1
+
+ def _check_close(self):
+ conn = self.msg.getheader('connection')
+ if self.version == 11:
+ # An HTTP/1.1 proxy is assumed to stay open unless
+ # explicitly closed.
+ conn = self.msg.getheader('connection')
+ if conn and "close" in conn.lower():
+ return True
+ return False
+
+ # Some HTTP/1.0 implementations have support for persistent
+ # connections, using rules different than HTTP/1.1.
+
+ # For older HTTP, Keep-Alive indicates persistent connection.
+ if self.msg.getheader('keep-alive'):
+ return False
+
+ # At least Akamai returns a "Connection: Keep-Alive" header,
+ # which was supposed to be sent by the client.
+ if conn and "keep-alive" in conn.lower():
+ return False
+
+ # Proxy-Connection is a netscape hack.
+ pconn = self.msg.getheader('proxy-connection')
+ if pconn and "keep-alive" in pconn.lower():
+ return False
+
+ # otherwise, assume it will close
+ return True
+
+ def close(self):
+ if self.fp:
+ self.fp.close()
+ self.fp = None
+
+ def isclosed(self):
+ # NOTE: it is possible that we will not ever call self.close(). This
+ # case occurs when will_close is TRUE, length is None, and we
+ # read up to the last byte, but NOT past it.
+ #
+ # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+ # called, meaning self.isclosed() is meaningful.
+ return self.fp is None
+
+ # XXX It would be nice to have readline and __iter__ for this, too.
+
+ def read(self, amt=None):
+ if self.fp is None:
+ return ''
+
+ if self._method == 'HEAD':
+ self.close()
+ return ''
+
+ if self.chunked:
+ return self._read_chunked(amt)
+
+ if amt is None:
+ # unbounded read
+ if self.length is None:
+ s = self.fp.read()
+ else:
+ s = self._safe_read(self.length)
+ self.length = 0
+ self.close() # we read everything
+ return s
+
+ if self.length is not None:
+ if amt > self.length:
+ # clip the read to the "end of response"
+ amt = self.length
+
+ # we do not use _safe_read() here because this may be a .will_close
+ # connection, and the user is reading more bytes than will be provided
+ # (for example, reading in 1k chunks)
+ s = self.fp.read(amt)
+ if self.length is not None:
+ self.length -= len(s)
+ if not self.length:
+ self.close()
+ return s
+
+ def _read_chunked(self, amt):
+ assert self.chunked != _UNKNOWN
+ chunk_left = self.chunk_left
+ value = []
+ while True:
+ if chunk_left is None:
+ line = self.fp.readline()
+ i = line.find(';')
+ if i >= 0:
+ line = line[:i] # strip chunk-extensions
+ try:
+ chunk_left = int(line, 16)
+ except ValueError:
+ # close the connection as protocol synchronisation is
+ # probably lost
+ self.close()
+ raise IncompleteRead(''.join(value))
+ if chunk_left == 0:
+ break
+ if amt is None:
+ value.append(self._safe_read(chunk_left))
+ elif amt < chunk_left:
+ value.append(self._safe_read(amt))
+ self.chunk_left = chunk_left - amt
+ return ''.join(value)
+ elif amt == chunk_left:
+ value.append(self._safe_read(amt))
+ self._safe_read(2) # toss the CRLF at the end of the chunk
+ self.chunk_left = None
+ return ''.join(value)
+ else:
+ value.append(self._safe_read(chunk_left))
+ amt -= chunk_left
+
+ # we read the whole chunk, get another
+ self._safe_read(2) # toss the CRLF at the end of the chunk
+ chunk_left = None
+
+ # read and discard trailer up to the CRLF terminator
+ ### note: we shouldn't have any trailers!
+ while True:
+ line = self.fp.readline()
+ if not line:
+ # a vanishingly small number of sites EOF without
+ # sending the trailer
+ break
+ if line == '\r\n':
+ break
+
+ # we read everything; close the "file"
+ self.close()
+
+ return ''.join(value)
+
+ def _safe_read(self, amt):
+ """Read the number of bytes requested, compensating for partial reads.
+
+ Normally, we have a blocking socket, but a read() can be interrupted
+ by a signal (resulting in a partial read).
+
+ Note that we cannot distinguish between EOF and an interrupt when zero
+ bytes have been read. IncompleteRead() will be raised in this
+ situation.
+
+ This function should be used when <amt> bytes "should" be present for
+ reading. If the bytes are truly not available (due to EOF), then the
+ IncompleteRead exception can be used to detect the problem.
+ """
+ # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+ # return less than x bytes unless EOF is encountered. It now handles
+ # signal interruptions (socket.error EINTR) internally. This code
+ # never caught that exception anyways. It seems largely pointless.
+ # self.fp.read(amt) will work fine.
+ s = []
+ while amt > 0:
+ chunk = self.fp.read(min(amt, MAXAMOUNT))
+ if not chunk:
+ raise IncompleteRead(''.join(s), amt)
+ s.append(chunk)
+ amt -= len(chunk)
+ return ''.join(s)
+
+ def fileno(self):
+ return self.fp.fileno()
+
+ def getheader(self, name, default=None):
+ if self.msg is None:
+ raise ResponseNotReady()
+ return self.msg.getheader(name, default)
+
+ def getheaders(self):
+ """Return list of (header, value) tuples."""
+ if self.msg is None:
+ raise ResponseNotReady()
+ return self.msg.items()
+
+
+class HTTPConnection:
+
+ _http_vsn = 11
+ _http_vsn_str = 'HTTP/1.1'
+
+ response_class = HTTPResponse
+ default_port = HTTP_PORT
+ auto_open = 1
+ debuglevel = 0
+ strict = 0
+
+ def __init__(self, host, port=None, strict=None,
+ timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+ self.timeout = timeout
+ self.source_address = source_address
+ self.sock = None
+ self._buffer = []
+ self.__response = None
+ self.__state = _CS_IDLE
+ self._method = None
+ self._tunnel_host = None
+ self._tunnel_port = None
+ self._tunnel_headers = {}
+
+ self._set_hostport(host, port)
+ if strict is not None:
+ self.strict = strict
+
+ def set_tunnel(self, host, port=None, headers=None):
+ """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+ The headers argument should be a mapping of extra HTTP headers
+ to send with the CONNECT request.
+ """
+ self._tunnel_host = host
+ self._tunnel_port = port
+ if headers:
+ self._tunnel_headers = headers
+ else:
+ self._tunnel_headers.clear()
+
+ def _set_hostport(self, host, port):
+ if port is None:
+ i = host.rfind(':')
+ j = host.rfind(']') # ipv6 addresses have [...]
+ if i > j:
+ try:
+ port = int(host[i+1:])
+ except ValueError:
+ raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+ host = host[:i]
+ else:
+ port = self.default_port
+ if host and host[0] == '[' and host[-1] == ']':
+ host = host[1:-1]
+ self.host = host
+ self.port = port
+
+ def set_debuglevel(self, level):
+ self.debuglevel = level
+
+ def _tunnel(self):
+ self._set_hostport(self._tunnel_host, self._tunnel_port)
+ self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+ for header, value in self._tunnel_headers.iteritems():
+ self.send("%s: %s\r\n" % (header, value))
+ self.send("\r\n")
+ response = self.response_class(self.sock, strict = self.strict,
+ method = self._method)
+ (version, code, message) = response._read_status()
+
+ if code != 200:
+ self.close()
+ raise socket.error("Tunnel connection failed: %d %s" % (code,
+ message.strip()))
+ while True:
+ line = response.fp.readline()
+ if line == '\r\n': break
+
+
+ def connect(self):
+ """Connect to the host and port specified in __init__."""
+ self.sock = socket.create_connection((self.host,self.port),
+ self.timeout, self.source_address)
+
+ if self._tunnel_host:
+ self._tunnel()
+
+ def close(self):
+ """Close the connection to the HTTP server."""
+ if self.sock:
+ self.sock.close() # close it manually... there may be other refs
+ self.sock = None
+ if self.__response:
+ self.__response.close()
+ self.__response = None
+ self.__state = _CS_IDLE
+
+ def send(self, data):
+ """Send `data' to the server."""
+ if self.sock is None:
+ if self.auto_open:
+ self.connect()
+ else:
+ raise NotConnected()
+
+ if self.debuglevel > 0:
+ print "send:", repr(data)
+ blocksize = 8192
+ if hasattr(data,'read') and not isinstance(data, array):
+ if self.debuglevel > 0: print "sendIng a read()able"
+ datablock = data.read(blocksize)
+ while datablock:
+ self.sock.sendall(datablock)
+ datablock = data.read(blocksize)
+ else:
+ self.sock.sendall(data)
+
+ def _output(self, s):
+ """Add a line of output to the current request buffer.
+
+ Assumes that the line does *not* end with \\r\\n.
+ """
+ self._buffer.append(s)
+
+ def _send_output(self, message_body=None):
+ """Send the currently buffered request and clear the buffer.
+
+ Appends an extra \\r\\n to the buffer.
+ A message_body may be specified, to be appended to the request.
+ """
+ self._buffer.extend(("", ""))
+ msg = "\r\n".join(self._buffer)
+ del self._buffer[:]
+ # If msg and message_body are sent in a single send() call,
+ # it will avoid performance problems caused by the interaction
+ # between delayed ack and the Nagle algorithim.
+ if isinstance(message_body, str):
+ msg += message_body
+ message_body = None
+ self.send(msg)
+ if message_body is not None:
+ #message_body was not a string (i.e. it is a file) and
+ #we must run the risk of Nagle
+ self.send(message_body)
+
+ def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+ """Send a request to the server.
+
+ `method' specifies an HTTP request method, e.g. 'GET'.
+ `url' specifies the object being requested, e.g. '/index.html'.
+ `skip_host' if True does not add automatically a 'Host:' header
+ `skip_accept_encoding' if True does not add automatically an
+ 'Accept-Encoding:' header
+ """
+
+ # if a prior response has been completed, then forget about it.
+ if self.__response and self.__response.isclosed():
+ self.__response = None
+
+
+ # in certain cases, we cannot issue another request on this connection.
+ # this occurs when:
+ # 1) we are in the process of sending a request. (_CS_REQ_STARTED)
+ # 2) a response to a previous request has signalled that it is going
+ # to close the connection upon completion.
+ # 3) the headers for the previous response have not been read, thus
+ # we cannot determine whether point (2) is true. (_CS_REQ_SENT)
+ #
+ # if there is no prior response, then we can request at will.
+ #
+ # if point (2) is true, then we will have passed the socket to the
+ # response (effectively meaning, "there is no prior response"), and
+ # will open a new one when a new request is made.
+ #
+ # Note: if a prior response exists, then we *can* start a new request.
+ # We are not allowed to begin fetching the response to this new
+ # request, however, until that prior response is complete.
+ #
+ if self.__state == _CS_IDLE:
+ self.__state = _CS_REQ_STARTED
+ else:
+ raise CannotSendRequest()
+
+ # Save the method we use, we need it later in the response phase
+ self._method = method
+ if not url:
+ url = '/'
+ hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+ self._output(hdr)
+
+ if self._http_vsn == 11:
+ # Issue some standard headers for better HTTP/1.1 compliance
+
+ if not skip_host:
+ # this header is issued *only* for HTTP/1.1
+ # connections. more specifically, this means it is
+ # only issued when the client uses the new
+ # HTTPConnection() class. backwards-compat clients
+ # will be using HTTP/1.0 and those clients may be
+ # issuing this header themselves. we should NOT issue
+ # it twice; some web servers (such as Apache) barf
+ # when they see two Host: headers
+
+ # If we need a non-standard port,include it in the
+ # header. If the request is going through a proxy,
+ # but the host of the actual URL, not the host of the
+ # proxy.
+
+ netloc = ''
+ if url.startswith('http'):
+ nil, netloc, nil, nil, nil = urlsplit(url)
+
+ if netloc:
+ try:
+ netloc_enc = netloc.encode("ascii")
+ except UnicodeEncodeError:
+ netloc_enc = netloc.encode("idna")
+ self.putheader('Host', netloc_enc)
+ else:
+ try:
+ host_enc = self.host.encode("ascii")
+ except UnicodeEncodeError:
+ host_enc = self.host.encode("idna")
+ # Wrap the IPv6 Host Header with [] (RFC 2732)
+ if host_enc.find(':') >= 0:
+ host_enc = "[" + host_enc + "]"
+ if self.port == self.default_port:
+ self.putheader('Host', host_enc)
+ else:
+ self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+ # note: we are assuming that clients will not attempt to set these
+ # headers since *this* library must deal with the
+ # consequences. this also means that when the supporting
+ # libraries are updated to recognize other forms, then this
+ # code should be changed (removed or updated).
+
+ # we only want a Content-Encoding of "identity" since we don't
+ # support encodings such as x-gzip or x-deflate.
+ if not skip_accept_encoding:
+ self.putheader('Accept-Encoding', 'identity')
+
+ # we can accept "chunked" Transfer-Encodings, but no others
+ # NOTE: no TE header implies *only* "chunked"
+ #self.putheader('TE', 'chunked')
+
+ # if TE is supplied in the header, then it must appear in a
+ # Connection header.
+ #self.putheader('Connection', 'TE')
+
+ else:
+ # For HTTP/1.0, the server will assume "not chunked"
+ pass
+
+ def putheader(self, header, *values):
+ """Send a request header line to the server.
+
+ For example: h.putheader('Accept', 'text/html')
+ """
+ if self.__state != _CS_REQ_STARTED:
+ raise CannotSendHeader()
+
+ hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+ self._output(hdr)
+
+ def endheaders(self, message_body=None):
+ """Indicate that the last header line has been sent to the server.
+
+ This method sends the request to the server. The optional
+ message_body argument can be used to pass message body
+ associated with the request. The message body will be sent in
+ the same packet as the message headers if possible. The
+ message_body should be a string.
+ """
+ if self.__state == _CS_REQ_STARTED:
+ self.__state = _CS_REQ_SENT
+ else:
+ raise CannotSendHeader()
+ self._send_output(message_body)
+
+ def request(self, method, url, body=None, headers={}):
+ """Send a complete request to the server."""
+ self._send_request(method, url, body, headers)
+
+ def _set_content_length(self, body):
+ # Set the content-length based on the body.
+ thelen = None
+ try:
+ thelen = str(len(body))
+ except TypeError, te:
+ # If this is a file-like object, try to
+ # fstat its file descriptor
+ try:
+ thelen = str(os.fstat(body.fileno()).st_size)
+ except (AttributeError, OSError):
+ # Don't send a length if this failed
+ if self.debuglevel > 0: print "Cannot stat!!"
+
+ if thelen is not None:
+ self.putheader('Content-Length', thelen)
+
+ def _send_request(self, method, url, body, headers):
+ # Honor explicitly requested Host: and Accept-Encoding: headers.
+ header_names = dict.fromkeys([k.lower() for k in headers])
+ skips = {}
+ if 'host' in header_names:
+ skips['skip_host'] = 1
+ if 'accept-encoding' in header_names:
+ skips['skip_accept_encoding'] = 1
+
+ self.putrequest(method, url, **skips)
+
+ if body and ('content-length' not in header_names):
+ self._set_content_length(body)
+ for hdr, value in headers.iteritems():
+ self.putheader(hdr, value)
+ self.endheaders(body)
+
+ def getresponse(self, buffering=False):
+ "Get the response from the server."
+
+ # if a prior response has been completed, then forget about it.
+ if self.__response and self.__response.isclosed():
+ self.__response = None
+
+ #
+ # if a prior response exists, then it must be completed (otherwise, we
+ # cannot read this response's header to determine the connection-close
+ # behavior)
+ #
+ # note: if a prior response existed, but was connection-close, then the
+ # socket and response were made independent of this HTTPConnection
+ # object since a new request requires that we open a whole new
+ # connection
+ #
+ # this means the prior response had one of two states:
+ # 1) will_close: this connection was reset and the prior socket and
+ # response operate independently
+ # 2) persistent: the response was retained and we await its
+ # isclosed() status to become true.
+ #
+ if self.__state != _CS_REQ_SENT or self.__response:
+ raise ResponseNotReady()
+
+ args = (self.sock,)
+ kwds = {"strict":self.strict, "method":self._method}
+ if self.debuglevel > 0:
+ args += (self.debuglevel,)
+ if buffering:
+ #only add this keyword if non-default, for compatibility with
+ #other response_classes.
+ kwds["buffering"] = True;
+ response = self.response_class(*args, **kwds)
+
+ try:
+ response.begin()
+ except:
+ response.close()
+ raise
+ assert response.will_close != _UNKNOWN
+ self.__state = _CS_IDLE
+
+ if response.will_close:
+ # this effectively passes the connection to the response
+ self.close()
+ else:
+ # remember this, so we can tell when it is complete
+ self.__response = response
+
+ return response
+
+
+class HTTP:
+ "Compatibility class with httplib.py from 1.5."
+
+ _http_vsn = 10
+ _http_vsn_str = 'HTTP/1.0'
+
+ debuglevel = 0
+
+ _connection_class = HTTPConnection
+
+ def __init__(self, host='', port=None, strict=None):
+ "Provide a default host, since the superclass requires one."
+
+ # some joker passed 0 explicitly, meaning default port
+ if port == 0:
+ port = None
+
+ # Note that we may pass an empty string as the host; this will throw
+ # an error when we attempt to connect. Presumably, the client code
+ # will call connect before then, with a proper host.
+ self._setup(self._connection_class(host, port, strict))
+
+ def _setup(self, conn):
+ self._conn = conn
+
+ # set up delegation to flesh out interface
+ self.send = conn.send
+ self.putrequest = conn.putrequest
+ self.putheader = conn.putheader
+ self.endheaders = conn.endheaders
+ self.set_debuglevel = conn.set_debuglevel
+
+ conn._http_vsn = self._http_vsn
+ conn._http_vsn_str = self._http_vsn_str
+
+ self.file = None
+
+ def connect(self, host=None, port=None):
+ "Accept arguments to set the host/port, since the superclass doesn't."
+
+ if host is not None:
+ self._conn._set_hostport(host, port)
+ self._conn.connect()
+
+ def getfile(self):
+ "Provide a getfile, since the superclass' does not use this concept."
+ return self.file
+
+ def getreply(self, buffering=False):
+ """Compat definition since superclass does not define it.
+
+ Returns a tuple consisting of:
+ - server status code (e.g. '200' if all goes well)
+ - server "reason" corresponding to status code
+ - any RFC822 headers in the response from the server
+ """
+ try:
+ if not buffering:
+ response = self._conn.getresponse()
+ else:
+ #only add this keyword if non-default for compatibility
+ #with other connection classes
+ response = self._conn.getresponse(buffering)
+ except BadStatusLine, e:
+ ### hmm. if getresponse() ever closes the socket on a bad request,
+ ### then we are going to have problems with self.sock
+
+ ### should we keep this behavior? do people use it?
+ # keep the socket open (as a file), and return it
+ self.file = self._conn.sock.makefile('rb', 0)
+
+ # close our socket -- we want to restart after any protocol error
+ self.close()
+
+ self.headers = None
+ return -1, e.line, None
+
+ self.headers = response.msg
+ self.file = response.fp
+ return response.status, response.reason, response.msg
+
+ def close(self):
+ self._conn.close()
+
+ # note that self.file == response.fp, which gets closed by the
+ # superclass. just clear the object ref here.
+ ### hmm. messy. if status==-1, then self.file is owned by us.
+ ### well... we aren't explicitly closing, but losing this ref will
+ ### do it
+ self.file = None
+
+try:
+ import ssl
+except ImportError:
+ pass
+else:
+ class HTTPSConnection(HTTPConnection):
+ "This class allows communication via SSL."
+
+ default_port = HTTPS_PORT
+
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ source_address=None):
+ HTTPConnection.__init__(self, host, port, strict, timeout,
+ source_address)
+ self.key_file = key_file
+ self.cert_file = cert_file
+
+ def connect(self):
+ "Connect to a host on a given (SSL) port."
+
+ sock = socket.create_connection((self.host, self.port),
+ self.timeout, self.source_address)
+ if self._tunnel_host:
+ self.sock = sock
+ self._tunnel()
+ self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+ __all__.append("HTTPSConnection")
+
+ class HTTPS(HTTP):
+ """Compatibility with 1.5 httplib interface
+
+ Python 1.5.2 did not have an HTTPS class, but it defined an
+ interface for sending http requests that is also useful for
+ https.
+ """
+
+ _connection_class = HTTPSConnection
+
+ def __init__(self, host='', port=None, key_file=None, cert_file=None,
+ strict=None):
+ # provide a default host, pass the X509 cert info
+
+ # urf. compensate for bad input.
+ if port == 0:
+ port = None
+ self._setup(self._connection_class(host, port, key_file,
+ cert_file, strict))
+
+ # we never actually use these for anything, but we keep them
+ # here for compatibility with post-1.5.2 CVS.
+ self.key_file = key_file
+ self.cert_file = cert_file
+
+
+ def FakeSocket (sock, sslobj):
+ warnings.warn("FakeSocket is deprecated, and won't be in 3.x. " +
+ "Use the result of ssl.wrap_socket() directly instead.",
+ DeprecationWarning, stacklevel=2)
+ return sslobj
+
+
+class HTTPException(Exception):
+ # Subclasses that define an __init__ must call Exception.__init__
+ # or define self.args. Otherwise, str() will fail.
+ pass
+
+class NotConnected(HTTPException):
+ pass
+
+class InvalidURL(HTTPException):
+ pass
+
+class UnknownProtocol(HTTPException):
+ def __init__(self, version):
+ self.args = version,
+ self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+ pass
+
+class UnimplementedFileMode(HTTPException):
+ pass
+
+class IncompleteRead(HTTPException):
+ def __init__(self, partial, expected=None):
+ self.args = partial,
+ self.partial = partial
+ self.expected = expected
+ def __repr__(self):
+ if self.expected is not None:
+ e = ', %i more expected' % self.expected
+ else:
+ e = ''
+ return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+ def __str__(self):
+ return repr(self)
+
+class ImproperConnectionState(HTTPException):
+ pass
+
+class CannotSendRequest(ImproperConnectionState):
+ pass
+
+class CannotSendHeader(ImproperConnectionState):
+ pass
+
+class ResponseNotReady(ImproperConnectionState):
+ pass
+
+class BadStatusLine(HTTPException):
+ def __init__(self, line):
+ if not line:
+ line = repr(line)
+ self.args = line,
+ self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+ """A limited file-like object for HTTP/0.9 responses."""
+
+ # The status-line parsing code calls readline(), which normally
+ # get the HTTP status line. For a 0.9 response, however, this is
+ # actually the first line of the body! Clients need to get a
+ # readable file object that contains that line.
+
+ def __init__(self, line, file):
+ self._line = line
+ self._file = file
+ self._line_consumed = 0
+ self._line_offset = 0
+ self._line_left = len(line)
+
+ def __getattr__(self, attr):
+ return getattr(self._file, attr)
+
+ def _done(self):
+ # called when the last byte is read from the line. After the
+ # call, all read methods are delegated to the underlying file
+ # object.
+ self._line_consumed = 1
+ self.read = self._file.read
+ self.readline = self._file.readline
+ self.readlines = self._file.readlines
+
+ def read(self, amt=None):
+ if self._line_consumed:
+ return self._file.read(amt)
+ assert self._line_left
+ if amt is None or amt > self._line_left:
+ s = self._line[self._line_offset:]
+ self._done()
+ if amt is None:
+ return s + self._file.read()
+ else:
+ return s + self._file.read(amt - len(s))
+ else:
+ assert amt <= self._line_left
+ i = self._line_offset
+ j = i + amt
+ s = self._line[i:j]
+ self._line_offset = j
+ self._line_left -= amt
+ if self._line_left == 0:
+ self._done()
+ return s
+
+ def readline(self):
+ if self._line_consumed:
+ return self._file.readline()
+ assert self._line_left
+ s = self._line[self._line_offset:]
+ self._done()
+ return s
+
+ def readlines(self, size=None):
+ if self._line_consumed:
+ return self._file.readlines(size)
+ assert self._line_left
+ L = [self._line[self._line_offset:]]
+ self._done()
+ if size is None:
+ return L + self._file.readlines()
+ else:
+ return L + self._file.readlines(size)
+
+def test():
+ """Test this module.
+
+ A hodge podge of tests collected here, because they have too many
+ external dependencies for the regular test suite.
+ """
+
+ import sys
+ import getopt
+ opts, args = getopt.getopt(sys.argv[1:], 'd')
+ dl = 0
+ for o, a in opts:
+ if o == '-d': dl = dl + 1
+ host = 'www.python.org'
+ selector = '/'
+ if args[0:]: host = args[0]
+ if args[1:]: selector = args[1]
+ h = HTTP()
+ h.set_debuglevel(dl)
+ h.connect(host)
+ h.putrequest('GET', selector)
+ h.endheaders()
+ status, reason, headers = h.getreply()
+ print 'status =', status
+ print 'reason =', reason
+ print "read", len(h.getfile().read())
+ print
+ if headers:
+ for header in headers.headers: print header.strip()
+ print
+
+ # minimal test that code to extract host from url works
+ class HTTP11(HTTP):
+ _http_vsn = 11
+ _http_vsn_str = 'HTTP/1.1'
+
+ h = HTTP11('www.python.org')
+ h.putrequest('GET', 'http://www.python.org/~jeremy/')
+ h.endheaders()
+ h.getreply()
+ h.close()
+
+ try:
+ import ssl
+ except ImportError:
+ pass
+ else:
+
+ for host, selector in (('sourceforge.net', '/projects/python'),
+ ):
+ print "https://%s%s" % (host, selector)
+ hs = HTTPS()
+ hs.set_debuglevel(dl)
+ hs.connect(host)
+ hs.putrequest('GET', selector)
+ hs.endheaders()
+ status, reason, headers = hs.getreply()
+ print 'status =', status
+ print 'reason =', reason
+ print "read", len(hs.getfile().read())
+ print
+ if headers:
+ for header in headers.headers: print header.strip()
+ print
+
+if __name__ == '__main__':
+ test()
diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
"""
import re
-try:
- from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
- c_encode_basestring_ascii = None
-try:
- from _json import make_encoder as c_make_encoder
-except ImportError:
- c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
'\t': '\\t',
}
for i in range(0x20):
- ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
- #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+ ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
# Assume this produces an infinity on all machines (probably not guaranteed)
INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
"""
def replace(match):
return ESCAPE_DCT[match.group(0)]
- return '"' + ESCAPE.sub(replace, s) + '"'
+ return ESCAPE.sub(replace, s)
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
"""Return an ASCII-only JSON representation of a Python string
"""
@@ -53,20 +44,18 @@
except KeyError:
n = ord(s)
if n < 0x10000:
- return '\\u{0:04x}'.format(n)
- #return '\\u%04x' % (n,)
+ return '\\u%04x' % (n,)
else:
# surrogate pair
n -= 0x10000
s1 = 0xd800 | ((n >> 10) & 0x3ff)
s2 = 0xdc00 | (n & 0x3ff)
- return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
- #return '\\u%04x\\u%04x' % (s1, s2)
- return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
- c_encode_basestring_ascii or py_encode_basestring_ascii)
+ return '\\u%04x\\u%04x' % (s1, s2)
+ if ESCAPE_ASCII.search(s):
+ return str(ESCAPE_ASCII.sub(replace, s))
+ return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
class JSONEncoder(object):
"""Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
self.skipkeys = skipkeys
self.ensure_ascii = ensure_ascii
+ if ensure_ascii:
+ self.encoder = encode_basestring_ascii
+ else:
+ self.encoder = encode_basestring
+ if encoding != 'utf-8':
+ orig_encoder = self.encoder
+ def encoder(o):
+ if isinstance(o, str):
+ o = o.decode(encoding)
+ return orig_encoder(o)
+ self.encoder = encoder
self.check_circular = check_circular
self.allow_nan = allow_nan
self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
'{"foo": ["bar", "baz"]}'
"""
- # This is for extremely simple cases and benchmarks.
+ if self.check_circular:
+ markers = {}
+ else:
+ markers = None
+ if self.ensure_ascii:
+ builder = StringBuilder()
+ else:
+ builder = UnicodeBuilder()
+ self._encode(o, markers, builder, 0)
+ return builder.build()
+
+ def _emit_indent(self, builder, _current_indent_level):
+ if self.indent is not None:
+ _current_indent_level += 1
+ newline_indent = '\n' + (' ' * (self.indent *
+ _current_indent_level))
+ separator = self.item_separator + newline_indent
+ builder.append(newline_indent)
+ else:
+ separator = self.item_separator
+ return separator, _current_indent_level
+
+ def _emit_unindent(self, builder, _current_indent_level):
+ if self.indent is not None:
+ builder.append('\n')
+ builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+ def _encode(self, o, markers, builder, _current_indent_level):
if isinstance(o, basestring):
- if isinstance(o, str):
- _encoding = self.encoding
- if (_encoding is not None
- and not (_encoding == 'utf-8')):
- o = o.decode(_encoding)
- if self.ensure_ascii:
- return encode_basestring_ascii(o)
+ builder.append('"')
+ builder.append(self.encoder(o))
+ builder.append('"')
+ elif o is None:
+ builder.append('null')
+ elif o is True:
+ builder.append('true')
+ elif o is False:
+ builder.append('false')
+ elif isinstance(o, (int, long)):
+ builder.append(str(o))
+ elif isinstance(o, float):
+ builder.append(self._floatstr(o))
+ elif isinstance(o, (list, tuple)):
+ if not o:
+ builder.append('[]')
+ return
+ self._encode_list(o, markers, builder, _current_indent_level)
+ elif isinstance(o, dict):
+ if not o:
+ builder.append('{}')
+ return
+ self._encode_dict(o, markers, builder, _current_indent_level)
+ else:
+ self._mark_markers(markers, o)
+ res = self.default(o)
+ self._encode(res, markers, builder, _current_indent_level)
+ self._remove_markers(markers, o)
+ return res
+
+ def _encode_list(self, l, markers, builder, _current_indent_level):
+ self._mark_markers(markers, l)
+ builder.append('[')
+ first = True
+ separator, _current_indent_level = self._emit_indent(builder,
+ _current_indent_level)
+ for elem in l:
+ if first:
+ first = False
else:
- return encode_basestring(o)
- # This doesn't pass the iterator directly to ''.join() because the
- # exceptions aren't as detailed. The list call should be roughly
- # equivalent to the PySequence_Fast that ''.join() would do.
- chunks = self.iterencode(o, _one_shot=True)
- if not isinstance(chunks, (list, tuple)):
- chunks = list(chunks)
- return ''.join(chunks)
+ builder.append(separator)
+ self._encode(elem, markers, builder, _current_indent_level)
+ del elem # XXX grumble
+ self._emit_unindent(builder, _current_indent_level)
+ builder.append(']')
+ self._remove_markers(markers, l)
+
+ def _encode_dict(self, d, markers, builder, _current_indent_level):
+ self._mark_markers(markers, d)
+ first = True
+ builder.append('{')
+ separator, _current_indent_level = self._emit_indent(builder,
+ _current_indent_level)
+ if self.sort_keys:
+ items = sorted(d.items(), key=lambda kv: kv[0])
+ else:
+ items = d.iteritems()
+
+ for key, v in items:
+ if first:
+ first = False
+ else:
+ builder.append(separator)
+ if isinstance(key, basestring):
+ pass
+ # JavaScript is weakly typed for these, so it makes sense to
+ # also allow them. Many encoders seem to do something like this.
+ elif isinstance(key, float):
+ key = self._floatstr(key)
+ elif key is True:
+ key = 'true'
+ elif key is False:
+ key = 'false'
+ elif key is None:
+ key = 'null'
+ elif isinstance(key, (int, long)):
+ key = str(key)
+ elif self.skipkeys:
+ continue
+ else:
+ raise TypeError("key " + repr(key) + " is not a string")
+ builder.append('"')
+ builder.append(self.encoder(key))
+ builder.append('"')
+ builder.append(self.key_separator)
+ self._encode(v, markers, builder, _current_indent_level)
+ del key
+ del v # XXX grumble
+ self._emit_unindent(builder, _current_indent_level)
+ builder.append('}')
+ self._remove_markers(markers, d)
def iterencode(self, o, _one_shot=False):
"""Encode the given object and yield each string
@@ -217,86 +319,54 @@
markers = {}
else:
markers = None
- if self.ensure_ascii:
- _encoder = encode_basestring_ascii
+ return self._iterencode(o, markers, 0)
+
+ def _floatstr(self, o):
+ # Check for specials. Note that this type of test is processor
+ # and/or platform-specific, so do tests which don't depend on the
+ # internals.
+
+ if o != o:
+ text = 'NaN'
+ elif o == INFINITY:
+ text = 'Infinity'
+ elif o == -INFINITY:
+ text = '-Infinity'
else:
- _encoder = encode_basestring
- if self.encoding != 'utf-8':
- def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
- if isinstance(o, str):
- o = o.decode(_encoding)
- return _orig_encoder(o)
+ return FLOAT_REPR(o)
- def floatstr(o, allow_nan=self.allow_nan,
- _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
- # Check for specials. Note that this type of test is processor
- # and/or platform-specific, so do tests which don't depend on the
- # internals.
+ if not self.allow_nan:
+ raise ValueError(
+ "Out of range float values are not JSON compliant: " +
+ repr(o))
- if o != o:
- text = 'NaN'
- elif o == _inf:
- text = 'Infinity'
- elif o == _neginf:
- text = '-Infinity'
- else:
- return _repr(o)
+ return text
- if not allow_nan:
- raise ValueError(
- "Out of range float values are not JSON compliant: " +
- repr(o))
+ def _mark_markers(self, markers, o):
+ if markers is not None:
+ if id(o) in markers:
+ raise ValueError("Circular reference detected")
+ markers[id(o)] = None
- return text
+ def _remove_markers(self, markers, o):
+ if markers is not None:
+ del markers[id(o)]
-
- if (_one_shot and c_make_encoder is not None
- and not self.indent and not self.sort_keys):
- _iterencode = c_make_encoder(
- markers, self.default, _encoder, self.indent,
- self.key_separator, self.item_separator, self.sort_keys,
- self.skipkeys, self.allow_nan)
- else:
- _iterencode = _make_iterencode(
- markers, self.default, _encoder, self.indent, floatstr,
- self.key_separator, self.item_separator, self.sort_keys,
- self.skipkeys, _one_shot)
- return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
- _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
- ## HACK: hand-optimized bytecode; turn globals into locals
- ValueError=ValueError,
- basestring=basestring,
- dict=dict,
- float=float,
- id=id,
- int=int,
- isinstance=isinstance,
- list=list,
- long=long,
- str=str,
- tuple=tuple,
- ):
-
- def _iterencode_list(lst, _current_indent_level):
+ def _iterencode_list(self, lst, markers, _current_indent_level):
if not lst:
yield '[]'
return
- if markers is not None:
- markerid = id(lst)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = lst
+ self._mark_markers(markers, lst)
buf = '['
- if _indent is not None:
+ if self.indent is not None:
_current_indent_level += 1
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- separator = _item_separator + newline_indent
+ newline_indent = '\n' + (' ' * (self.indent *
+ _current_indent_level))
+ separator = self.item_separator + newline_indent
buf += newline_indent
else:
newline_indent = None
- separator = _item_separator
+ separator = self.item_separator
first = True
for value in lst:
if first:
@@ -304,7 +374,7 @@
else:
buf = separator
if isinstance(value, basestring):
- yield buf + _encoder(value)
+ yield buf + '"' + self.encoder(value) + '"'
elif value is None:
yield buf + 'null'
elif value is True:
@@ -314,44 +384,43 @@
elif isinstance(value, (int, long)):
yield buf + str(value)
elif isinstance(value, float):
- yield buf + _floatstr(value)
+ yield buf + self._floatstr(value)
else:
yield buf
if isinstance(value, (list, tuple)):
- chunks = _iterencode_list(value, _current_indent_level)
+ chunks = self._iterencode_list(value, markers,
+ _current_indent_level)
elif isinstance(value, dict):
- chunks = _iterencode_dict(value, _current_indent_level)
+ chunks = self._iterencode_dict(value, markers,
+ _current_indent_level)
else:
- chunks = _iterencode(value, _current_indent_level)
+ chunks = self._iterencode(value, markers,
+ _current_indent_level)
for chunk in chunks:
yield chunk
if newline_indent is not None:
_current_indent_level -= 1
- yield '\n' + (' ' * (_indent * _current_indent_level))
+ yield '\n' + (' ' * (self.indent * _current_indent_level))
yield ']'
- if markers is not None:
- del markers[markerid]
+ self._remove_markers(markers, lst)
- def _iterencode_dict(dct, _current_indent_level):
+ def _iterencode_dict(self, dct, markers, _current_indent_level):
if not dct:
yield '{}'
return
- if markers is not None:
- markerid = id(dct)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = dct
+ self._mark_markers(markers, dct)
yield '{'
- if _indent is not None:
+ if self.indent is not None:
_current_indent_level += 1
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- item_separator = _item_separator + newline_indent
+ newline_indent = '\n' + (' ' * (self.indent *
+ _current_indent_level))
+ item_separator = self.item_separator + newline_indent
yield newline_indent
else:
newline_indent = None
- item_separator = _item_separator
+ item_separator = self.item_separator
first = True
- if _sort_keys:
+ if self.sort_keys:
items = sorted(dct.items(), key=lambda kv: kv[0])
else:
items = dct.iteritems()
@@ -361,7 +430,7 @@
# JavaScript is weakly typed for these, so it makes sense to
# also allow them. Many encoders seem to do something like this.
elif isinstance(key, float):
- key = _floatstr(key)
+ key = self._floatstr(key)
elif key is True:
key = 'true'
elif key is False:
@@ -370,7 +439,7 @@
key = 'null'
elif isinstance(key, (int, long)):
key = str(key)
- elif _skipkeys:
+ elif self.skipkeys:
continue
else:
raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
first = False
else:
yield item_separator
- yield _encoder(key)
- yield _key_separator
+ yield '"' + self.encoder(key) + '"'
+ yield self.key_separator
if isinstance(value, basestring):
- yield _encoder(value)
+ yield '"' + self.encoder(value) + '"'
elif value is None:
yield 'null'
elif value is True:
@@ -391,26 +460,28 @@
elif isinstance(value, (int, long)):
yield str(value)
elif isinstance(value, float):
- yield _floatstr(value)
+ yield self._floatstr(value)
else:
if isinstance(value, (list, tuple)):
- chunks = _iterencode_list(value, _current_indent_level)
+ chunks = self._iterencode_list(value, markers,
+ _current_indent_level)
elif isinstance(value, dict):
- chunks = _iterencode_dict(value, _current_indent_level)
+ chunks = self._iterencode_dict(value, markers,
+ _current_indent_level)
else:
- chunks = _iterencode(value, _current_indent_level)
+ chunks = self._iterencode(value, markers,
+ _current_indent_level)
for chunk in chunks:
yield chunk
if newline_indent is not None:
_current_indent_level -= 1
- yield '\n' + (' ' * (_indent * _current_indent_level))
+ yield '\n' + (' ' * (self.indent * _current_indent_level))
yield '}'
- if markers is not None:
- del markers[markerid]
+ self._remove_markers(markers, dct)
- def _iterencode(o, _current_indent_level):
+ def _iterencode(self, o, markers, _current_indent_level):
if isinstance(o, basestring):
- yield _encoder(o)
+ yield '"' + self.encoder(o) + '"'
elif o is None:
yield 'null'
elif o is True:
@@ -420,23 +491,19 @@
elif isinstance(o, (int, long)):
yield str(o)
elif isinstance(o, float):
- yield _floatstr(o)
+ yield self._floatstr(o)
elif isinstance(o, (list, tuple)):
- for chunk in _iterencode_list(o, _current_indent_level):
+ for chunk in self._iterencode_list(o, markers,
+ _current_indent_level):
yield chunk
elif isinstance(o, dict):
- for chunk in _iterencode_dict(o, _current_indent_level):
+ for chunk in self._iterencode_dict(o, markers,
+ _current_indent_level):
yield chunk
else:
- if markers is not None:
- markerid = id(o)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = o
- o = _default(o)
- for chunk in _iterencode(o, _current_indent_level):
+ self._mark_markers(markers, o)
+ obj = self.default(o)
+ for chunk in self._iterencode(obj, markers,
+ _current_indent_level):
yield chunk
- if markers is not None:
- del markers[markerid]
-
- return _iterencode
+ self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
# Issue 10038.
self.assertEqual(type(json.loads('"foo"')), unicode)
+
+ def test_encode_not_utf_8(self):
+ self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+ '"\\u0105\\u0107"')
+ self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+ '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/opcode.py b/lib-python/modified-2.7/opcode.py
--- a/lib-python/modified-2.7/opcode.py
+++ b/lib-python/modified-2.7/opcode.py
@@ -189,7 +189,6 @@
def_op('MAP_ADD', 147)
# pypy modification, experimental bytecode
-def_op('CALL_LIKELY_BUILTIN', 200) # #args + (#kwargs << 8)
def_op('LOOKUP_METHOD', 201) # Index in name list
hasname.append(201)
def_op('CALL_METHOD', 202) # #args not including 'self'
diff --git a/lib-python/modified-2.7/pickle.py b/lib-python/modified-2.7/pickle.py
--- a/lib-python/modified-2.7/pickle.py
+++ b/lib-python/modified-2.7/pickle.py
@@ -168,7 +168,7 @@
# Pickling machinery
-class Pickler:
+class Pickler(object):
def __init__(self, file, protocol=None):
"""This takes a file-like object for writing a pickle data stream.
@@ -873,7 +873,7 @@
# Unpickling machinery
-class Unpickler:
+class Unpickler(object):
def __init__(self, file):
"""This takes a file-like object for reading a pickle data stream.
diff --git a/lib-python/2.7/pkgutil.py b/lib-python/modified-2.7/pkgutil.py
copy from lib-python/2.7/pkgutil.py
copy to lib-python/modified-2.7/pkgutil.py
--- a/lib-python/2.7/pkgutil.py
+++ b/lib-python/modified-2.7/pkgutil.py
@@ -244,7 +244,8 @@
return mod
def get_data(self, pathname):
- return open(pathname, "rb").read()
+ with open(pathname, "rb") as f:
+ return f.read()
def _reopen(self):
if self.file and self.file.closed:
diff --git a/lib-python/modified-2.7/site.py b/lib-python/modified-2.7/site.py
--- a/lib-python/modified-2.7/site.py
+++ b/lib-python/modified-2.7/site.py
@@ -454,10 +454,10 @@
__builtin__.copyright = _Printer("copyright", sys.copyright)
__builtin__.credits = _Printer(
"credits",
- "PyPy is maintained by the PyPy developers: http://codespeak.net/pypy")
+ "PyPy is maintained by the PyPy developers: http://pypy.org/")
__builtin__.license = _Printer(
"license",
- "See http://codespeak.net/svn/pypy/dist/LICENSE")
+ "See https://bitbucket.org/pypy/pypy/src/default/LICENSE")
diff --git a/lib-python/modified-2.7/sqlite3/test/regression.py b/lib-python/modified-2.7/sqlite3/test/regression.py
--- a/lib-python/modified-2.7/sqlite3/test/regression.py
+++ b/lib-python/modified-2.7/sqlite3/test/regression.py
@@ -274,6 +274,18 @@
cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
self.assertEqual(cur.description, None)
+ def CheckStatementCache(self):
+ cur = self.con.cursor()
+ cur.execute("CREATE TABLE foo (id INTEGER)")
+ values = [(i,) for i in xrange(5)]
+ cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+ cur.execute("SELECT id FROM foo")
+ self.assertEqual(list(cur), values)
+ self.con.commit()
+ cur.execute("SELECT id FROM foo")
+ self.assertEqual(list(cur), values)
+
def suite():
regression_suite = unittest.makeSuite(RegressionTests, "Check")
return unittest.TestSuite((regression_suite,))
diff --git a/lib-python/modified-2.7/ssl.py b/lib-python/modified-2.7/ssl.py
--- a/lib-python/modified-2.7/ssl.py
+++ b/lib-python/modified-2.7/ssl.py
@@ -62,7 +62,6 @@
from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
from _ssl import SSLError
from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
from _ssl import RAND_status, RAND_egd, RAND_add
from _ssl import \
SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
SSL_ERROR_WANT_CONNECT, \
SSL_ERROR_EOF, \
SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+ PROTOCOL_TLSv1: "TLSv1",
+ PROTOCOL_SSLv23: "SSLv23",
+ PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+ from _ssl import PROTOCOL_SSLv2
+except ImportError:
+ pass
+else:
+ _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
from socket import socket, _fileobject, error as socket_error
from socket import getnameinfo as _getnameinfo
@@ -400,16 +411,7 @@
return DER_cert_to_PEM_cert(dercert)
def get_protocol_name(protocol_code):
- if protocol_code == PROTOCOL_TLSv1:
- return "TLSv1"
- elif protocol_code == PROTOCOL_SSLv23:
- return "SSLv23"
- elif protocol_code == PROTOCOL_SSLv2:
- return "SSLv2"
- elif protocol_code == PROTOCOL_SSLv3:
- return "SSLv3"
- else:
- return "<unknown>"
+ return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
# a replacement for the old socket.ssl function
diff --git a/lib-python/modified-2.7/test/regrtest.py b/lib-python/modified-2.7/test/regrtest.py
--- a/lib-python/modified-2.7/test/regrtest.py
+++ b/lib-python/modified-2.7/test/regrtest.py
@@ -1403,7 +1403,26 @@
test_zipimport
test_zlib
""",
- 'openbsd3':
+ 'openbsd4':
+ """
+ test_ascii_formatd
+ test_bsddb
+ test_bsddb3
+ test_ctypes
+ test_dl
+ test_epoll
+ test_gdbm
+ test_locale
+ test_normalization
+ test_ossaudiodev
+ test_pep277
+ test_tcl
+ test_tk
+ test_ttk_guionly
+ test_ttk_textonly
+ test_multiprocessing
+ """,
+ 'openbsd5':
"""
test_ascii_formatd
test_bsddb
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
)
b = array.array(self.badtypecode())
- self.assertRaises(TypeError, "a + b")
-
- self.assertRaises(TypeError, "a + 'bad'")
+ with self.assertRaises(TypeError):
+ a + b
+ with self.assertRaises(TypeError):
+ a + 'bad'
def test_iadd(self):
a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
)
b = array.array(self.badtypecode())
- self.assertRaises(TypeError, "a += b")
-
- self.assertRaises(TypeError, "a += 'bad'")
+ with self.assertRaises(TypeError):
+ a += b
+ with self.assertRaises(TypeError):
+ a += 'bad'
def test_mul(self):
a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
array.array(self.typecode)
)
- self.assertRaises(TypeError, "a * 'bad'")
+ with self.assertRaises(TypeError):
+ a * 'bad'
def test_imul(self):
a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
a *= -1
self.assertEqual(a, array.array(self.typecode))
- self.assertRaises(TypeError, "a *= 'bad'")
+ with self.assertRaises(TypeError):
+ a *= 'bad'
def test_getitem(self):
a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_bz2.py b/lib-python/modified-2.7/test/test_bz2.py
--- a/lib-python/modified-2.7/test/test_bz2.py
+++ b/lib-python/modified-2.7/test/test_bz2.py
@@ -50,6 +50,7 @@
self.filename = TESTFN
def tearDown(self):
+ test_support.gc_collect()
if os.path.isfile(self.filename):
os.unlink(self.filename)
diff --git a/lib-python/modified-2.7/test/test_codecs.py b/lib-python/modified-2.7/test/test_codecs.py
deleted file mode 100644
--- a/lib-python/modified-2.7/test/test_codecs.py
+++ /dev/null
@@ -1,1615 +0,0 @@
-from test import test_support
-import unittest
-import codecs
-import sys, StringIO, _testcapi
-
-class Queue(object):
- """
- queue: write bytes at one end, read bytes from the other end
- """
- def __init__(self):
- self._buffer = ""
-
- def write(self, chars):
- self._buffer += chars
-
- def read(self, size=-1):
- if size<0:
- s = self._buffer
- self._buffer = ""
- return s
- else:
- s = self._buffer[:size]
- self._buffer = self._buffer[size:]
- return s
-
-class ReadTest(unittest.TestCase):
- def check_partial(self, input, partialresults):
- # get a StreamReader for the encoding and feed the bytestring version
- # of input to the reader byte by byte. Read everything available from
- # the StreamReader and check that the results equal the appropriate
- # entries from partialresults.
- q = Queue()
- r = codecs.getreader(self.encoding)(q)
- result = u""
- for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
- q.write(c)
- result += r.read()
- self.assertEqual(result, partialresult)
- # check that there's nothing left in the buffers
- self.assertEqual(r.read(), u"")
- self.assertEqual(r.bytebuffer, "")
- self.assertEqual(r.charbuffer, u"")
-
- # do the check again, this time using a incremental decoder
- d = codecs.getincrementaldecoder(self.encoding)()
- result = u""
- for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
- result += d.decode(c)
- self.assertEqual(result, partialresult)
- # check that there's nothing left in the buffers
- self.assertEqual(d.decode("", True), u"")
- self.assertEqual(d.buffer, "")
-
- # Check whether the reset method works properly
- d.reset()
- result = u""
- for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
- result += d.decode(c)
- self.assertEqual(result, partialresult)
- # check that there's nothing left in the buffers
- self.assertEqual(d.decode("", True), u"")
- self.assertEqual(d.buffer, "")
-
- # check iterdecode()
- encoded = input.encode(self.encoding)
- self.assertEqual(
- input,
- u"".join(codecs.iterdecode(encoded, self.encoding))
- )
-
- def test_readline(self):
- def getreader(input):
- stream = StringIO.StringIO(input.encode(self.encoding))
- return codecs.getreader(self.encoding)(stream)
-
- def readalllines(input, keepends=True, size=None):
- reader = getreader(input)
- lines = []
- while True:
- line = reader.readline(size=size, keepends=keepends)
- if not line:
- break
- lines.append(line)
- return "|".join(lines)
-
- s = u"foo\nbar\r\nbaz\rspam\u2028eggs"
- sexpected = u"foo\n|bar\r\n|baz\r|spam\u2028|eggs"
- sexpectednoends = u"foo|bar|baz|spam|eggs"
- self.assertEqual(readalllines(s, True), sexpected)
- self.assertEqual(readalllines(s, False), sexpectednoends)
- self.assertEqual(readalllines(s, True, 10), sexpected)
- self.assertEqual(readalllines(s, False, 10), sexpectednoends)
-
- # Test long lines (multiple calls to read() in readline())
- vw = []
- vwo = []
- for (i, lineend) in enumerate(u"\n \r\n \r \u2028".split()):
- vw.append((i*200)*u"\3042" + lineend)
- vwo.append((i*200)*u"\3042")
- self.assertEqual(readalllines("".join(vw), True), "".join(vw))
- self.assertEqual(readalllines("".join(vw), False),"".join(vwo))
-
- # Test lines where the first read might end with \r, so the
- # reader has to look ahead whether this is a lone \r or a \r\n
- for size in xrange(80):
- for lineend in u"\n \r\n \r \u2028".split():
- s = 10*(size*u"a" + lineend + u"xxx\n")
- reader = getreader(s)
- for i in xrange(10):
- self.assertEqual(
- reader.readline(keepends=True),
- size*u"a" + lineend,
- )
- reader = getreader(s)
- for i in xrange(10):
- self.assertEqual(
- reader.readline(keepends=False),
- size*u"a",
- )
-
- def test_bug1175396(self):
- s = [
- '<%!--===================================================\r\n',
- ' BLOG index page: show recent articles,\r\n',
- ' today\'s articles, or articles of a specific date.\r\n',
- '========================================================--%>\r\n',
- '<%@inputencoding="ISO-8859-1"%>\r\n',
- '<%@pagetemplate=TEMPLATE.y%>\r\n',
- '<%@import=import frog.util, frog%>\r\n',
- '<%@import=import frog.objects%>\r\n',
- '<%@import=from frog.storageerrors import StorageError%>\r\n',
- '<%\r\n',
- '\r\n',
- 'import logging\r\n',
- 'log=logging.getLogger("Snakelets.logger")\r\n',
- '\r\n',
- '\r\n',
- 'user=self.SessionCtx.user\r\n',
- 'storageEngine=self.SessionCtx.storageEngine\r\n',
- '\r\n',
- '\r\n',
- 'def readArticlesFromDate(date, count=None):\r\n',
- ' entryids=storageEngine.listBlogEntries(date)\r\n',
- ' entryids.reverse() # descending\r\n',
- ' if count:\r\n',
- ' entryids=entryids[:count]\r\n',
- ' try:\r\n',
- ' return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n',
- ' except StorageError,x:\r\n',
- ' log.error("Error loading articles: "+str(x))\r\n',
- ' self.abort("cannot load articles")\r\n',
- '\r\n',
- 'showdate=None\r\n',
- '\r\n',
- 'arg=self.Request.getArg()\r\n',
- 'if arg=="today":\r\n',
- ' #-------------------- TODAY\'S ARTICLES\r\n',
- ' self.write("<h2>Today\'s articles</h2>")\r\n',
- ' showdate = frog.util.isodatestr() \r\n',
- ' entries = readArticlesFromDate(showdate)\r\n',
- 'elif arg=="active":\r\n',
- ' #-------------------- ACTIVE ARTICLES redirect\r\n',
- ' self.Yredirect("active.y")\r\n',
- 'elif arg=="login":\r\n',
- ' #-------------------- LOGIN PAGE redirect\r\n',
- ' self.Yredirect("login.y")\r\n',
- 'elif arg=="date":\r\n',
- ' #-------------------- ARTICLES OF A SPECIFIC DATE\r\n',
- ' showdate = self.Request.getParameter("date")\r\n',
- ' self.write("<h2>Articles written on %s</h2>"% frog.util.mediumdatestr(showdate))\r\n',
- ' entries = readArticlesFromDate(showdate)\r\n',
- 'else:\r\n',
- ' #-------------------- RECENT ARTICLES\r\n',
- ' self.write("<h2>Recent articles</h2>")\r\n',
- ' dates=storageEngine.listBlogEntryDates()\r\n',
- ' if dates:\r\n',
- ' entries=[]\r\n',
- ' SHOWAMOUNT=10\r\n',
- ' for showdate in dates:\r\n',
- ' entries.extend( readArticlesFromDate(showdate, SHOWAMOUNT-len(entries)) )\r\n',
- ' if len(entries)>=SHOWAMOUNT:\r\n',
- ' break\r\n',
- ' \r\n',
- ]
- stream = StringIO.StringIO("".join(s).encode(self.encoding))
- reader = codecs.getreader(self.encoding)(stream)
- for (i, line) in enumerate(reader):
- self.assertEqual(line, s[i])
-
- def test_readlinequeue(self):
- q = Queue()
- writer = codecs.getwriter(self.encoding)(q)
- reader = codecs.getreader(self.encoding)(q)
-
- # No lineends
- writer.write(u"foo\r")
- self.assertEqual(reader.readline(keepends=False), u"foo")
- writer.write(u"\nbar\r")
- self.assertEqual(reader.readline(keepends=False), u"")
- self.assertEqual(reader.readline(keepends=False), u"bar")
- writer.write(u"baz")
- self.assertEqual(reader.readline(keepends=False), u"baz")
- self.assertEqual(reader.readline(keepends=False), u"")
-
- # Lineends
- writer.write(u"foo\r")
- self.assertEqual(reader.readline(keepends=True), u"foo\r")
- writer.write(u"\nbar\r")
- self.assertEqual(reader.readline(keepends=True), u"\n")
- self.assertEqual(reader.readline(keepends=True), u"bar\r")
- writer.write(u"baz")
- self.assertEqual(reader.readline(keepends=True), u"baz")
- self.assertEqual(reader.readline(keepends=True), u"")
- writer.write(u"foo\r\n")
- self.assertEqual(reader.readline(keepends=True), u"foo\r\n")
-
- def test_bug1098990_a(self):
- s1 = u"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
- s2 = u"offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
- s3 = u"next line.\r\n"
-
- s = (s1+s2+s3).encode(self.encoding)
- stream = StringIO.StringIO(s)
- reader = codecs.getreader(self.encoding)(stream)
- self.assertEqual(reader.readline(), s1)
- self.assertEqual(reader.readline(), s2)
- self.assertEqual(reader.readline(), s3)
- self.assertEqual(reader.readline(), u"")
-
- def test_bug1098990_b(self):
- s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
- s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
- s3 = u"stillokay:bbbbxx\r\n"
- s4 = u"broken!!!!badbad\r\n"
- s5 = u"againokay.\r\n"
-
- s = (s1+s2+s3+s4+s5).encode(self.encoding)
- stream = StringIO.StringIO(s)
- reader = codecs.getreader(self.encoding)(stream)
- self.assertEqual(reader.readline(), s1)
- self.assertEqual(reader.readline(), s2)
- self.assertEqual(reader.readline(), s3)
- self.assertEqual(reader.readline(), s4)
- self.assertEqual(reader.readline(), s5)
- self.assertEqual(reader.readline(), u"")
-
-class UTF32Test(ReadTest):
- encoding = "utf-32"
-
- spamle = ('\xff\xfe\x00\x00'
- 's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00'
- 's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00')
- spambe = ('\x00\x00\xfe\xff'
- '\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m'
- '\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m')
-
- def test_only_one_bom(self):
- _,_,reader,writer = codecs.lookup(self.encoding)
- # encode some stream
- s = StringIO.StringIO()
- f = writer(s)
- f.write(u"spam")
- f.write(u"spam")
- d = s.getvalue()
- # check whether there is exactly one BOM in it
- self.assertTrue(d == self.spamle or d == self.spambe)
- # try to read it back
- s = StringIO.StringIO(d)
- f = reader(s)
- self.assertEqual(f.read(), u"spamspam")
-
- def test_badbom(self):
- s = StringIO.StringIO(4*"\xff")
- f = codecs.getreader(self.encoding)(s)
- self.assertRaises(UnicodeError, f.read)
-
- s = StringIO.StringIO(8*"\xff")
- f = codecs.getreader(self.encoding)(s)
- self.assertRaises(UnicodeError, f.read)
-
- def test_partial(self):
- self.check_partial(
- u"\x00\xff\u0100\uffff",
- [
- u"", # first byte of BOM read
- u"", # second byte of BOM read
- u"", # third byte of BOM read
- u"", # fourth byte of BOM read => byteorder known
- u"",
- u"",
- u"",
- u"\x00",
- u"\x00",
- u"\x00",
- u"\x00",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100\uffff",
- ]
- )
-
- def test_handlers(self):
- self.assertEqual((u'\ufffd', 1),
- codecs.utf_32_decode('\x01', 'replace', True))
- self.assertEqual((u'', 1),
- codecs.utf_32_decode('\x01', 'ignore', True))
-
- def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode,
- "\xff", "strict", True)
-
- def test_issue8941(self):
- # Issue #8941: insufficient result allocation when decoding into
- # surrogate pairs on UCS-2 builds.
- encoded_le = '\xff\xfe\x00\x00' + '\x00\x00\x01\x00' * 1024
- self.assertEqual(u'\U00010000' * 1024,
- codecs.utf_32_decode(encoded_le)[0])
- encoded_be = '\x00\x00\xfe\xff' + '\x00\x01\x00\x00' * 1024
- self.assertEqual(u'\U00010000' * 1024,
- codecs.utf_32_decode(encoded_be)[0])
-
-class UTF32LETest(ReadTest):
- encoding = "utf-32-le"
-
- def test_partial(self):
- self.check_partial(
- u"\x00\xff\u0100\uffff",
- [
- u"",
- u"",
- u"",
- u"\x00",
- u"\x00",
- u"\x00",
- u"\x00",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100\uffff",
- ]
- )
-
- def test_simple(self):
- self.assertEqual(u"\U00010203".encode(self.encoding), "\x03\x02\x01\x00")
-
- def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode,
- "\xff", "strict", True)
-
- def test_issue8941(self):
- # Issue #8941: insufficient result allocation when decoding into
- # surrogate pairs on UCS-2 builds.
- encoded = '\x00\x00\x01\x00' * 1024
- self.assertEqual(u'\U00010000' * 1024,
- codecs.utf_32_le_decode(encoded)[0])
-
-class UTF32BETest(ReadTest):
- encoding = "utf-32-be"
-
- def test_partial(self):
- self.check_partial(
- u"\x00\xff\u0100\uffff",
- [
- u"",
- u"",
- u"",
- u"\x00",
- u"\x00",
- u"\x00",
- u"\x00",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100\uffff",
- ]
- )
-
- def test_simple(self):
- self.assertEqual(u"\U00010203".encode(self.encoding), "\x00\x01\x02\x03")
-
- def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode,
- "\xff", "strict", True)
-
- def test_issue8941(self):
- # Issue #8941: insufficient result allocation when decoding into
- # surrogate pairs on UCS-2 builds.
- encoded = '\x00\x01\x00\x00' * 1024
- self.assertEqual(u'\U00010000' * 1024,
- codecs.utf_32_be_decode(encoded)[0])
-
-
-class UTF16Test(ReadTest):
- encoding = "utf-16"
-
- spamle = '\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00'
- spambe = '\xfe\xff\x00s\x00p\x00a\x00m\x00s\x00p\x00a\x00m'
-
- def test_only_one_bom(self):
- _,_,reader,writer = codecs.lookup(self.encoding)
- # encode some stream
- s = StringIO.StringIO()
- f = writer(s)
- f.write(u"spam")
- f.write(u"spam")
- d = s.getvalue()
- # check whether there is exactly one BOM in it
- self.assertTrue(d == self.spamle or d == self.spambe)
- # try to read it back
- s = StringIO.StringIO(d)
- f = reader(s)
- self.assertEqual(f.read(), u"spamspam")
-
- def test_badbom(self):
- s = StringIO.StringIO("\xff\xff")
- f = codecs.getreader(self.encoding)(s)
- self.assertRaises(UnicodeError, f.read)
-
- s = StringIO.StringIO("\xff\xff\xff\xff")
- f = codecs.getreader(self.encoding)(s)
- self.assertRaises(UnicodeError, f.read)
-
- def test_partial(self):
- self.check_partial(
- u"\x00\xff\u0100\uffff",
- [
- u"", # first byte of BOM read
- u"", # second byte of BOM read => byteorder known
- u"",
- u"\x00",
- u"\x00",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100\uffff",
- ]
- )
-
- def test_handlers(self):
- self.assertEqual((u'\ufffd', 1),
- codecs.utf_16_decode('\x01', 'replace', True))
- self.assertEqual((u'', 1),
- codecs.utf_16_decode('\x01', 'ignore', True))
-
- def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_16_decode, "\xff", "strict", True)
-
- def test_bug691291(self):
- # Files are always opened in binary mode, even if no binary mode was
- # specified. This means that no automatic conversion of '\n' is done
- # on reading and writing.
- s1 = u'Hello\r\nworld\r\n'
-
- s = s1.encode(self.encoding)
- try:
- with open(test_support.TESTFN, 'wb') as fp:
- fp.write(s)
- with codecs.open(test_support.TESTFN, 'U', encoding=self.encoding) as reader:
- self.assertEqual(reader.read(), s1)
- finally:
- test_support.unlink(test_support.TESTFN)
-
-class UTF16LETest(ReadTest):
- encoding = "utf-16-le"
-
- def test_partial(self):
- self.check_partial(
- u"\x00\xff\u0100\uffff",
- [
- u"",
- u"\x00",
- u"\x00",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100\uffff",
- ]
- )
-
- def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode, "\xff", "strict", True)
-
-class UTF16BETest(ReadTest):
- encoding = "utf-16-be"
-
- def test_partial(self):
- self.check_partial(
- u"\x00\xff\u0100\uffff",
- [
- u"",
- u"\x00",
- u"\x00",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100",
- u"\x00\xff\u0100\uffff",
- ]
- )
-
- def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, "\xff", "strict", True)
-
-class UTF8Test(ReadTest):
- encoding = "utf-8"
-
- def test_partial(self):
- self.check_partial(
- u"\x00\xff\u07ff\u0800\uffff",
- [
- u"\x00",
- u"\x00",
- u"\x00\xff",
- u"\x00\xff",
- u"\x00\xff\u07ff",
- u"\x00\xff\u07ff",
- u"\x00\xff\u07ff",
- u"\x00\xff\u07ff\u0800",
- u"\x00\xff\u07ff\u0800",
- u"\x00\xff\u07ff\u0800",
- u"\x00\xff\u07ff\u0800\uffff",
- ]
- )
-
-class UTF7Test(ReadTest):
- encoding = "utf-7"
-
- def test_partial(self):
- self.check_partial(
- u"a+-b",
- [
- u"a",
- u"a",
- u"a+",
- u"a+-",
- u"a+-b",
- ]
- )
-
-class UTF16ExTest(unittest.TestCase):
-
- def test_errors(self):
- self.assertRaises(UnicodeDecodeError, codecs.utf_16_ex_decode, "\xff", "strict", 0, True)
-
- def test_bad_args(self):
- self.assertRaises(TypeError, codecs.utf_16_ex_decode)
-
-class ReadBufferTest(unittest.TestCase):
-
- def test_array(self):
- import array
- self.assertEqual(
- codecs.readbuffer_encode(array.array("c", "spam")),
- ("spam", 4)
- )
-
- def test_empty(self):
- self.assertEqual(codecs.readbuffer_encode(""), ("", 0))
-
- def test_bad_args(self):
- self.assertRaises(TypeError, codecs.readbuffer_encode)
- self.assertRaises(TypeError, codecs.readbuffer_encode, 42)
-
-class CharBufferTest(unittest.TestCase):
-
- def test_string(self):
- self.assertEqual(codecs.charbuffer_encode("spam"), ("spam", 4))
-
- def test_empty(self):
- self.assertEqual(codecs.charbuffer_encode(""), ("", 0))
-
- def test_bad_args(self):
- self.assertRaises(TypeError, codecs.charbuffer_encode)
- self.assertRaises(TypeError, codecs.charbuffer_encode, 42)
-
-class UTF8SigTest(ReadTest):
- encoding = "utf-8-sig"
-
- def test_partial(self):
- self.check_partial(
- u"\ufeff\x00\xff\u07ff\u0800\uffff",
- [
- u"",
- u"",
- u"", # First BOM has been read and skipped
- u"",
- u"",
- u"\ufeff", # Second BOM has been read and emitted
- u"\ufeff\x00", # "\x00" read and emitted
- u"\ufeff\x00", # First byte of encoded u"\xff" read
- u"\ufeff\x00\xff", # Second byte of encoded u"\xff" read
- u"\ufeff\x00\xff", # First byte of encoded u"\u07ff" read
- u"\ufeff\x00\xff\u07ff", # Second byte of encoded u"\u07ff" read
- u"\ufeff\x00\xff\u07ff",
- u"\ufeff\x00\xff\u07ff",
- u"\ufeff\x00\xff\u07ff\u0800",
- u"\ufeff\x00\xff\u07ff\u0800",
- u"\ufeff\x00\xff\u07ff\u0800",
- u"\ufeff\x00\xff\u07ff\u0800\uffff",
- ]
- )
-
- def test_bug1601501(self):
- # SF bug #1601501: check that the codec works with a buffer
- unicode("\xef\xbb\xbf", "utf-8-sig")
-
- def test_bom(self):
- d = codecs.getincrementaldecoder("utf-8-sig")()
- s = u"spam"
- self.assertEqual(d.decode(s.encode("utf-8-sig")), s)
-
- def test_stream_bom(self):
- unistring = u"ABC\u00A1\u2200XYZ"
- bytestring = codecs.BOM_UTF8 + "ABC\xC2\xA1\xE2\x88\x80XYZ"
-
- reader = codecs.getreader("utf-8-sig")
- for sizehint in [None] + range(1, 11) + \
- [64, 128, 256, 512, 1024]:
- istream = reader(StringIO.StringIO(bytestring))
- ostream = StringIO.StringIO()
- while 1:
- if sizehint is not None:
- data = istream.read(sizehint)
- else:
- data = istream.read()
-
- if not data:
- break
- ostream.write(data)
-
- got = ostream.getvalue()
- self.assertEqual(got, unistring)
-
- def test_stream_bare(self):
- unistring = u"ABC\u00A1\u2200XYZ"
- bytestring = "ABC\xC2\xA1\xE2\x88\x80XYZ"
-
- reader = codecs.getreader("utf-8-sig")
- for sizehint in [None] + range(1, 11) + \
- [64, 128, 256, 512, 1024]:
- istream = reader(StringIO.StringIO(bytestring))
- ostream = StringIO.StringIO()
- while 1:
- if sizehint is not None:
- data = istream.read(sizehint)
- else:
- data = istream.read()
-
- if not data:
- break
- ostream.write(data)
-
- got = ostream.getvalue()
- self.assertEqual(got, unistring)
-
-class EscapeDecodeTest(unittest.TestCase):
- def test_empty(self):
- self.assertEqual(codecs.escape_decode(""), ("", 0))
-
-class RecodingTest(unittest.TestCase):
- def test_recoding(self):
- f = StringIO.StringIO()
- f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8")
- f2.write(u"a")
- f2.close()
- # Python used to crash on this at exit because of a refcount
- # bug in _codecsmodule.c
-
-# From RFC 3492
-punycode_testcases = [
- # A Arabic (Egyptian):
- (u"\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644"
- u"\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F",
- "egbpdaj6bu4bxfgehfvwxn"),
- # B Chinese (simplified):
- (u"\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587",
- "ihqwcrb4cv8a8dqg056pqjye"),
- # C Chinese (traditional):
- (u"\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587",
- "ihqwctvzc91f659drss3x8bo0yb"),
- # D Czech: Pro<ccaron>prost<ecaron>nemluv<iacute><ccaron>esky
- (u"\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074"
- u"\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D"
- u"\u0065\u0073\u006B\u0079",
- "Proprostnemluvesky-uyb24dma41a"),
- # E Hebrew:
- (u"\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8"
- u"\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2"
- u"\u05D1\u05E8\u05D9\u05EA",
- "4dbcagdahymbxekheh6e0a7fei0b"),
- # F Hindi (Devanagari):
- (u"\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D"
- u"\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939"
- u"\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947"
- u"\u0939\u0948\u0902",
- "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd"),
-
- #(G) Japanese (kanji and hiragana):
- (u"\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092"
- u"\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B",
- "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa"),
-
- # (H) Korean (Hangul syllables):
- (u"\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774"
- u"\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74"
- u"\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C",
- "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j"
- "psd879ccm6fea98c"),
-
- # (I) Russian (Cyrillic):
- (u"\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E"
- u"\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440"
- u"\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A"
- u"\u0438",
- "b1abfaaepdrnnbgefbaDotcwatmq2g4l"),
-
- # (J) Spanish: Porqu<eacute>nopuedensimplementehablarenEspa<ntilde>ol
- (u"\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070"
- u"\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070"
- u"\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061"
- u"\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070"
- u"\u0061\u00F1\u006F\u006C",
- "PorqunopuedensimplementehablarenEspaol-fmd56a"),
-
- # (K) Vietnamese:
- # T<adotbelow>isaoh<odotbelow>kh<ocirc>ngth<ecirchookabove>ch\
- # <ihookabove>n<oacute>iti<ecircacute>ngVi<ecircdotbelow>t
- (u"\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B"
- u"\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068"
- u"\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067"
- u"\u0056\u0069\u1EC7\u0074",
- "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g"),
-
- #(L) 3<nen>B<gumi><kinpachi><sensei>
- (u"\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F",
- "3B-ww4c5e180e575a65lsy2b"),
-
- # (M) <amuro><namie>-with-SUPER-MONKEYS
- (u"\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074"
- u"\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D"
- u"\u004F\u004E\u004B\u0045\u0059\u0053",
- "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n"),
-
- # (N) Hello-Another-Way-<sorezore><no><basho>
- (u"\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F"
- u"\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D"
- u"\u305D\u308C\u305E\u308C\u306E\u5834\u6240",
- "Hello-Another-Way--fc4qua05auwb3674vfr0b"),
-
- # (O) <hitotsu><yane><no><shita>2
- (u"\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032",
- "2-u9tlzr9756bt3uc0v"),
-
- # (P) Maji<de>Koi<suru>5<byou><mae>
- (u"\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059"
- u"\u308B\u0035\u79D2\u524D",
- "MajiKoi5-783gue6qz075azm5e"),
-
- # (Q) <pafii>de<runba>
- (u"\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0",
- "de-jg4avhby1noc0d"),
-
- # (R) <sono><supiido><de>
- (u"\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067",
- "d9juau41awczczp"),
-
- # (S) -> $1.00 <-
- (u"\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020"
- u"\u003C\u002D",
- "-> $1.00 <--")
- ]
-
-for i in punycode_testcases:
- if len(i)!=2:
- print repr(i)
-
-class PunycodeTest(unittest.TestCase):
- def test_encode(self):
- for uni, puny in punycode_testcases:
- # Need to convert both strings to lower case, since
- # some of the extended encodings use upper case, but our
- # code produces only lower case. Converting just puny to
- # lower is also insufficient, since some of the input characters
- # are upper case.
- self.assertEqual(uni.encode("punycode").lower(), puny.lower())
-
- def test_decode(self):
- for uni, puny in punycode_testcases:
- self.assertEqual(uni, puny.decode("punycode"))
-
-class UnicodeInternalTest(unittest.TestCase):
- def test_bug1251300(self):
- # Decoding with unicode_internal used to not correctly handle "code
- # points" above 0x10ffff on UCS-4 builds.
- if sys.maxunicode > 0xffff:
- ok = [
- ("\x00\x10\xff\xff", u"\U0010ffff"),
- ("\x00\x00\x01\x01", u"\U00000101"),
- ("", u""),
- ]
- not_ok = [
- "\x7f\xff\xff\xff",
- "\x80\x00\x00\x00",
- "\x81\x00\x00\x00",
- "\x00",
- "\x00\x00\x00\x00\x00",
- ]
- for internal, uni in ok:
- if sys.byteorder == "little":
- internal = "".join(reversed(internal))
- self.assertEqual(uni, internal.decode("unicode_internal"))
- for internal in not_ok:
- if sys.byteorder == "little":
- internal = "".join(reversed(internal))
- self.assertRaises(UnicodeDecodeError, internal.decode,
- "unicode_internal")
-
- def test_decode_error_attributes(self):
- if sys.maxunicode > 0xffff:
- try:
- "\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal")
- except UnicodeDecodeError, ex:
- self.assertEqual("unicode_internal", ex.encoding)
- self.assertEqual("\x00\x00\x00\x00\x00\x11\x11\x00", ex.object)
- self.assertEqual(4, ex.start)
- self.assertEqual(8, ex.end)
- else:
- self.fail()
-
- def test_decode_callback(self):
- if sys.maxunicode > 0xffff:
- codecs.register_error("UnicodeInternalTest", codecs.ignore_errors)
- decoder = codecs.getdecoder("unicode_internal")
- ab = u"ab".encode("unicode_internal")
- ignored = decoder("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]),
- "UnicodeInternalTest")
- self.assertEqual((u"ab", 12), ignored)
-
- def test_encode_length(self):
- # Issue 3739
- encoder = codecs.getencoder("unicode_internal")
- self.assertEqual(encoder(u"a")[1], 1)
- self.assertEqual(encoder(u"\xe9\u0142")[1], 2)
-
- encoder = codecs.getencoder("string-escape")
- self.assertEqual(encoder(r'\x00')[1], 4)
-
-# From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
-nameprep_tests = [
- # 3.1 Map to nothing.
- ('foo\xc2\xad\xcd\x8f\xe1\xa0\x86\xe1\xa0\x8bbar'
- '\xe2\x80\x8b\xe2\x81\xa0baz\xef\xb8\x80\xef\xb8\x88\xef'
- '\xb8\x8f\xef\xbb\xbf',
- 'foobarbaz'),
- # 3.2 Case folding ASCII U+0043 U+0041 U+0046 U+0045.
- ('CAFE',
- 'cafe'),
- # 3.3 Case folding 8bit U+00DF (german sharp s).
- # The original test case is bogus; it says \xc3\xdf
- ('\xc3\x9f',
- 'ss'),
- # 3.4 Case folding U+0130 (turkish capital I with dot).
- ('\xc4\xb0',
- 'i\xcc\x87'),
- # 3.5 Case folding multibyte U+0143 U+037A.
- ('\xc5\x83\xcd\xba',
- '\xc5\x84 \xce\xb9'),
- # 3.6 Case folding U+2121 U+33C6 U+1D7BB.
- # XXX: skip this as it fails in UCS-2 mode
- #('\xe2\x84\xa1\xe3\x8f\x86\xf0\x9d\x9e\xbb',
- # 'telc\xe2\x88\x95kg\xcf\x83'),
- (None, None),
- # 3.7 Normalization of U+006a U+030c U+00A0 U+00AA.
- ('j\xcc\x8c\xc2\xa0\xc2\xaa',
- '\xc7\xb0 a'),
- # 3.8 Case folding U+1FB7 and normalization.
- ('\xe1\xbe\xb7',
- '\xe1\xbe\xb6\xce\xb9'),
- # 3.9 Self-reverting case folding U+01F0 and normalization.
- # The original test case is bogus, it says `\xc7\xf0'
- ('\xc7\xb0',
- '\xc7\xb0'),
- # 3.10 Self-reverting case folding U+0390 and normalization.
- ('\xce\x90',
- '\xce\x90'),
- # 3.11 Self-reverting case folding U+03B0 and normalization.
- ('\xce\xb0',
- '\xce\xb0'),
- # 3.12 Self-reverting case folding U+1E96 and normalization.
- ('\xe1\xba\x96',
- '\xe1\xba\x96'),
- # 3.13 Self-reverting case folding U+1F56 and normalization.
- ('\xe1\xbd\x96',
- '\xe1\xbd\x96'),
- # 3.14 ASCII space character U+0020.
- (' ',
- ' '),
- # 3.15 Non-ASCII 8bit space character U+00A0.
- ('\xc2\xa0',
- ' '),
- # 3.16 Non-ASCII multibyte space character U+1680.
- ('\xe1\x9a\x80',
- None),
- # 3.17 Non-ASCII multibyte space character U+2000.
- ('\xe2\x80\x80',
- ' '),
- # 3.18 Zero Width Space U+200b.
- ('\xe2\x80\x8b',
- ''),
- # 3.19 Non-ASCII multibyte space character U+3000.
- ('\xe3\x80\x80',
- ' '),
- # 3.20 ASCII control characters U+0010 U+007F.
- ('\x10\x7f',
- '\x10\x7f'),
- # 3.21 Non-ASCII 8bit control character U+0085.
- ('\xc2\x85',
- None),
- # 3.22 Non-ASCII multibyte control character U+180E.
- ('\xe1\xa0\x8e',
- None),
- # 3.23 Zero Width No-Break Space U+FEFF.
- ('\xef\xbb\xbf',
- ''),
- # 3.24 Non-ASCII control character U+1D175.
- ('\xf0\x9d\x85\xb5',
- None),
- # 3.25 Plane 0 private use character U+F123.
- ('\xef\x84\xa3',
- None),
- # 3.26 Plane 15 private use character U+F1234.
- ('\xf3\xb1\x88\xb4',
- None),
- # 3.27 Plane 16 private use character U+10F234.
- ('\xf4\x8f\x88\xb4',
- None),
- # 3.28 Non-character code point U+8FFFE.
- ('\xf2\x8f\xbf\xbe',
- None),
- # 3.29 Non-character code point U+10FFFF.
- ('\xf4\x8f\xbf\xbf',
- None),
- # 3.30 Surrogate code U+DF42.
- ('\xed\xbd\x82',
- None),
- # 3.31 Non-plain text character U+FFFD.
- ('\xef\xbf\xbd',
- None),
- # 3.32 Ideographic description character U+2FF5.
- ('\xe2\xbf\xb5',
- None),
- # 3.33 Display property character U+0341.
- ('\xcd\x81',
- '\xcc\x81'),
- # 3.34 Left-to-right mark U+200E.
- ('\xe2\x80\x8e',
- None),
- # 3.35 Deprecated U+202A.
- ('\xe2\x80\xaa',
- None),
- # 3.36 Language tagging character U+E0001.
- ('\xf3\xa0\x80\x81',
- None),
- # 3.37 Language tagging character U+E0042.
- ('\xf3\xa0\x81\x82',
- None),
- # 3.38 Bidi: RandALCat character U+05BE and LCat characters.
- ('foo\xd6\xbebar',
- None),
- # 3.39 Bidi: RandALCat character U+FD50 and LCat characters.
- ('foo\xef\xb5\x90bar',
- None),
- # 3.40 Bidi: RandALCat character U+FB38 and LCat characters.
- ('foo\xef\xb9\xb6bar',
- 'foo \xd9\x8ebar'),
- # 3.41 Bidi: RandALCat without trailing RandALCat U+0627 U+0031.
- ('\xd8\xa71',
- None),
- # 3.42 Bidi: RandALCat character U+0627 U+0031 U+0628.
- ('\xd8\xa71\xd8\xa8',
- '\xd8\xa71\xd8\xa8'),
- # 3.43 Unassigned code point U+E0002.
- # Skip this test as we allow unassigned
- #('\xf3\xa0\x80\x82',
- # None),
- (None, None),
- # 3.44 Larger test (shrinking).
- # Original test case reads \xc3\xdf
- ('X\xc2\xad\xc3\x9f\xc4\xb0\xe2\x84\xa1j\xcc\x8c\xc2\xa0\xc2'
- '\xaa\xce\xb0\xe2\x80\x80',
- 'xssi\xcc\x87tel\xc7\xb0 a\xce\xb0 '),
- # 3.45 Larger test (expanding).
- # Original test case reads \xc3\x9f
- ('X\xc3\x9f\xe3\x8c\x96\xc4\xb0\xe2\x84\xa1\xe2\x92\x9f\xe3\x8c'
- '\x80',
- 'xss\xe3\x82\xad\xe3\x83\xad\xe3\x83\xa1\xe3\x83\xbc\xe3'
- '\x83\x88\xe3\x83\xabi\xcc\x87tel\x28d\x29\xe3\x82'
- '\xa2\xe3\x83\x91\xe3\x83\xbc\xe3\x83\x88')
- ]
-
-
-class NameprepTest(unittest.TestCase):
- def test_nameprep(self):
- from encodings.idna import nameprep
- for pos, (orig, prepped) in enumerate(nameprep_tests):
- if orig is None:
- # Skipped
- continue
- # The Unicode strings are given in UTF-8
- orig = unicode(orig, "utf-8")
- if prepped is None:
- # Input contains prohibited characters
- self.assertRaises(UnicodeError, nameprep, orig)
- else:
- prepped = unicode(prepped, "utf-8")
- try:
- self.assertEqual(nameprep(orig), prepped)
- except Exception,e:
- raise test_support.TestFailed("Test 3.%d: %s" % (pos+1, str(e)))
-
-class IDNACodecTest(unittest.TestCase):
- def test_builtin_decode(self):
- self.assertEqual(unicode("python.org", "idna"), u"python.org")
- self.assertEqual(unicode("python.org.", "idna"), u"python.org.")
- self.assertEqual(unicode("xn--pythn-mua.org", "idna"), u"pyth\xf6n.org")
- self.assertEqual(unicode("xn--pythn-mua.org.", "idna"), u"pyth\xf6n.org.")
-
- def test_builtin_encode(self):
- self.assertEqual(u"python.org".encode("idna"), "python.org")
- self.assertEqual("python.org.".encode("idna"), "python.org.")
- self.assertEqual(u"pyth\xf6n.org".encode("idna"), "xn--pythn-mua.org")
- self.assertEqual(u"pyth\xf6n.org.".encode("idna"), "xn--pythn-mua.org.")
-
- def test_stream(self):
- import StringIO
- r = codecs.getreader("idna")(StringIO.StringIO("abc"))
- r.read(3)
- self.assertEqual(r.read(), u"")
-
- def test_incremental_decode(self):
- self.assertEqual(
- "".join(codecs.iterdecode("python.org", "idna")),
- u"python.org"
- )
- self.assertEqual(
- "".join(codecs.iterdecode("python.org.", "idna")),
- u"python.org."
- )
- self.assertEqual(
- "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
- u"pyth\xf6n.org."
- )
- self.assertEqual(
- "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
- u"pyth\xf6n.org."
- )
-
- decoder = codecs.getincrementaldecoder("idna")()
- self.assertEqual(decoder.decode("xn--xam", ), u"")
- self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
- self.assertEqual(decoder.decode(u"rg"), u"")
- self.assertEqual(decoder.decode(u"", True), u"org")
-
- decoder.reset()
- self.assertEqual(decoder.decode("xn--xam", ), u"")
- self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
- self.assertEqual(decoder.decode("rg."), u"org.")
- self.assertEqual(decoder.decode("", True), u"")
-
- def test_incremental_encode(self):
- self.assertEqual(
- "".join(codecs.iterencode(u"python.org", "idna")),
- "python.org"
- )
- self.assertEqual(
- "".join(codecs.iterencode(u"python.org.", "idna")),
- "python.org."
- )
- self.assertEqual(
- "".join(codecs.iterencode(u"pyth\xf6n.org.", "idna")),
- "xn--pythn-mua.org."
- )
- self.assertEqual(
- "".join(codecs.iterencode(u"pyth\xf6n.org.", "idna")),
- "xn--pythn-mua.org."
- )
-
- encoder = codecs.getincrementalencoder("idna")()
- self.assertEqual(encoder.encode(u"\xe4x"), "")
- self.assertEqual(encoder.encode(u"ample.org"), "xn--xample-9ta.")
- self.assertEqual(encoder.encode(u"", True), "org")
-
- encoder.reset()
- self.assertEqual(encoder.encode(u"\xe4x"), "")
- self.assertEqual(encoder.encode(u"ample.org."), "xn--xample-9ta.org.")
- self.assertEqual(encoder.encode(u"", True), "")
-
-class CodecsModuleTest(unittest.TestCase):
-
- def test_decode(self):
- self.assertEqual(codecs.decode('\xe4\xf6\xfc', 'latin-1'),
- u'\xe4\xf6\xfc')
- self.assertRaises(TypeError, codecs.decode)
- self.assertEqual(codecs.decode('abc'), u'abc')
- self.assertRaises(UnicodeDecodeError, codecs.decode, '\xff', 'ascii')
-
- def test_encode(self):
- self.assertEqual(codecs.encode(u'\xe4\xf6\xfc', 'latin-1'),
- '\xe4\xf6\xfc')
- self.assertRaises(TypeError, codecs.encode)
- self.assertRaises(LookupError, codecs.encode, "foo", "__spam__")
- self.assertEqual(codecs.encode(u'abc'), 'abc')
- self.assertRaises(UnicodeEncodeError, codecs.encode, u'\xffff', 'ascii')
-
- def test_register(self):
- self.assertRaises(TypeError, codecs.register)
- self.assertRaises(TypeError, codecs.register, 42)
-
- def test_lookup(self):
- self.assertRaises(TypeError, codecs.lookup)
- self.assertRaises(LookupError, codecs.lookup, "__spam__")
- self.assertRaises(LookupError, codecs.lookup, " ")
-
- def test_getencoder(self):
- self.assertRaises(TypeError, codecs.getencoder)
- self.assertRaises(LookupError, codecs.getencoder, "__spam__")
-
- def test_getdecoder(self):
- self.assertRaises(TypeError, codecs.getdecoder)
- self.assertRaises(LookupError, codecs.getdecoder, "__spam__")
-
- def test_getreader(self):
- self.assertRaises(TypeError, codecs.getreader)
- self.assertRaises(LookupError, codecs.getreader, "__spam__")
-
- def test_getwriter(self):
- self.assertRaises(TypeError, codecs.getwriter)
- self.assertRaises(LookupError, codecs.getwriter, "__spam__")
-
-class StreamReaderTest(unittest.TestCase):
-
- def setUp(self):
- self.reader = codecs.getreader('utf-8')
- self.stream = StringIO.StringIO('\xed\x95\x9c\n\xea\xb8\x80')
-
- def test_readlines(self):
- f = self.reader(self.stream)
- self.assertEqual(f.readlines(), [u'\ud55c\n', u'\uae00'])
-
-class EncodedFileTest(unittest.TestCase):
-
- def test_basic(self):
- f = StringIO.StringIO('\xed\x95\x9c\n\xea\xb8\x80')
- ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8')
- self.assertEqual(ef.read(), '\\\xd5\n\x00\x00\xae')
-
- f = StringIO.StringIO()
- ef = codecs.EncodedFile(f, 'utf-8', 'latin1')
- ef.write('\xc3\xbc')
- self.assertEqual(f.getvalue(), '\xfc')
-
-class Str2StrTest(unittest.TestCase):
-
- def test_read(self):
- sin = "\x80".encode("base64_codec")
- reader = codecs.getreader("base64_codec")(StringIO.StringIO(sin))
- sout = reader.read()
- self.assertEqual(sout, "\x80")
- self.assertIsInstance(sout, str)
-
- def test_readline(self):
- sin = "\x80".encode("base64_codec")
- reader = codecs.getreader("base64_codec")(StringIO.StringIO(sin))
- sout = reader.readline()
- self.assertEqual(sout, "\x80")
- self.assertIsInstance(sout, str)
-
-all_unicode_encodings = [
- "ascii",
- "base64_codec",
- ## "big5",
- ## "big5hkscs",
- "charmap",
- "cp037",
- "cp1006",
- "cp1026",
- "cp1140",
- "cp1250",
- "cp1251",
- "cp1252",
- "cp1253",
- "cp1254",
- "cp1255",
- "cp1256",
- "cp1257",
- "cp1258",
- "cp424",
- "cp437",
- "cp500",
- "cp720",
- "cp737",
- "cp775",
- "cp850",
- "cp852",
- "cp855",
- "cp856",
- "cp857",
- "cp858",
- "cp860",
- "cp861",
- "cp862",
- "cp863",
- "cp864",
- "cp865",
- "cp866",
- "cp869",
- "cp874",
- "cp875",
- ## "cp932",
- ## "cp949",
- ## "cp950",
- ## "euc_jis_2004",
- ## "euc_jisx0213",
- ## "euc_jp",
- ## "euc_kr",
- ## "gb18030",
- ## "gb2312",
- ## "gbk",
- "hex_codec",
- "hp_roman8",
- ## "hz",
- "idna",
- ## "iso2022_jp",
- ## "iso2022_jp_1",
- ## "iso2022_jp_2",
- ## "iso2022_jp_2004",
- ## "iso2022_jp_3",
- ## "iso2022_jp_ext",
- ## "iso2022_kr",
- "iso8859_1",
- "iso8859_10",
- "iso8859_11",
- "iso8859_13",
- "iso8859_14",
- "iso8859_15",
- "iso8859_16",
- "iso8859_2",
- "iso8859_3",
- "iso8859_4",
- "iso8859_5",
- "iso8859_6",
- "iso8859_7",
- "iso8859_8",
- "iso8859_9",
- ## "johab",
- "koi8_r",
- "koi8_u",
- "latin_1",
- "mac_cyrillic",
- "mac_greek",
- "mac_iceland",
- "mac_latin2",
- "mac_roman",
- "mac_turkish",
- "palmos",
- "ptcp154",
- "punycode",
- "raw_unicode_escape",
- "rot_13",
- ## "shift_jis",
- ## "shift_jis_2004",
- ## "shift_jisx0213",
- "tis_620",
- "unicode_escape",
- "unicode_internal",
- "utf_16",
- "utf_16_be",
- "utf_16_le",
- "utf_7",
- "utf_8",
-]
-
-if hasattr(codecs, "mbcs_encode"):
- all_unicode_encodings.append("mbcs")
-
-# The following encodings work only with str, not unicode
-all_string_encodings = [
- "quopri_codec",
- "string_escape",
- "uu_codec",
-]
-
-# The following encoding is not tested, because it's not supposed
-# to work:
-# "undefined"
-
-# The following encodings don't work in stateful mode
-broken_unicode_with_streams = [
- "base64_codec",
- "hex_codec",
- "punycode",
- "unicode_internal"
-]
-broken_incremental_coders = broken_unicode_with_streams[:]
-
-# The following encodings only support "strict" mode
-only_strict_mode = [
- "idna",
- "zlib_codec",
- "bz2_codec",
-]
-
-try:
- import bz2
-except ImportError:
- pass
-else:
- all_unicode_encodings.append("bz2_codec")
- broken_unicode_with_streams.append("bz2_codec")
-
-try:
- import zlib
-except ImportError:
- pass
-else:
- all_unicode_encodings.append("zlib_codec")
- broken_unicode_with_streams.append("zlib_codec")
-
-class BasicUnicodeTest(unittest.TestCase):
- def test_basics(self):
- s = u"abc123" # all codecs should be able to encode these
- for encoding in all_unicode_encodings:
- name = codecs.lookup(encoding).name
- if encoding.endswith("_codec"):
- name += "_codec"
- elif encoding == "latin_1":
- name = "latin_1"
- self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-"))
- (bytes, size) = codecs.getencoder(encoding)(s)
- self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
- (chars, size) = codecs.getdecoder(encoding)(bytes)
- self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
-
- if encoding not in broken_unicode_with_streams:
- # check stream reader/writer
- q = Queue()
- writer = codecs.getwriter(encoding)(q)
- encodedresult = ""
- for c in s:
- writer.write(c)
- encodedresult += q.read()
- q = Queue()
- reader = codecs.getreader(encoding)(q)
- decodedresult = u""
- for c in encodedresult:
- q.write(c)
- decodedresult += reader.read()
- self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
- if encoding not in broken_incremental_coders:
- # check incremental decoder/encoder (fetched via the Python
- # and C API) and iterencode()/iterdecode()
- try:
- encoder = codecs.getincrementalencoder(encoding)()
- cencoder = _testcapi.codec_incrementalencoder(encoding)
- except LookupError: # no IncrementalEncoder
- pass
- else:
- # check incremental decoder/encoder
- encodedresult = ""
- for c in s:
- encodedresult += encoder.encode(c)
- encodedresult += encoder.encode(u"", True)
- decoder = codecs.getincrementaldecoder(encoding)()
- decodedresult = u""
- for c in encodedresult:
- decodedresult += decoder.decode(c)
- decodedresult += decoder.decode("", True)
- self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
- # check C API
- encodedresult = ""
- for c in s:
- encodedresult += cencoder.encode(c)
- encodedresult += cencoder.encode(u"", True)
- cdecoder = _testcapi.codec_incrementaldecoder(encoding)
- decodedresult = u""
- for c in encodedresult:
- decodedresult += cdecoder.decode(c)
- decodedresult += cdecoder.decode("", True)
- self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
- # check iterencode()/iterdecode()
- result = u"".join(codecs.iterdecode(codecs.iterencode(s, encoding), encoding))
- self.assertEqual(result, s, "%r != %r (encoding=%r)" % (result, s, encoding))
-
- # check iterencode()/iterdecode() with empty string
- result = u"".join(codecs.iterdecode(codecs.iterencode(u"", encoding), encoding))
- self.assertEqual(result, u"")
-
- if encoding not in only_strict_mode:
- # check incremental decoder/encoder with errors argument
- try:
- encoder = codecs.getincrementalencoder(encoding)("ignore")
- cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore")
- except LookupError: # no IncrementalEncoder
- pass
- else:
- encodedresult = "".join(encoder.encode(c) for c in s)
- decoder = codecs.getincrementaldecoder(encoding)("ignore")
- decodedresult = u"".join(decoder.decode(c) for c in encodedresult)
- self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
- encodedresult = "".join(cencoder.encode(c) for c in s)
- cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore")
- decodedresult = u"".join(cdecoder.decode(c) for c in encodedresult)
- self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
-
- def test_seek(self):
- # all codecs should be able to encode these
- s = u"%s\n%s\n" % (100*u"abc123", 100*u"def456")
- for encoding in all_unicode_encodings:
- if encoding == "idna": # FIXME: See SF bug #1163178
- continue
- if encoding in broken_unicode_with_streams:
- continue
- reader = codecs.getreader(encoding)(StringIO.StringIO(s.encode(encoding)))
- for t in xrange(5):
- # Test that calling seek resets the internal codec state and buffers
- reader.seek(0, 0)
- line = reader.readline()
- self.assertEqual(s[:len(line)], line)
-
- def test_bad_decode_args(self):
- for encoding in all_unicode_encodings:
- decoder = codecs.getdecoder(encoding)
- self.assertRaises(TypeError, decoder)
- if encoding not in ("idna", "punycode"):
- self.assertRaises(TypeError, decoder, 42)
-
- def test_bad_encode_args(self):
- for encoding in all_unicode_encodings:
- encoder = codecs.getencoder(encoding)
- self.assertRaises(TypeError, encoder)
-
- def test_encoding_map_type_initialized(self):
- from encodings import cp1140
- # This used to crash, we are only verifying there's no crash.
- table_type = type(cp1140.encoding_table)
- self.assertEqual(table_type, table_type)
-
-class BasicStrTest(unittest.TestCase):
- def test_basics(self):
- s = "abc123"
- for encoding in all_string_encodings:
- (bytes, size) = codecs.getencoder(encoding)(s)
- self.assertEqual(size, len(s))
- (chars, size) = codecs.getdecoder(encoding)(bytes)
- self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
-
-class CharmapTest(unittest.TestCase):
- def test_decode_with_string_map(self):
- self.assertEqual(
- codecs.charmap_decode("\x00\x01\x02", "strict", u"abc"),
- (u"abc", 3)
- )
-
- self.assertEqual(
- codecs.charmap_decode("\x00\x01\x02", "replace", u"ab"),
- (u"ab\ufffd", 3)
- )
-
- self.assertEqual(
- codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe"),
- (u"ab\ufffd", 3)
- )
-
- self.assertEqual(
- codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab"),
- (u"ab", 3)
- )
-
- self.assertEqual(
- codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"),
- (u"ab", 3)
- )
-
- allbytes = "".join(chr(i) for i in xrange(256))
- self.assertEqual(
- codecs.charmap_decode(allbytes, "ignore", u""),
- (u"", len(allbytes))
- )
-
-class WithStmtTest(unittest.TestCase):
- def test_encodedfile(self):
- f = StringIO.StringIO("\xc3\xbc")
- with codecs.EncodedFile(f, "latin-1", "utf-8") as ef:
- self.assertEqual(ef.read(), "\xfc")
-
- def test_streamreaderwriter(self):
- f = StringIO.StringIO("\xc3\xbc")
- info = codecs.lookup("utf-8")
- with codecs.StreamReaderWriter(f, info.streamreader,
- info.streamwriter, 'strict') as srw:
- self.assertEqual(srw.read(), u"\xfc")
-
-
-class BomTest(unittest.TestCase):
- def test_seek0(self):
- data = u"1234567890"
- tests = ("utf-16",
- "utf-16-le",
- "utf-16-be",
- "utf-32",
- "utf-32-le",
- "utf-32-be")
- for encoding in tests:
- # Check if the BOM is written only once
- with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
- f.write(data)
- f.write(data)
- f.seek(0)
- self.assertEqual(f.read(), data * 2)
- f.seek(0)
- self.assertEqual(f.read(), data * 2)
-
- # Check that the BOM is written after a seek(0)
- with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
- f.write(data[0])
- self.assertNotEqual(f.tell(), 0)
- f.seek(0)
- f.write(data)
- f.seek(0)
- self.assertEqual(f.read(), data)
-
- # (StreamWriter) Check that the BOM is written after a seek(0)
- with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
- f.writer.write(data[0])
- self.assertNotEqual(f.writer.tell(), 0)
- f.writer.seek(0)
- f.writer.write(data)
- f.seek(0)
- self.assertEqual(f.read(), data)
-
- # Check that the BOM is not written after a seek() at a position
- # different than the start
- with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
- f.write(data)
- f.seek(f.tell())
- f.write(data)
- f.seek(0)
- self.assertEqual(f.read(), data * 2)
-
- # (StreamWriter) Check that the BOM is not written after a seek()
- # at a position different than the start
- with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
- f.writer.write(data)
- f.writer.seek(f.writer.tell())
- f.writer.write(data)
- f.seek(0)
- self.assertEqual(f.read(), data * 2)
-
-
-def test_main():
- test_support.run_unittest(
- UTF32Test,
- UTF32LETest,
- UTF32BETest,
- UTF16Test,
- UTF16LETest,
- UTF16BETest,
- UTF8Test,
- UTF8SigTest,
- UTF7Test,
- UTF16ExTest,
- ReadBufferTest,
- CharBufferTest,
- EscapeDecodeTest,
- RecodingTest,
- PunycodeTest,
- UnicodeInternalTest,
- NameprepTest,
- IDNACodecTest,
- CodecsModuleTest,
- StreamReaderTest,
- EncodedFileTest,
- Str2StrTest,
- BasicUnicodeTest,
- BasicStrTest,
- CharmapTest,
- WithStmtTest,
- BomTest,
- )
-
-
-if __name__ == "__main__":
- test_main()
diff --git a/lib-python/modified-2.7/test/test_descr.py b/lib-python/modified-2.7/test/test_descr.py
--- a/lib-python/modified-2.7/test/test_descr.py
+++ b/lib-python/modified-2.7/test/test_descr.py
@@ -4399,13 +4399,10 @@
self.assertTrue(l.__add__ != [5].__add__)
self.assertTrue(l.__add__ != l.__mul__)
self.assertTrue(l.__add__.__name__ == '__add__')
- if hasattr(l.__add__, '__self__'):
- # CPython
- self.assertTrue(l.__add__.__self__ is l)
+ self.assertTrue(l.__add__.__self__ is l)
+ if hasattr(l.__add__, '__objclass__'): # CPython
self.assertTrue(l.__add__.__objclass__ is list)
- else:
- # Python implementations where [].__add__ is a normal bound method
- self.assertTrue(l.__add__.im_self is l)
+ else: # PyPy
self.assertTrue(l.__add__.im_class is list)
self.assertEqual(l.__add__.__doc__, list.__add__.__doc__)
try:
diff --git a/lib-python/modified-2.7/test/test_dis.py b/lib-python/modified-2.7/test/test_dis.py
deleted file mode 100644
--- a/lib-python/modified-2.7/test/test_dis.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# Minimal tests for dis module
-
-from test.test_support import run_unittest
-import unittest
-import sys
-import dis
-import StringIO
-
-
-def _f(a):
- print a
- return 1
-
-dis_f = """\
- %-4d 0 LOAD_FAST 0 (a)
- 3 PRINT_ITEM
- 4 PRINT_NEWLINE
-
- %-4d 5 LOAD_CONST 1 (1)
- 8 RETURN_VALUE
-"""%(_f.func_code.co_firstlineno + 1,
- _f.func_code.co_firstlineno + 2)
-
-
-# we "call" rangexxx() instead of range() to disable the
-# pypy optimization that turns it into CALL_LIKELY_BUILTIN.
-def bug708901():
- for res in rangexxx(1,
- 10):
- pass
-
-dis_bug708901 = """\
- %-4d 0 SETUP_LOOP 23 (to 26)
- 3 LOAD_GLOBAL 0 (rangexxx)
- 6 LOAD_CONST 1 (1)
-
- %-4d 9 LOAD_CONST 2 (10)
- 12 CALL_FUNCTION 2
- 15 GET_ITER
- >> 16 FOR_ITER 6 (to 25)
- 19 STORE_FAST 0 (res)
-
- %-4d 22 JUMP_ABSOLUTE 16
- >> 25 POP_BLOCK
- >> 26 LOAD_CONST 0 (None)
- 29 RETURN_VALUE
-"""%(bug708901.func_code.co_firstlineno + 1,
- bug708901.func_code.co_firstlineno + 2,
- bug708901.func_code.co_firstlineno + 3)
-
-
-def bug1333982(x=[]):
- assert 0, ([s for s in x] +
- 1)
- pass
-
-dis_bug1333982 = """\
- %-4d 0 LOAD_CONST 1 (0)
- 3 POP_JUMP_IF_TRUE 38
- 6 LOAD_GLOBAL 0 (AssertionError)
- 9 BUILD_LIST 0
- 12 LOAD_FAST 0 (x)
- 15 GET_ITER
- >> 16 FOR_ITER 12 (to 31)
- 19 STORE_FAST 1 (s)
- 22 LOAD_FAST 1 (s)
- 25 LIST_APPEND 2
- 28 JUMP_ABSOLUTE 16
-
- %-4d >> 31 LOAD_CONST 2 (1)
- 34 BINARY_ADD
- 35 RAISE_VARARGS 2
-
- %-4d >> 38 LOAD_CONST 0 (None)
- 41 RETURN_VALUE
-"""%(bug1333982.func_code.co_firstlineno + 1,
- bug1333982.func_code.co_firstlineno + 2,
- bug1333982.func_code.co_firstlineno + 3)
-
-_BIG_LINENO_FORMAT = """\
-%3d 0 LOAD_GLOBAL 0 (spam)
- 3 POP_TOP
- 4 LOAD_CONST 0 (None)
- 7 RETURN_VALUE
-"""
-
-class DisTests(unittest.TestCase):
- def do_disassembly_test(self, func, expected):
- s = StringIO.StringIO()
- save_stdout = sys.stdout
- sys.stdout = s
- dis.dis(func)
- sys.stdout = save_stdout
- got = s.getvalue()
- # Trim trailing blanks (if any).
- lines = got.split('\n')
- lines = [line.rstrip() for line in lines]
- expected = expected.split("\n")
- import difflib
- if expected != lines:
- self.fail(
- "events did not match expectation:\n" +
- "\n".join(difflib.ndiff(expected,
- lines)))
-
- def test_opmap(self):
- self.assertEqual(dis.opmap["STOP_CODE"], 0)
- self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst)
- self.assertIn(dis.opmap["STORE_NAME"], dis.hasname)
-
- def test_opname(self):
- self.assertEqual(dis.opname[dis.opmap["LOAD_FAST"]], "LOAD_FAST")
-
- def test_boundaries(self):
- self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG)
- self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT)
-
- def test_dis(self):
- self.do_disassembly_test(_f, dis_f)
-
- def test_bug_708901(self):
- self.do_disassembly_test(bug708901, dis_bug708901)
-
- def test_bug_1333982(self):
- # This one is checking bytecodes generated for an `assert` statement,
- # so fails if the tests are run with -O. Skip this test then.
- if __debug__:
- self.do_disassembly_test(bug1333982, dis_bug1333982)
-
- def test_big_linenos(self):
- def func(count):
- namespace = {}
- func = "def foo():\n " + "".join(["\n "] * count + ["spam\n"])
- exec func in namespace
- return namespace['foo']
-
- # Test all small ranges
- for i in xrange(1, 300):
- expected = _BIG_LINENO_FORMAT % (i + 2)
- self.do_disassembly_test(func(i), expected)
-
- # Test some larger ranges too
- for i in xrange(300, 5000, 10):
- expected = _BIG_LINENO_FORMAT % (i + 2)
- self.do_disassembly_test(func(i), expected)
-
-def test_main():
- run_unittest(DisTests)
-
-
-if __name__ == "__main__":
- test_main()
diff --git a/lib-python/modified-2.7/test/test_extcall.py b/lib-python/modified-2.7/test/test_extcall.py
--- a/lib-python/modified-2.7/test/test_extcall.py
+++ b/lib-python/modified-2.7/test/test_extcall.py
@@ -299,7 +299,7 @@
def f(a):
return a
self.assertEqual(f(**{u'a': 4}), 4)
- self.assertRaises(TypeError, lambda: f(**{u'stören': 4}))
+ self.assertRaises(TypeError, f, **{u'stören': 4})
self.assertRaises(TypeError, f, **{u'someLongString':2})
try:
f(a=4, **{u'a': 4})
diff --git a/lib-python/modified-2.7/test/test_fcntl.py b/lib-python/modified-2.7/test/test_fcntl.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/test/test_fcntl.py
@@ -0,0 +1,108 @@
+"""Test program for the fcntl C module.
+
+OS/2+EMX doesn't support the file locking operations.
+
+"""
+import os
+import struct
+import sys
+import unittest
+from test.test_support import (verbose, TESTFN, unlink, run_unittest,
+ import_module)
+
+# Skip test if no fnctl module.
+fcntl = import_module('fcntl')
+
+
+# TODO - Write tests for flock() and lockf().
+
+def get_lockdata():
+ if sys.platform.startswith('atheos'):
+ start_len = "qq"
+ else:
+ try:
+ os.O_LARGEFILE
+ except AttributeError:
+ start_len = "ll"
+ else:
+ start_len = "qq"
+
+ if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3',
+ 'Darwin1.2', 'darwin',
+ 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5',
+ 'freebsd6', 'freebsd7', 'freebsd8',
+ 'bsdos2', 'bsdos3', 'bsdos4',
+ 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', 'openbsd5'):
+ if struct.calcsize('l') == 8:
+ off_t = 'l'
+ pid_t = 'i'
+ else:
+ off_t = 'lxxxx'
+ pid_t = 'l'
+ lockdata = struct.pack(off_t + off_t + pid_t + 'hh', 0, 0, 0,
+ fcntl.F_WRLCK, 0)
+ elif sys.platform in ['aix3', 'aix4', 'hp-uxB', 'unixware7']:
+ lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0)
+ elif sys.platform in ['os2emx']:
+ lockdata = None
+ else:
+ lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
+ if lockdata:
+ if verbose:
+ print 'struct.pack: ', repr(lockdata)
+ return lockdata
+
+lockdata = get_lockdata()
+
+
+class TestFcntl(unittest.TestCase):
+
+ def setUp(self):
+ self.f = None
+
+ def tearDown(self):
+ if self.f and not self.f.closed:
+ self.f.close()
+ unlink(TESTFN)
+
+ def test_fcntl_fileno(self):
+ # the example from the library docs
+ self.f = open(TESTFN, 'w')
+ rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
+ if verbose:
+ print 'Status from fcntl with O_NONBLOCK: ', rv
+ if sys.platform not in ['os2emx']:
+ rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata)
+ if verbose:
+ print 'String from fcntl with F_SETLKW: ', repr(rv)
+ self.f.close()
+
+ def test_fcntl_file_descriptor(self):
+ # again, but pass the file rather than numeric descriptor
+ self.f = open(TESTFN, 'w')
+ rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK)
+ if sys.platform not in ['os2emx']:
+ rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata)
+ self.f.close()
+
+ def test_fcntl_64_bit(self):
+ # Issue #1309352: fcntl shouldn't fail when the third arg fits in a
+ # C 'long' but not in a C 'int'.
+ try:
+ cmd = fcntl.F_NOTIFY
+ # This flag is larger than 2**31 in 64-bit builds
+ flags = fcntl.DN_MULTISHOT
+ except AttributeError:
+ self.skipTest("F_NOTIFY or DN_MULTISHOT unavailable")
+ fd = os.open(os.path.dirname(os.path.abspath(TESTFN)), os.O_RDONLY)
+ try:
+ fcntl.fcntl(fd, cmd, flags)
+ finally:
+ os.close(fd)
+
+
+def test_main():
+ run_unittest(TestFcntl)
+
+if __name__ == '__main__':
+ test_main()
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -186,6 +186,11 @@
self.assertFalse(sys.modules['heapq'] is self.module)
self.assertTrue(hasattr(self.module.heapify, 'func_code'))
+ def test_islice_protection(self):
+ m = self.module
+ self.assertFalse(m.nsmallest(-1, [1]))
+ self.assertFalse(m.nlargest(-1, [1]))
+
class TestHeapC(TestHeap):
module = c_heapq
diff --git a/lib-python/modified-2.7/test/test_import.py b/lib-python/modified-2.7/test/test_import.py
--- a/lib-python/modified-2.7/test/test_import.py
+++ b/lib-python/modified-2.7/test/test_import.py
@@ -64,6 +64,7 @@
except ImportError, err:
self.fail("import from %s failed: %s" % (ext, err))
else:
+ # XXX importing .pyw is missing on Windows
self.assertEqual(mod.a, a,
"module loaded (%s) but contents invalid" % mod)
self.assertEqual(mod.b, b,
diff --git a/lib-python/2.7/test/test_multibytecodec.py b/lib-python/modified-2.7/test/test_multibytecodec.py
copy from lib-python/2.7/test/test_multibytecodec.py
copy to lib-python/modified-2.7/test/test_multibytecodec.py
--- a/lib-python/2.7/test/test_multibytecodec.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec.py
@@ -42,7 +42,7 @@
dec = codecs.getdecoder('euc-kr')
myreplace = lambda exc: (u'', sys.maxint+1)
codecs.register_error('test.cjktest', myreplace)
- self.assertRaises(IndexError, dec,
+ self.assertRaises((IndexError, OverflowError), dec,
'apple\x92ham\x93spam', 'test.cjktest')
def test_codingspec(self):
@@ -148,7 +148,8 @@
class Test_StreamReader(unittest.TestCase):
def test_bug1728403(self):
try:
- open(TESTFN, 'w').write('\xa1')
+ with open(TESTFN, 'w') as f:
+ f.write('\xa1')
f = codecs.open(TESTFN, encoding='cp949')
self.assertRaises(UnicodeDecodeError, f.read, 2)
finally:
diff --git a/lib-python/2.7/test/test_multibytecodec_support.py b/lib-python/modified-2.7/test/test_multibytecodec_support.py
copy from lib-python/2.7/test/test_multibytecodec_support.py
copy to lib-python/modified-2.7/test/test_multibytecodec_support.py
--- a/lib-python/2.7/test/test_multibytecodec_support.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec_support.py
@@ -107,8 +107,8 @@
def myreplace(exc):
return (u'x', sys.maxint + 1)
codecs.register_error("test.cjktest", myreplace)
- self.assertRaises(IndexError, self.encode, self.unmappedunicode,
- 'test.cjktest')
+ self.assertRaises((IndexError, OverflowError), self.encode,
+ self.unmappedunicode, 'test.cjktest')
def test_callback_None_index(self):
def myreplace(exc):
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
p.join()
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_qsize(self):
q = self.Queue()
try:
@@ -532,7 +531,6 @@
time.sleep(DELTA)
q.task_done()
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_task_done(self):
queue = self.JoinableQueue()
@@ -1091,7 +1089,6 @@
class _TestPoolWorkerLifetime(BaseTestCase):
ALLOWED_TYPES = ('processes', )
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_pool_worker_lifetime(self):
p = multiprocessing.Pool(3, maxtasksperchild=10)
self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
queue = manager.get_queue()
queue.put('hello world')
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_rapid_restart(self):
authkey = os.urandom(32)
manager = QueueManager(
@@ -1297,6 +1293,7 @@
queue = manager.get_queue()
self.assertEqual(queue.get(), 'hello world')
del queue
+ test_support.gc_collect()
manager.shutdown()
manager = QueueManager(
address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
ALLOWED_TYPES = ('processes',)
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_heap(self):
iterations = 5000
maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_repr.py b/lib-python/modified-2.7/test/test_repr.py
--- a/lib-python/modified-2.7/test/test_repr.py
+++ b/lib-python/modified-2.7/test/test_repr.py
@@ -254,8 +254,14 @@
eq = self.assertEqual
touch(os.path.join(self.subpkgname, self.pkgname + os.extsep + 'py'))
from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation
- eq(repr(areallylongpackageandmodulenametotestreprtruncation),
- "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+ # On PyPy, we use %r to format the file name; on CPython it is done
+ # with '%s'. It seems to me that %r is safer <arigo>.
+ if '__pypy__' in sys.builtin_module_names:
+ eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+ "<module %r from %r>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+ else:
+ eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+ "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
eq(repr(sys), "<module 'sys' (built-in)>")
def test_type(self):
diff --git a/lib-python/2.7/test/test_sets.py b/lib-python/modified-2.7/test/test_sets.py
copy from lib-python/2.7/test/test_sets.py
copy to lib-python/modified-2.7/test/test_sets.py
--- a/lib-python/2.7/test/test_sets.py
+++ b/lib-python/modified-2.7/test/test_sets.py
@@ -686,7 +686,9 @@
set_list = sorted(self.set)
self.assertEqual(len(dup_list), len(set_list))
for i, el in enumerate(dup_list):
- self.assertIs(el, set_list[i])
+ # Object identity is not guarnteed for immutable objects, so we
+ # can't use assertIs here.
+ self.assertEqual(el, set_list[i])
def test_deep_copy(self):
dup = copy.deepcopy(self.set)
diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
def skip_if_broken_ubuntu_ssl(func):
- # We need to access the lower-level wrapper in order to create an
- # implicit SSL context without trying to connect or listen.
- try:
- import _ssl
- except ImportError:
- # The returned function won't get executed, just ignore the error
- pass
- @functools.wraps(func)
- def f(*args, **kwargs):
+ if hasattr(ssl, 'PROTOCOL_SSLv2'):
+ # We need to access the lower-level wrapper in order to create an
+ # implicit SSL context without trying to connect or listen.
try:
- s = socket.socket(socket.AF_INET)
- _ssl.sslwrap(s._sock, 0, None, None,
- ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
- except ssl.SSLError as e:
- if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
- platform.linux_distribution() == ('debian', 'squeeze/sid', '')
- and 'Invalid SSL protocol variant specified' in str(e)):
- raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
- return func(*args, **kwargs)
- return f
+ import _ssl
+ except ImportError:
+ # The returned function won't get executed, just ignore the error
+ pass
+ @functools.wraps(func)
+ def f(*args, **kwargs):
+ try:
+ s = socket.socket(socket.AF_INET)
+ _ssl.sslwrap(s._sock, 0, None, None,
+ ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+ except ssl.SSLError as e:
+ if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+ platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+ and 'Invalid SSL protocol variant specified' in str(e)):
+ raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+ return func(*args, **kwargs)
+ return f
+ else:
+ return func
class BasicSocketTests(unittest.TestCase):
def test_constants(self):
- ssl.PROTOCOL_SSLv2
+ #ssl.PROTOCOL_SSLv2
ssl.PROTOCOL_SSLv23
ssl.PROTOCOL_SSLv3
ssl.PROTOCOL_TLSv1
@@ -105,7 +108,6 @@
print "didn't raise TypeError"
ssl.RAND_add("this is a random string", 75.0)
- @test_support.impl_detail("obscure test")
def test_parse_cert(self):
# note that this uses an 'unofficial' function in _ssl.c,
# provided solely for this test, to exercise the certificate
@@ -840,6 +842,8 @@
c = socket.socket()
c.connect((HOST, port))
listener_gone.wait()
+ # XXX why is it necessary?
+ test_support.gc_collect()
try:
ssl_sock = ssl.wrap_socket(c)
except IOError:
@@ -965,7 +969,8 @@
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
- try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+ if hasattr(ssl, 'PROTOCOL_SSLv2'):
+ try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
@@ -977,7 +982,8 @@
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
- try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+ if hasattr(ssl, 'PROTOCOL_SSLv2'):
+ try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
diff --git a/lib-python/2.7/test/test_subprocess.py b/lib-python/modified-2.7/test/test_subprocess.py
copy from lib-python/2.7/test/test_subprocess.py
copy to lib-python/modified-2.7/test/test_subprocess.py
--- a/lib-python/2.7/test/test_subprocess.py
+++ b/lib-python/modified-2.7/test/test_subprocess.py
@@ -16,11 +16,11 @@
# Depends on the following external programs: Python
#
-if mswindows:
- SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
- 'os.O_BINARY);')
-else:
- SETBINARY = ''
+#if mswindows:
+# SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
+# 'os.O_BINARY);')
+#else:
+# SETBINARY = ''
try:
@@ -420,8 +420,9 @@
self.assertStderrEqual(stderr, "")
def test_universal_newlines(self):
- p = subprocess.Popen([sys.executable, "-c",
- 'import sys,os;' + SETBINARY +
+ # NB. replaced SETBINARY with the -u flag
+ p = subprocess.Popen([sys.executable, "-u", "-c",
+ 'import sys,os;' + #SETBINARY +
'sys.stdout.write("line1\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line2\\r");'
@@ -448,8 +449,9 @@
def test_universal_newlines_communicate(self):
# universal newlines through communicate()
- p = subprocess.Popen([sys.executable, "-c",
- 'import sys,os;' + SETBINARY +
+ # NB. replaced SETBINARY with the -u flag
+ p = subprocess.Popen([sys.executable, "-u", "-c",
+ 'import sys,os;' + #SETBINARY +
'sys.stdout.write("line1\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line2\\r");'
diff --git a/lib-python/modified-2.7/test/test_support.py b/lib-python/modified-2.7/test/test_support.py
--- a/lib-python/modified-2.7/test/test_support.py
+++ b/lib-python/modified-2.7/test/test_support.py
@@ -1066,7 +1066,7 @@
if '--pdb' in sys.argv:
import pdb, traceback
traceback.print_tb(exc_info[2])
- pdb.post_mortem(exc_info[2], pdb.Pdb)
+ pdb.post_mortem(exc_info[2])
# ----------------------------------
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
self.compare_events(func.func_code.co_firstlineno,
tracer.events, func.events)
- def set_and_retrieve_none(self):
+ def test_set_and_retrieve_none(self):
sys.settrace(None)
assert sys.gettrace() is None
- def set_and_retrieve_func(self):
+ def test_set_and_retrieve_func(self):
def fn(*args):
pass
diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/modified-2.7/test/test_tarfile.py
copy from lib-python/2.7/test/test_tarfile.py
copy to lib-python/modified-2.7/test/test_tarfile.py
--- a/lib-python/2.7/test/test_tarfile.py
+++ b/lib-python/modified-2.7/test/test_tarfile.py
@@ -169,6 +169,7 @@
except tarfile.ReadError:
self.fail("tarfile.open() failed on empty archive")
self.assertListEqual(tar.getmembers(), [])
+ tar.close()
def test_null_tarfile(self):
# Test for issue6123: Allow opening empty archives.
@@ -207,16 +208,21 @@
fobj = open(self.tarname, "rb")
tar = tarfile.open(fileobj=fobj, mode=self.mode)
self.assertEqual(tar.name, os.path.abspath(fobj.name))
+ tar.close()
def test_no_name_attribute(self):
- data = open(self.tarname, "rb").read()
+ f = open(self.tarname, "rb")
+ data = f.read()
+ f.close()
fobj = StringIO.StringIO(data)
self.assertRaises(AttributeError, getattr, fobj, "name")
tar = tarfile.open(fileobj=fobj, mode=self.mode)
self.assertEqual(tar.name, None)
def test_empty_name_attribute(self):
- data = open(self.tarname, "rb").read()
+ f = open(self.tarname, "rb")
+ data = f.read()
+ f.close()
fobj = StringIO.StringIO(data)
fobj.name = ""
tar = tarfile.open(fileobj=fobj, mode=self.mode)
@@ -515,6 +521,7 @@
self.tar = tarfile.open(self.tarname, mode=self.mode, encoding="iso8859-1")
tarinfo = self.tar.getmember("pax/umlauts-�������")
self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+ self.tar.close()
class LongnameTest(ReadTest):
@@ -675,6 +682,7 @@
tar = tarfile.open(tmpname, self.mode)
tarinfo = tar.gettarinfo(path)
self.assertEqual(tarinfo.size, 0)
+ tar.close()
finally:
os.rmdir(path)
@@ -692,6 +700,7 @@
tar.gettarinfo(target)
tarinfo = tar.gettarinfo(link)
self.assertEqual(tarinfo.size, 0)
+ tar.close()
finally:
os.remove(target)
os.remove(link)
@@ -704,6 +713,7 @@
tar = tarfile.open(tmpname, self.mode)
tarinfo = tar.gettarinfo(path)
self.assertEqual(tarinfo.size, 0)
+ tar.close()
finally:
os.remove(path)
@@ -722,6 +732,7 @@
tar.add(dstname)
os.chdir(cwd)
self.assertTrue(tar.getnames() == [], "added the archive to itself")
+ tar.close()
def test_exclude(self):
tempdir = os.path.join(TEMPDIR, "exclude")
@@ -742,6 +753,7 @@
tar = tarfile.open(tmpname, "r")
self.assertEqual(len(tar.getmembers()), 1)
self.assertEqual(tar.getnames()[0], "empty_dir")
+ tar.close()
finally:
shutil.rmtree(tempdir)
@@ -859,7 +871,9 @@
fobj.close()
elif self.mode.endswith("bz2"):
dec = bz2.BZ2Decompressor()
- data = open(tmpname, "rb").read()
+ f = open(tmpname, "rb")
+ data = f.read()
+ f.close()
data = dec.decompress(data)
self.assertTrue(len(dec.unused_data) == 0,
"found trailing data")
@@ -938,6 +952,7 @@
"unable to read longname member")
self.assertEqual(tarinfo.linkname, member.linkname,
"unable to read longname member")
+ tar.close()
def test_longname_1023(self):
self._test(("longnam/" * 127) + "longnam")
@@ -1030,6 +1045,7 @@
else:
n = tar.getmembers()[0].name
self.assertTrue(name == n, "PAX longname creation failed")
+ tar.close()
def test_pax_global_header(self):
pax_headers = {
@@ -1058,6 +1074,7 @@
tarfile.PAX_NUMBER_FIELDS[key](val)
except (TypeError, ValueError):
self.fail("unable to convert pax header field")
+ tar.close()
def test_pax_extended_header(self):
# The fields from the pax header have priority over the
@@ -1077,6 +1094,7 @@
self.assertEqual(t.pax_headers, pax_headers)
self.assertEqual(t.name, "foo")
self.assertEqual(t.uid, 123)
+ tar.close()
class UstarUnicodeTest(unittest.TestCase):
@@ -1120,6 +1138,7 @@
tarinfo.name = "foo"
tarinfo.uname = u"���"
self.assertRaises(UnicodeError, tar.addfile, tarinfo)
+ tar.close()
def test_unicode_argument(self):
tar = tarfile.open(tarname, "r", encoding="iso8859-1", errors="strict")
@@ -1174,6 +1193,7 @@
tar = tarfile.open(tmpname, format=self.format, encoding="ascii",
errors=handler)
self.assertEqual(tar.getnames()[0], name)
+ tar.close()
self.assertRaises(UnicodeError, tarfile.open, tmpname,
encoding="ascii", errors="strict")
@@ -1186,6 +1206,7 @@
tar = tarfile.open(tmpname, format=self.format, encoding="iso8859-1",
errors="utf-8")
self.assertEqual(tar.getnames()[0], "���/" + u"�".encode("utf8"))
+ tar.close()
class AppendTest(unittest.TestCase):
@@ -1213,6 +1234,7 @@
def _test(self, names=["bar"], fileobj=None):
tar = tarfile.open(self.tarname, fileobj=fileobj)
self.assertEqual(tar.getnames(), names)
+ tar.close()
def test_non_existing(self):
self._add_testfile()
@@ -1231,7 +1253,9 @@
def test_fileobj(self):
self._create_testtar()
- data = open(self.tarname).read()
+ f = open(self.tarname)
+ data = f.read()
+ f.close()
fobj = StringIO.StringIO(data)
self._add_testfile(fobj)
fobj.seek(0)
@@ -1257,7 +1281,9 @@
# Append mode is supposed to fail if the tarfile to append to
# does not end with a zero block.
def _test_error(self, data):
- open(self.tarname, "wb").write(data)
+ f = open(self.tarname, "wb")
+ f.write(data)
+ f.close()
self.assertRaises(tarfile.ReadError, self._add_testfile)
def test_null(self):
diff --git a/lib-python/modified-2.7/test/test_tempfile.py b/lib-python/modified-2.7/test/test_tempfile.py
--- a/lib-python/modified-2.7/test/test_tempfile.py
+++ b/lib-python/modified-2.7/test/test_tempfile.py
@@ -23,8 +23,8 @@
# TEST_FILES may need to be tweaked for systems depending on the maximum
# number of files that can be opened at one time (see ulimit -n)
-if sys.platform in ('openbsd3', 'openbsd4'):
- TEST_FILES = 48
+if sys.platform.startswith("openbsd"):
+ TEST_FILES = 64 # ulimit -n defaults to 128 for normal users
else:
TEST_FILES = 100
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
def getresponse(self):
return MockHTTPResponse(MockFile(), {}, 200, "OK")
+ def close(self):
+ pass
+
class MockHandler:
# useful for testing handler machinery
# see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/test/test_weakref.py b/lib-python/modified-2.7/test/test_weakref.py
--- a/lib-python/modified-2.7/test/test_weakref.py
+++ b/lib-python/modified-2.7/test/test_weakref.py
@@ -993,13 +993,13 @@
self.assertTrue(len(weakdict) == 2)
k, v = weakdict.popitem()
self.assertTrue(len(weakdict) == 1)
- if k is key1:
+ if k == key1:
self.assertTrue(v is value1)
else:
self.assertTrue(v is value2)
k, v = weakdict.popitem()
self.assertTrue(len(weakdict) == 0)
- if k is key1:
+ if k == key1:
self.assertTrue(v is value1)
else:
self.assertTrue(v is value2)
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1436 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below). It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work. Each Handler implements a particular protocol or
+option. The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL. For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns. The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib. pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back. One difference is that you can also pass
+a Request instance instead of URL. Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers. Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate. If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request. The
+state can be as simple as the URL. It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+ uri='https://mahler:8092/site-updates.py',
+ user='klem',
+ passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled? The client needs to know the HTTP error code. But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+ addinfourl, splitport, splittag,
+ splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+ global _opener
+ if _opener is None:
+ _opener = build_opener()
+ return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+ global _opener
+ _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden. we just want to be
+# subtypes.
+
+class URLError(IOError):
+ # URLError is a sub-type of IOError, but it doesn't share any of
+ # the implementation. need to override __init__ and __str__.
+ # It sets self.args for compatibility with other EnvironmentError
+ # subclasses, but args doesn't have the typical format with errno in
+ # slot 0 and strerror in slot 1. This may be better than nothing.
+ def __init__(self, reason):
+ self.args = reason,
+ self.reason = reason
+
+ def __str__(self):
+ return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+ """Raised when HTTP error occurs, but also acts like non-error return"""
+ __super_init = addinfourl.__init__
+
+ def __init__(self, url, code, msg, hdrs, fp):
+ self.code = code
+ self.msg = msg
+ self.hdrs = hdrs
+ self.fp = fp
+ self.filename = url
+ # The addinfourl classes depend on fp being a valid file
+ # object. In some cases, the HTTPError may not have a valid
+ # file object. If this happens, the simplest workaround is to
+ # not initialize the base classes.
+ if fp is not None:
+ self.__super_init(fp, hdrs, url, code)
+
+ def __str__(self):
+ return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+ """Return request-host, as defined by RFC 2965.
+
+ Variation from RFC: returned value is lowercased, for convenient
+ comparison.
+
+ """
+ url = request.get_full_url()
+ host = urlparse.urlparse(url)[1]
+ if host == "":
+ host = request.get_header("Host", "")
+
+ # remove port, if present
+ host = _cut_port_re.sub("", host, 1)
+ return host.lower()
+
+class Request:
+
+ def __init__(self, url, data=None, headers={},
+ origin_req_host=None, unverifiable=False):
+ # unwrap('<URL:type://host/path>') --> 'type://host/path'
+ self.__original = unwrap(url)
+ self.__original, fragment = splittag(self.__original)
+ self.type = None
+ # self.__r_type is what's left after doing the splittype
+ self.host = None
+ self.port = None
+ self._tunnel_host = None
+ self.data = data
+ self.headers = {}
+ for key, value in headers.items():
+ self.add_header(key, value)
+ self.unredirected_hdrs = {}
+ if origin_req_host is None:
+ origin_req_host = request_host(self)
+ self.origin_req_host = origin_req_host
+ self.unverifiable = unverifiable
+
+ def __getattr__(self, attr):
+ # XXX this is a fallback mechanism to guard against these
+ # methods getting called in a non-standard order. this may be
+ # too complicated and/or unnecessary.
+ # XXX should the __r_XXX attributes be public?
+ if attr[:12] == '_Request__r_':
+ name = attr[12:]
+ if hasattr(Request, 'get_' + name):
+ getattr(self, 'get_' + name)()
+ return getattr(self, attr)
+ raise AttributeError, attr
+
+ def get_method(self):
+ if self.has_data():
+ return "POST"
+ else:
+ return "GET"
+
+ # XXX these helper methods are lame
+
+ def add_data(self, data):
+ self.data = data
+
+ def has_data(self):
+ return self.data is not None
+
+ def get_data(self):
+ return self.data
+
+ def get_full_url(self):
+ return self.__original
+
+ def get_type(self):
+ if self.type is None:
+ self.type, self.__r_type = splittype(self.__original)
+ if self.type is None:
+ raise ValueError, "unknown url type: %s" % self.__original
+ return self.type
+
+ def get_host(self):
+ if self.host is None:
+ self.host, self.__r_host = splithost(self.__r_type)
+ if self.host:
+ self.host = unquote(self.host)
+ return self.host
+
+ def get_selector(self):
+ return self.__r_host
+
+ def set_proxy(self, host, type):
+ if self.type == 'https' and not self._tunnel_host:
+ self._tunnel_host = self.host
+ else:
+ self.type = type
+ self.__r_host = self.__original
+
+ self.host = host
+
+ def has_proxy(self):
+ return self.__r_host == self.__original
+
+ def get_origin_req_host(self):
+ return self.origin_req_host
+
+ def is_unverifiable(self):
+ return self.unverifiable
+
+ def add_header(self, key, val):
+ # useful for something like authentication
+ self.headers[key.capitalize()] = val
+
+ def add_unredirected_header(self, key, val):
+ # will not be added to a redirected request
+ self.unredirected_hdrs[key.capitalize()] = val
+
+ def has_header(self, header_name):
+ return (header_name in self.headers or
+ header_name in self.unredirected_hdrs)
+
+ def get_header(self, header_name, default=None):
+ return self.headers.get(
+ header_name,
+ self.unredirected_hdrs.get(header_name, default))
+
+ def header_items(self):
+ hdrs = self.unredirected_hdrs.copy()
+ hdrs.update(self.headers)
+ return hdrs.items()
+
+class OpenerDirector:
+ def __init__(self):
+ client_version = "Python-urllib/%s" % __version__
+ self.addheaders = [('User-agent', client_version)]
+ # manage the individual handlers
+ self.handlers = []
+ self.handle_open = {}
+ self.handle_error = {}
+ self.process_response = {}
+ self.process_request = {}
+
+ def add_handler(self, handler):
+ if not hasattr(handler, "add_parent"):
+ raise TypeError("expected BaseHandler instance, got %r" %
+ type(handler))
+
+ added = False
+ for meth in dir(handler):
+ if meth in ["redirect_request", "do_open", "proxy_open"]:
+ # oops, coincidental match
+ continue
+
+ i = meth.find("_")
+ protocol = meth[:i]
+ condition = meth[i+1:]
+
+ if condition.startswith("error"):
+ j = condition.find("_") + i + 1
+ kind = meth[j+1:]
+ try:
+ kind = int(kind)
+ except ValueError:
+ pass
+ lookup = self.handle_error.get(protocol, {})
+ self.handle_error[protocol] = lookup
+ elif condition == "open":
+ kind = protocol
+ lookup = self.handle_open
+ elif condition == "response":
+ kind = protocol
+ lookup = self.process_response
+ elif condition == "request":
+ kind = protocol
+ lookup = self.process_request
+ else:
+ continue
+
+ handlers = lookup.setdefault(kind, [])
+ if handlers:
+ bisect.insort(handlers, handler)
+ else:
+ handlers.append(handler)
+ added = True
+
+ if added:
+ # the handlers must work in an specific order, the order
+ # is specified in a Handler attribute
+ bisect.insort(self.handlers, handler)
+ handler.add_parent(self)
+
+ def close(self):
+ # Only exists for backwards compatibility.
+ pass
+
+ def _call_chain(self, chain, kind, meth_name, *args):
+ # Handlers raise an exception if no one else should try to handle
+ # the request, or return None if they can't but another handler
+ # could. Otherwise, they return the response.
+ handlers = chain.get(kind, ())
+ for handler in handlers:
+ func = getattr(handler, meth_name)
+
+ result = func(*args)
+ if result is not None:
+ return result
+
+ def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+ # accept a URL or a Request object
+ if isinstance(fullurl, basestring):
+ req = Request(fullurl, data)
+ else:
+ req = fullurl
+ if data is not None:
+ req.add_data(data)
+
+ req.timeout = timeout
+ protocol = req.get_type()
+
+ # pre-process request
+ meth_name = protocol+"_request"
+ for processor in self.process_request.get(protocol, []):
+ meth = getattr(processor, meth_name)
+ req = meth(req)
+
+ response = self._open(req, data)
+
+ # post-process response
+ meth_name = protocol+"_response"
+ for processor in self.process_response.get(protocol, []):
+ meth = getattr(processor, meth_name)
+ response = meth(req, response)
+
+ return response
+
+ def _open(self, req, data=None):
+ result = self._call_chain(self.handle_open, 'default',
+ 'default_open', req)
+ if result:
+ return result
+
+ protocol = req.get_type()
+ result = self._call_chain(self.handle_open, protocol, protocol +
+ '_open', req)
+ if result:
+ return result
+
+ return self._call_chain(self.handle_open, 'unknown',
+ 'unknown_open', req)
+
+ def error(self, proto, *args):
+ if proto in ('http', 'https'):
+ # XXX http[s] protocols are special-cased
+ dict = self.handle_error['http'] # https is not different than http
+ proto = args[2] # YUCK!
+ meth_name = 'http_error_%s' % proto
+ http_err = 1
+ orig_args = args
+ else:
+ dict = self.handle_error
+ meth_name = proto + '_error'
+ http_err = 0
+ args = (dict, proto, meth_name) + args
+ result = self._call_chain(*args)
+ if result:
+ return result
+
+ if http_err:
+ args = (dict, 'default', 'http_error_default') + orig_args
+ return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+ """Create an opener object from a list of handlers.
+
+ The opener will use several default handlers, including support
+ for HTTP, FTP and when applicable, HTTPS.
+
+ If any of the handlers passed as arguments are subclasses of the
+ default handlers, the default handlers will not be used.
+ """
+ import types
+ def isclass(obj):
+ return isinstance(obj, (types.ClassType, type))
+
+ opener = OpenerDirector()
+ default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+ HTTPDefaultErrorHandler, HTTPRedirectHandler,
+ FTPHandler, FileHandler, HTTPErrorProcessor]
+ if hasattr(httplib, 'HTTPS'):
+ default_classes.append(HTTPSHandler)
+ skip = set()
+ for klass in default_classes:
+ for check in handlers:
+ if isclass(check):
+ if issubclass(check, klass):
+ skip.add(klass)
+ elif isinstance(check, klass):
+ skip.add(klass)
+ for klass in skip:
+ default_classes.remove(klass)
+
+ for klass in default_classes:
+ opener.add_handler(klass())
+
+ for h in handlers:
+ if isclass(h):
+ h = h()
+ opener.add_handler(h)
+ return opener
+
+class BaseHandler:
+ handler_order = 500
+
+ def add_parent(self, parent):
+ self.parent = parent
+
+ def close(self):
+ # Only exists for backwards compatibility
+ pass
+
+ def __lt__(self, other):
+ if not hasattr(other, "handler_order"):
+ # Try to preserve the old behavior of having custom classes
+ # inserted after default ones (works only for custom user
+ # classes which are not aware of handler_order).
+ return True
+ return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+ """Process HTTP error responses."""
+ handler_order = 1000 # after all other processing
+
+ def http_response(self, request, response):
+ code, msg, hdrs = response.code, response.msg, response.info()
+
+ # According to RFC 2616, "2xx" code indicates that the client's
+ # request was successfully received, understood, and accepted.
+ if not (200 <= code < 300):
+ response = self.parent.error(
+ 'http', request, response, code, msg, hdrs)
+
+ return response
+
+ https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+ def http_error_default(self, req, fp, code, msg, hdrs):
+ raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+ # maximum number of redirections to any single URL
+ # this is needed because of the state that cookies introduce
+ max_repeats = 4
+ # maximum total number of redirections (regardless of URL) before
+ # assuming we're in a loop
+ max_redirections = 10
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ """Return a Request or None in response to a redirect.
+
+ This is called by the http_error_30x methods when a
+ redirection response is received. If a redirection should
+ take place, return a new Request to allow http_error_30x to
+ perform the redirect. Otherwise, raise HTTPError if no-one
+ else should try to handle this url. Return None if you can't
+ but another Handler might.
+ """
+ m = req.get_method()
+ if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+ or code in (301, 302, 303) and m == "POST"):
+ # Strictly (according to RFC 2616), 301 or 302 in response
+ # to a POST MUST NOT cause a redirection without confirmation
+ # from the user (of urllib2, in this case). In practice,
+ # essentially all clients do redirect in this case, so we
+ # do the same.
+ # be conciliant with URIs containing a space
+ newurl = newurl.replace(' ', '%20')
+ newheaders = dict((k,v) for k,v in req.headers.items()
+ if k.lower() not in ("content-length", "content-type")
+ )
+ return Request(newurl,
+ headers=newheaders,
+ origin_req_host=req.get_origin_req_host(),
+ unverifiable=True)
+ else:
+ raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+ # Implementation note: To avoid the server sending us into an
+ # infinite loop, the request object needs to track what URLs we
+ # have already seen. Do this by adding a handler-specific
+ # attribute to the Request object.
+ def http_error_302(self, req, fp, code, msg, headers):
+ # Some servers (incorrectly) return multiple Location headers
+ # (so probably same goes for URI). Use first header.
+ if 'location' in headers:
+ newurl = headers.getheaders('location')[0]
+ elif 'uri' in headers:
+ newurl = headers.getheaders('uri')[0]
+ else:
+ return
+
+ # fix a possible malformed URL
+ urlparts = urlparse.urlparse(newurl)
+ if not urlparts.path:
+ urlparts = list(urlparts)
+ urlparts[2] = "/"
+ newurl = urlparse.urlunparse(urlparts)
+
+ newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+ # XXX Probably want to forget about the state of the current
+ # request, although that might interact poorly with other
+ # handlers that also use handler-specific request attributes
+ new = self.redirect_request(req, fp, code, msg, headers, newurl)
+ if new is None:
+ return
+
+ # loop detection
+ # .redirect_dict has a key url if url was previously visited.
+ if hasattr(req, 'redirect_dict'):
+ visited = new.redirect_dict = req.redirect_dict
+ if (visited.get(newurl, 0) >= self.max_repeats or
+ len(visited) >= self.max_redirections):
+ raise HTTPError(req.get_full_url(), code,
+ self.inf_msg + msg, headers, fp)
+ else:
+ visited = new.redirect_dict = req.redirect_dict = {}
+ visited[newurl] = visited.get(newurl, 0) + 1
+
+ # Don't close the fp until we are sure that we won't use it
+ # with HTTPError.
+ fp.read()
+ fp.close()
+
+ return self.parent.open(new, timeout=req.timeout)
+
+ http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+ inf_msg = "The HTTP server returned a redirect error that would " \
+ "lead to an infinite loop.\n" \
+ "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+ """Return (scheme, user, password, host/port) given a URL or an authority.
+
+ If a URL is supplied, it must have an authority (host:port) component.
+ According to RFC 3986, having an authority component means the URL must
+ have two slashes after the scheme:
+
+ >>> _parse_proxy('file:/ftp.example.com/')
+ Traceback (most recent call last):
+ ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+ The first three items of the returned tuple may be None.
+
+ Examples of authority parsing:
+
+ >>> _parse_proxy('proxy.example.com')
+ (None, None, None, 'proxy.example.com')
+ >>> _parse_proxy('proxy.example.com:3128')
+ (None, None, None, 'proxy.example.com:3128')
+
+ The authority component may optionally include userinfo (assumed to be
+ username:password):
+
+ >>> _parse_proxy('joe:password at proxy.example.com')
+ (None, 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('joe:password at proxy.example.com:3128')
+ (None, 'joe', 'password', 'proxy.example.com:3128')
+
+ Same examples, but with URLs instead:
+
+ >>> _parse_proxy('http://proxy.example.com/')
+ ('http', None, None, 'proxy.example.com')
+ >>> _parse_proxy('http://proxy.example.com:3128/')
+ ('http', None, None, 'proxy.example.com:3128')
+ >>> _parse_proxy('http://joe:password@proxy.example.com/')
+ ('http', 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('http://joe:password@proxy.example.com:3128')
+ ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+ Everything after the authority is ignored:
+
+ >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')
+ ('ftp', 'joe', 'password', 'proxy.example.com')
+
+ Test for no trailing '/' case:
+
+ >>> _parse_proxy('http://joe:password@proxy.example.com')
+ ('http', 'joe', 'password', 'proxy.example.com')
+
+ """
+ scheme, r_scheme = splittype(proxy)
+ if not r_scheme.startswith("/"):
+ # authority
+ scheme = None
+ authority = proxy
+ else:
+ # URL
+ if not r_scheme.startswith("//"):
+ raise ValueError("proxy URL with no authority: %r" % proxy)
+ # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+ # and 3.3.), path is empty or starts with '/'
+ end = r_scheme.find("/", 2)
+ if end == -1:
+ end = None
+ authority = r_scheme[2:end]
+ userinfo, hostport = splituser(authority)
+ if userinfo is not None:
+ user, password = splitpasswd(userinfo)
+ else:
+ user = password = None
+ return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+ # Proxies must be in front
+ handler_order = 100
+
+ def __init__(self, proxies=None):
+ if proxies is None:
+ proxies = getproxies()
+ assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+ self.proxies = proxies
+ for type, url in proxies.items():
+ setattr(self, '%s_open' % type,
+ lambda r, proxy=url, type=type, meth=self.proxy_open: \
+ meth(r, proxy, type))
+
+ def proxy_open(self, req, proxy, type):
+ orig_type = req.get_type()
+ proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+ if proxy_type is None:
+ proxy_type = orig_type
+
+ if req.host and proxy_bypass(req.host):
+ return None
+
+ if user and password:
+ user_pass = '%s:%s' % (unquote(user), unquote(password))
+ creds = base64.b64encode(user_pass).strip()
+ req.add_header('Proxy-authorization', 'Basic ' + creds)
+ hostport = unquote(hostport)
+ req.set_proxy(hostport, proxy_type)
+
+ if orig_type == proxy_type or orig_type == 'https':
+ # let other handlers take care of it
+ return None
+ else:
+ # need to start over, because the other handlers don't
+ # grok the proxy's URL type
+ # e.g. if we have a constructor arg proxies like so:
+ # {'http': 'ftp://proxy.example.com'}, we may end up turning
+ # a request for http://acme.example.com/a into one for
+ # ftp://proxy.example.com/a
+ return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+ def __init__(self):
+ self.passwd = {}
+
+ def add_password(self, realm, uri, user, passwd):
+ # uri could be a single URI or a sequence
+ if isinstance(uri, basestring):
+ uri = [uri]
+ if not realm in self.passwd:
+ self.passwd[realm] = {}
+ for default_port in True, False:
+ reduced_uri = tuple(
+ [self.reduce_uri(u, default_port) for u in uri])
+ self.passwd[realm][reduced_uri] = (user, passwd)
+
+ def find_user_password(self, realm, authuri):
+ domains = self.passwd.get(realm, {})
+ for default_port in True, False:
+ reduced_authuri = self.reduce_uri(authuri, default_port)
+ for uris, authinfo in domains.iteritems():
+ for uri in uris:
+ if self.is_suburi(uri, reduced_authuri):
+ return authinfo
+ return None, None
+
+ def reduce_uri(self, uri, default_port=True):
+ """Accept authority or URI and extract only the authority and path."""
+ # note HTTP URLs do not have a userinfo component
+ parts = urlparse.urlsplit(uri)
+ if parts[1]:
+ # URI
+ scheme = parts[0]
+ authority = parts[1]
+ path = parts[2] or '/'
+ else:
+ # host or host:port
+ scheme = None
+ authority = uri
+ path = '/'
+ host, port = splitport(authority)
+ if default_port and port is None and scheme is not None:
+ dport = {"http": 80,
+ "https": 443,
+ }.get(scheme)
+ if dport is not None:
+ authority = "%s:%d" % (host, dport)
+ return authority, path
+
+ def is_suburi(self, base, test):
+ """Check if test is below base in a URI tree
+
+ Both args must be URIs in reduced form.
+ """
+ if base == test:
+ return True
+ if base[0] != test[0]:
+ return False
+ common = posixpath.commonprefix((base[1], test[1]))
+ if len(common) == len(base[1]):
+ return True
+ return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+ def find_user_password(self, realm, authuri):
+ user, password = HTTPPasswordMgr.find_user_password(self, realm,
+ authuri)
+ if user is not None:
+ return user, password
+ return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+ # XXX this allows for multiple auth-schemes, but will stupidly pick
+ # the last one with a realm specified.
+
+ # allow for double- and single-quoted realm values
+ # (single quotes are a violation of the RFC, but appear in the wild)
+ rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+ 'realm=(["\'])(.*?)\\2', re.I)
+
+ # XXX could pre-emptively send auth info already accepted (RFC 2617,
+ # end of section 2, and section 1.2 immediately after "credentials"
+ # production).
+
+ def __init__(self, password_mgr=None):
+ if password_mgr is None:
+ password_mgr = HTTPPasswordMgr()
+ self.passwd = password_mgr
+ self.add_password = self.passwd.add_password
+ self.retried = 0
+
+ def reset_retry_count(self):
+ self.retried = 0
+
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ # host may be an authority (without userinfo) or a URL with an
+ # authority
+ # XXX could be multiple headers
+ authreq = headers.get(authreq, None)
+
+ if self.retried > 5:
+ # retry sending the username:password 5 times before failing.
+ raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+
+ if authreq:
+ mo = AbstractBasicAuthHandler.rx.search(authreq)
+ if mo:
+ scheme, quote, realm = mo.groups()
+ if scheme.lower() == 'basic':
+ response = self.retry_http_basic_auth(host, req, realm)
+ if response and response.code != 401:
+ self.retried = 0
+ return response
+
+ def retry_http_basic_auth(self, host, req, realm):
+ user, pw = self.passwd.find_user_password(realm, host)
+ if pw is not None:
+ raw = "%s:%s" % (user, pw)
+ auth = 'Basic %s' % base64.b64encode(raw).strip()
+ if req.headers.get(self.auth_header, None) == auth:
+ return None
+ req.add_unredirected_header(self.auth_header, auth)
+ return self.parent.open(req, timeout=req.timeout)
+ else:
+ return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Authorization'
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ url = req.get_full_url()
+ response = self.http_error_auth_reqed('www-authenticate',
+ url, req, headers)
+ self.reset_retry_count()
+ return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Proxy-authorization'
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ # http_error_auth_reqed requires that there is no userinfo component in
+ # authority. Assume there isn't one, since urllib2 does not (and
+ # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+ # userinfo.
+ authority = req.get_host()
+ response = self.http_error_auth_reqed('proxy-authenticate',
+ authority, req, headers)
+ self.reset_retry_count()
+ return response
+
+
+def randombytes(n):
+ """Return n random bytes."""
+ # Use /dev/urandom if it is available. Fall back to random module
+ # if not. It might be worthwhile to extend this function to use
+ # other platform-specific mechanisms for getting random bytes.
+ if os.path.exists("/dev/urandom"):
+ f = open("/dev/urandom")
+ s = f.read(n)
+ f.close()
+ return s
+ else:
+ L = [chr(random.randrange(0, 256)) for i in range(n)]
+ return "".join(L)
+
+class AbstractDigestAuthHandler:
+ # Digest authentication is specified in RFC 2617.
+
+ # XXX The client does not inspect the Authentication-Info header
+ # in a successful response.
+
+ # XXX It should be possible to test this implementation against
+ # a mock server that just generates a static set of challenges.
+
+ # XXX qop="auth-int" supports is shaky
+
+ def __init__(self, passwd=None):
+ if passwd is None:
+ passwd = HTTPPasswordMgr()
+ self.passwd = passwd
+ self.add_password = self.passwd.add_password
+ self.retried = 0
+ self.nonce_count = 0
+ self.last_nonce = None
+
+ def reset_retry_count(self):
+ self.retried = 0
+
+ def http_error_auth_reqed(self, auth_header, host, req, headers):
+ authreq = headers.get(auth_header, None)
+ if self.retried > 5:
+ # Don't fail endlessly - if we failed once, we'll probably
+ # fail a second time. Hm. Unless the Password Manager is
+ # prompting for the information. Crap. This isn't great
+ # but it's better than the current 'repeat until recursion
+ # depth exceeded' approach <wink>
+ raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+ if authreq:
+ scheme = authreq.split()[0]
+ if scheme.lower() == 'digest':
+ return self.retry_http_digest_auth(req, authreq)
+
+ def retry_http_digest_auth(self, req, auth):
+ token, challenge = auth.split(' ', 1)
+ chal = parse_keqv_list(parse_http_list(challenge))
+ auth = self.get_authorization(req, chal)
+ if auth:
+ auth_val = 'Digest %s' % auth
+ if req.headers.get(self.auth_header, None) == auth_val:
+ return None
+ req.add_unredirected_header(self.auth_header, auth_val)
+ resp = self.parent.open(req, timeout=req.timeout)
+ return resp
+
+ def get_cnonce(self, nonce):
+ # The cnonce-value is an opaque
+ # quoted string value provided by the client and used by both client
+ # and server to avoid chosen plaintext attacks, to provide mutual
+ # authentication, and to provide some message integrity protection.
+ # This isn't a fabulous effort, but it's probably Good Enough.
+ dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+ randombytes(8))).hexdigest()
+ return dig[:16]
+
+ def get_authorization(self, req, chal):
+ try:
+ realm = chal['realm']
+ nonce = chal['nonce']
+ qop = chal.get('qop')
+ algorithm = chal.get('algorithm', 'MD5')
+ # mod_digest doesn't send an opaque, even though it isn't
+ # supposed to be optional
+ opaque = chal.get('opaque', None)
+ except KeyError:
+ return None
+
+ H, KD = self.get_algorithm_impls(algorithm)
+ if H is None:
+ return None
+
+ user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+ if user is None:
+ return None
+
+ # XXX not implemented yet
+ if req.has_data():
+ entdig = self.get_entity_digest(req.get_data(), chal)
+ else:
+ entdig = None
+
+ A1 = "%s:%s:%s" % (user, realm, pw)
+ A2 = "%s:%s" % (req.get_method(),
+ # XXX selector: what about proxies and full urls
+ req.get_selector())
+ if qop == 'auth':
+ if nonce == self.last_nonce:
+ self.nonce_count += 1
+ else:
+ self.nonce_count = 1
+ self.last_nonce = nonce
+
+ ncvalue = '%08x' % self.nonce_count
+ cnonce = self.get_cnonce(nonce)
+ noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+ respdig = KD(H(A1), noncebit)
+ elif qop is None:
+ respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+ else:
+ # XXX handle auth-int.
+ raise URLError("qop '%s' is not supported." % qop)
+
+ # XXX should the partial digests be encoded too?
+
+ base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+ 'response="%s"' % (user, realm, nonce, req.get_selector(),
+ respdig)
+ if opaque:
+ base += ', opaque="%s"' % opaque
+ if entdig:
+ base += ', digest="%s"' % entdig
+ base += ', algorithm="%s"' % algorithm
+ if qop:
+ base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+ return base
+
+ def get_algorithm_impls(self, algorithm):
+ # algorithm should be case-insensitive according to RFC2617
+ algorithm = algorithm.upper()
+ # lambdas assume digest modules are imported at the top level
+ if algorithm == 'MD5':
+ H = lambda x: hashlib.md5(x).hexdigest()
+ elif algorithm == 'SHA':
+ H = lambda x: hashlib.sha1(x).hexdigest()
+ # XXX MD5-sess
+ KD = lambda s, d: H("%s:%s" % (s, d))
+ return H, KD
+
+ def get_entity_digest(self, data, chal):
+ # XXX not implemented yet
+ return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+ """An authentication protocol defined by RFC 2069
+
+ Digest authentication improves on basic authentication because it
+ does not transmit passwords in the clear.
+ """
+
+ auth_header = 'Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ host = urlparse.urlparse(req.get_full_url())[1]
+ retry = self.http_error_auth_reqed('www-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+ auth_header = 'Proxy-Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ host = req.get_host()
+ retry = self.http_error_auth_reqed('proxy-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+ def __init__(self, debuglevel=0):
+ self._debuglevel = debuglevel
+
+ def set_http_debuglevel(self, level):
+ self._debuglevel = level
+
+ def do_request_(self, request):
+ host = request.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ if request.has_data(): # POST
+ data = request.get_data()
+ if not request.has_header('Content-type'):
+ request.add_unredirected_header(
+ 'Content-type',
+ 'application/x-www-form-urlencoded')
+ if not request.has_header('Content-length'):
+ request.add_unredirected_header(
+ 'Content-length', '%d' % len(data))
+
+ sel_host = host
+ if request.has_proxy():
+ scheme, sel = splittype(request.get_selector())
+ sel_host, sel_path = splithost(sel)
+
+ if not request.has_header('Host'):
+ request.add_unredirected_header('Host', sel_host)
+ for name, value in self.parent.addheaders:
+ name = name.capitalize()
+ if not request.has_header(name):
+ request.add_unredirected_header(name, value)
+
+ return request
+
+ def do_open(self, http_class, req):
+ """Return an addinfourl object for the request, using http_class.
+
+ http_class must implement the HTTPConnection API from httplib.
+ The addinfourl return value is a file-like object. It also
+ has methods and attributes including:
+ - info(): return a mimetools.Message object for the headers
+ - geturl(): return the original request URL
+ - code: HTTP status code
+ """
+ host = req.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ h = http_class(host, timeout=req.timeout) # will parse host:port
+ h.set_debuglevel(self._debuglevel)
+
+ headers = dict(req.unredirected_hdrs)
+ headers.update(dict((k, v) for k, v in req.headers.items()
+ if k not in headers))
+
+ # We want to make an HTTP/1.1 request, but the addinfourl
+ # class isn't prepared to deal with a persistent connection.
+ # It will try to read all remaining data from the socket,
+ # which will block while the server waits for the next request.
+ # So make sure the connection gets closed after the (only)
+ # request.
+ headers["Connection"] = "close"
+ headers = dict(
+ (name.title(), val) for name, val in headers.items())
+
+ if req._tunnel_host:
+ tunnel_headers = {}
+ proxy_auth_hdr = "Proxy-Authorization"
+ if proxy_auth_hdr in headers:
+ tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+ # Proxy-Authorization should not be sent to origin
+ # server.
+ del headers[proxy_auth_hdr]
+ h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+ try:
+ h.request(req.get_method(), req.get_selector(), req.data, headers)
+ try:
+ r = h.getresponse(buffering=True)
+ except TypeError: #buffering kw not supported
+ r = h.getresponse()
+ except socket.error, err: # XXX what error?
+ h.close()
+ raise URLError(err)
+
+ # Pick apart the HTTPResponse object to get the addinfourl
+ # object initialized properly.
+
+ # Wrap the HTTPResponse object in socket's file object adapter
+ # for Windows. That adapter calls recv(), so delegate recv()
+ # to read(). This weird wrapping allows the returned object to
+ # have readline() and readlines() methods.
+
+ # XXX It might be better to extract the read buffering code
+ # out of socket._fileobject() and into a base class.
+
+ r.recv = r.read
+ fp = socket._fileobject(r, close=True)
+
+ resp = addinfourl(fp, r.msg, req.get_full_url())
+ resp.code = r.status
+ resp.msg = r.reason
+ return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+ def http_open(self, req):
+ return self.do_open(httplib.HTTPConnection, req)
+
+ http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+ class HTTPSHandler(AbstractHTTPHandler):
+
+ def https_open(self, req):
+ return self.do_open(httplib.HTTPSConnection, req)
+
+ https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+ def __init__(self, cookiejar=None):
+ import cookielib
+ if cookiejar is None:
+ cookiejar = cookielib.CookieJar()
+ self.cookiejar = cookiejar
+
+ def http_request(self, request):
+ self.cookiejar.add_cookie_header(request)
+ return request
+
+ def http_response(self, request, response):
+ self.cookiejar.extract_cookies(response, request)
+ return response
+
+ https_request = http_request
+ https_response = http_response
+
+class UnknownHandler(BaseHandler):
+ def unknown_open(self, req):
+ type = req.get_type()
+ raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+ """Parse list of key=value strings where keys are not duplicated."""
+ parsed = {}
+ for elt in l:
+ k, v = elt.split('=', 1)
+ if v[0] == '"' and v[-1] == '"':
+ v = v[1:-1]
+ parsed[k] = v
+ return parsed
+
+def parse_http_list(s):
+ """Parse lists as described by RFC 2068 Section 2.
+
+ In particular, parse comma-separated lists where the elements of
+ the list may include quoted-strings. A quoted-string could
+ contain a comma. A non-quoted string could have quotes in the
+ middle. Neither commas nor quotes count if they are escaped.
+ Only double-quotes count, not single-quotes.
+ """
+ res = []
+ part = ''
+
+ escape = quote = False
+ for cur in s:
+ if escape:
+ part += cur
+ escape = False
+ continue
+ if quote:
+ if cur == '\\':
+ escape = True
+ continue
+ elif cur == '"':
+ quote = False
+ part += cur
+ continue
+
+ if cur == ',':
+ res.append(part)
+ part = ''
+ continue
+
+ if cur == '"':
+ quote = True
+
+ part += cur
+
+ # append last part
+ if part:
+ res.append(part)
+
+ return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+ try:
+ return socket.gethostbyname(host)
+ except socket.gaierror:
+ return None
+
+class FileHandler(BaseHandler):
+ # Use local file or FTP depending on form of URL
+ def file_open(self, req):
+ url = req.get_selector()
+ if url[:2] == '//' and url[2:3] != '/' and (req.host and
+ req.host != 'localhost'):
+ req.type = 'ftp'
+ return self.parent.open(req)
+ else:
+ return self.open_local_file(req)
+
+ # names for the localhost
+ names = None
+ def get_names(self):
+ if FileHandler.names is None:
+ try:
+ FileHandler.names = tuple(
+ socket.gethostbyname_ex('localhost')[2] +
+ socket.gethostbyname_ex(socket.gethostname())[2])
+ except socket.gaierror:
+ FileHandler.names = (socket.gethostbyname('localhost'),)
+ return FileHandler.names
+
+ # not entirely sure what the rules are here
+ def open_local_file(self, req):
+ import email.utils
+ import mimetypes
+ host = req.get_host()
+ filename = req.get_selector()
+ localfile = url2pathname(filename)
+ try:
+ stats = os.stat(localfile)
+ size = stats.st_size
+ modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+ mtype = mimetypes.guess_type(filename)[0]
+ headers = mimetools.Message(StringIO(
+ 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+ (mtype or 'text/plain', size, modified)))
+ if host:
+ host, port = splitport(host)
+ if not host or \
+ (not port and _safe_gethostbyname(host) in self.get_names()):
+ if host:
+ origurl = 'file://' + host + filename
+ else:
+ origurl = 'file://' + filename
+ return addinfourl(open(localfile, 'rb'), headers, origurl)
+ except OSError, msg:
+ # urllib2 users shouldn't expect OSErrors coming from urlopen()
+ raise URLError(msg)
+ raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+ def ftp_open(self, req):
+ import ftplib
+ import mimetypes
+ host = req.get_host()
+ if not host:
+ raise URLError('ftp error: no host given')
+ host, port = splitport(host)
+ if port is None:
+ port = ftplib.FTP_PORT
+ else:
+ port = int(port)
+
+ # username/password handling
+ user, host = splituser(host)
+ if user:
+ user, passwd = splitpasswd(user)
+ else:
+ passwd = None
+ host = unquote(host)
+ user = user or ''
+ passwd = passwd or ''
+
+ try:
+ host = socket.gethostbyname(host)
+ except socket.error, msg:
+ raise URLError(msg)
+ path, attrs = splitattr(req.get_selector())
+ dirs = path.split('/')
+ dirs = map(unquote, dirs)
+ dirs, file = dirs[:-1], dirs[-1]
+ if dirs and not dirs[0]:
+ dirs = dirs[1:]
+ try:
+ fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+ type = file and 'I' or 'D'
+ for attr in attrs:
+ attr, value = splitvalue(attr)
+ if attr.lower() == 'type' and \
+ value in ('a', 'A', 'i', 'I', 'd', 'D'):
+ type = value.upper()
+ fp, retrlen = fw.retrfile(file, type)
+ headers = ""
+ mtype = mimetypes.guess_type(req.get_full_url())[0]
+ if mtype:
+ headers += "Content-type: %s\n" % mtype
+ if retrlen is not None and retrlen >= 0:
+ headers += "Content-length: %d\n" % retrlen
+ sf = StringIO(headers)
+ headers = mimetools.Message(sf)
+ return addinfourl(fp, headers, req.get_full_url())
+ except ftplib.all_errors, msg:
+ raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+## fw.ftp.set_debuglevel(1)
+ return fw
+
+class CacheFTPHandler(FTPHandler):
+ # XXX would be nice to have pluggable cache strategies
+ # XXX this stuff is definitely not thread safe
+ def __init__(self):
+ self.cache = {}
+ self.timeout = {}
+ self.soonest = 0
+ self.delay = 60
+ self.max_conns = 16
+
+ def setTimeout(self, t):
+ self.delay = t
+
+ def setMaxConns(self, m):
+ self.max_conns = m
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ key = user, host, port, '/'.join(dirs), timeout
+ if key in self.cache:
+ self.timeout[key] = time.time() + self.delay
+ else:
+ self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+ self.timeout[key] = time.time() + self.delay
+ self.check_cache()
+ return self.cache[key]
+
+ def check_cache(self):
+ # first check for old ones
+ t = time.time()
+ if self.soonest <= t:
+ for k, v in self.timeout.items():
+ if v < t:
+ self.cache[k].close()
+ del self.cache[k]
+ del self.timeout[k]
+ self.soonest = min(self.timeout.values())
+
+ # then check the size
+ if len(self.cache) == self.max_conns:
+ for k, v in self.timeout.items():
+ if v == self.soonest:
+ del self.cache[k]
+ del self.timeout[k]
+ break
+ self.soonest = min(self.timeout.values())
diff --git a/lib-python/2.7/uuid.py b/lib-python/modified-2.7/uuid.py
copy from lib-python/2.7/uuid.py
copy to lib-python/modified-2.7/uuid.py
--- a/lib-python/2.7/uuid.py
+++ b/lib-python/modified-2.7/uuid.py
@@ -406,8 +406,12 @@
continue
if hasattr(lib, 'uuid_generate_random'):
_uuid_generate_random = lib.uuid_generate_random
+ _uuid_generate_random.argtypes = [ctypes.c_char * 16]
+ _uuid_generate_random.restype = None
if hasattr(lib, 'uuid_generate_time'):
_uuid_generate_time = lib.uuid_generate_time
+ _uuid_generate_time.argtypes = [ctypes.c_char * 16]
+ _uuid_generate_time.restype = None
# The uuid_generate_* functions are broken on MacOS X 10.5, as noted
# in issue #8621 the function generates the same sequence of values
@@ -436,6 +440,9 @@
lib = None
_UuidCreate = getattr(lib, 'UuidCreateSequential',
getattr(lib, 'UuidCreate', None))
+ if _UuidCreate is not None:
+ _UuidCreate.argtypes = [ctypes.c_char * 16]
+ _UuidCreate.restype = ctypes.c_int
except:
pass
diff --git a/lib_pypy/_codecs_cn.py b/lib_pypy/_codecs_cn.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_cn.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+# 'gb2312', 'gbk', 'gb18030', 'hz'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_hk.py b/lib_pypy/_codecs_hk.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_hk.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+# 'big5hkscs'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_iso2022.py b/lib_pypy/_codecs_iso2022.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_iso2022.py
@@ -0,0 +1,8 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+# 'iso2022_kr', 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2',
+# 'iso2022_jp_2004', 'iso2022_jp_3', 'iso2022_jp_ext'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_jp.py b/lib_pypy/_codecs_jp.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_jp.py
@@ -0,0 +1,8 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+# 'shift_jis', 'cp932', 'euc_jp', 'shift_jis_2004',
+# 'euc_jis_2004', 'euc_jisx0213', 'shift_jisx0213'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_kr.py b/lib_pypy/_codecs_kr.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_kr.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+# 'euc_kr', 'cp949', 'johab'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_codecs_tw.py b/lib_pypy/_codecs_tw.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_codecs_tw.py
@@ -0,0 +1,7 @@
+# this getcodec() function supports any multibyte codec, although
+# for compatibility with CPython it should only be used for the
+# codecs from this module, i.e.:
+#
+# 'big5', 'cp950'
+
+from _multibytecodec import __getcodec as getcodec
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -379,12 +379,14 @@
class defaultdict(dict):
def __init__(self, *args, **kwds):
- self.default_factory = None
- if 'default_factory' in kwds:
- self.default_factory = kwds.pop('default_factory')
- elif len(args) > 0 and (callable(args[0]) or args[0] is None):
- self.default_factory = args[0]
+ if len(args) > 0:
+ default_factory = args[0]
args = args[1:]
+ if not callable(default_factory) and default_factory is not None:
+ raise TypeError("first argument must be callable")
+ else:
+ default_factory = None
+ self.default_factory = default_factory
super(defaultdict, self).__init__(*args, **kwds)
def __missing__(self, key):
@@ -404,7 +406,7 @@
recurse.remove(id(self))
def copy(self):
- return type(self)(self, default_factory=self.default_factory)
+ return type(self)(self.default_factory, self)
def __copy__(self):
return self.copy()
diff --git a/lib_pypy/_ctypes/__init__.py b/lib_pypy/_ctypes/__init__.py
--- a/lib_pypy/_ctypes/__init__.py
+++ b/lib_pypy/_ctypes/__init__.py
@@ -18,7 +18,16 @@
if _os.name in ("nt", "ce"):
from _rawffi import FormatError
from _rawffi import check_HRESULT as _check_HRESULT
- CopyComPointer = None # XXX
+
+ def CopyComPointer(src, dst):
+ from ctypes import c_void_p, cast
+ if src:
+ hr = src[0][0].AddRef(src)
+ if hr & 0x80000000:
+ return hr
+ dst[0] = cast(src, c_void_p).value
+ return 0
+
LoadLibrary = dlopen
from _rawffi import FUNCFLAG_STDCALL, FUNCFLAG_CDECL, FUNCFLAG_PYTHONAPI
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -208,6 +208,9 @@
def _get_buffer_value(self):
return self._buffer.buffer
+ def _to_ffi_param(self):
+ return self._get_buffer_value()
+
ARRAY_CACHE = {}
def create_array_type(base, length):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -1,5 +1,6 @@
import _rawffi
+import _ffi
import sys
keepalive_key = str # XXX fix this when provided with test
@@ -46,6 +47,16 @@
else:
return self.from_param(as_parameter)
+ def get_ffi_param(self, value):
+ cdata = self.from_param(value)
+ return cdata, cdata._to_ffi_param()
+
+ def get_ffi_argtype(self):
+ if self._ffiargtype:
+ return self._ffiargtype
+ self._ffiargtype = _shape_to_ffi_type(self._ffiargshape)
+ return self._ffiargtype
+
def _CData_output(self, resbuffer, base=None, index=-1):
#assert isinstance(resbuffer, _rawffi.ArrayInstance)
"""Used when data exits ctypes and goes into user code.
@@ -99,6 +110,7 @@
"""
__metaclass__ = _CDataMeta
_objects = None
+ _ffiargtype = None
def __init__(self, *args, **kwds):
raise TypeError("%s has no type" % (type(self),))
@@ -119,11 +131,20 @@
def _get_buffer_value(self):
return self._buffer[0]
+ def _to_ffi_param(self):
+ if self.__class__._is_pointer_like():
+ return self._get_buffer_value()
+ else:
+ return self.value
+
def __buffer__(self):
return buffer(self._buffer)
def _get_b_base(self):
- return self._base
+ try:
+ return self._base
+ except AttributeError:
+ return None
_b_base_ = property(_get_b_base)
_b_needsfree_ = False
@@ -146,11 +167,12 @@
return tp._alignmentofinstances()
def byref(cdata):
- from ctypes import pointer
+ # "pointer" is imported at the end of this module to avoid circular
+ # imports
return pointer(cdata)
def cdata_from_address(self, address):
- # fix the address, in case it's unsigned
+ # fix the address: turn it into as unsigned, in case it's a negative number
address = address & (sys.maxint * 2 + 1)
instance = self.__new__(self)
lgt = getattr(self, '_length_', 1)
@@ -159,3 +181,54 @@
def addressof(tp):
return tp._buffer.buffer
+
+
+# ----------------------------------------------------------------------
+
+def is_struct_shape(shape):
+ # see the corresponding code to set the shape in
+ # _ctypes.structure._set_shape
+ return (isinstance(shape, tuple) and
+ len(shape) == 2 and
+ isinstance(shape[0], _rawffi.Structure) and
+ shape[1] == 1)
+
+def _shape_to_ffi_type(shape):
+ try:
+ return _shape_to_ffi_type.typemap[shape]
+ except KeyError:
+ pass
+ if is_struct_shape(shape):
+ return shape[0].get_ffi_type()
+ #
+ assert False, 'unknown shape %s' % (shape,)
+
+
+_shape_to_ffi_type.typemap = {
+ 'c' : _ffi.types.char,
+ 'b' : _ffi.types.sbyte,
+ 'B' : _ffi.types.ubyte,
+ 'h' : _ffi.types.sshort,
+ 'u' : _ffi.types.unichar,
+ 'H' : _ffi.types.ushort,
+ 'i' : _ffi.types.sint,
+ 'I' : _ffi.types.uint,
+ 'l' : _ffi.types.slong,
+ 'L' : _ffi.types.ulong,
+ 'q' : _ffi.types.slonglong,
+ 'Q' : _ffi.types.ulonglong,
+ 'f' : _ffi.types.float,
+ 'd' : _ffi.types.double,
+ 's' : _ffi.types.void_p,
+ 'P' : _ffi.types.void_p,
+ 'z' : _ffi.types.void_p,
+ 'O' : _ffi.types.void_p,
+ 'Z' : _ffi.types.void_p,
+ 'X' : _ffi.types.void_p,
+ 'v' : _ffi.types.sshort,
+ '?' : _ffi.types.ubyte,
+ }
+
+
+# used by "byref"
+from _ctypes.pointer import pointer
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,12 +1,15 @@
+
+from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
+from _ctypes.primitive import SimpleType, _SimpleCData
+from _ctypes.basics import ArgumentError, keepalive_key
+from _ctypes.basics import is_struct_shape
+from _ctypes.builtin import set_errno, set_last_error
import _rawffi
+import _ffi
import sys
import traceback
import warnings
-from _ctypes.basics import ArgumentError, keepalive_key
-from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
-from _ctypes.builtin import set_errno, set_last_error
-from _ctypes.primitive import SimpleType
# XXX this file needs huge refactoring I fear
@@ -24,6 +27,7 @@
WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
+
def get_com_error(errcode, riid, pIunk):
"Win32 specific: build a COM Error exception"
# XXX need C support code
@@ -36,6 +40,7 @@
funcptr.restype = int
return funcptr(*args)
+
class CFuncPtrType(_CDataMeta):
# XXX write down here defaults and such things
@@ -50,6 +55,7 @@
from_address = cdata_from_address
+
class CFuncPtr(_CData):
__metaclass__ = CFuncPtrType
@@ -65,12 +71,12 @@
callable = None
_ptr = None
_buffer = None
+ _address = None
# win32 COM properties
_paramflags = None
_com_index = None
_com_iid = None
-
- __restype_set = False
+ _is_fastpath = False
def _getargtypes(self):
return self._argtypes_
@@ -85,9 +91,14 @@
raise TypeError(
"item %d in _argtypes_ has no from_param method" % (
i + 1,))
- self._argtypes_ = argtypes
+ self._argtypes_ = list(argtypes)
+ self._check_argtypes_for_fastpath()
+ argtypes = property(_getargtypes, _setargtypes)
- argtypes = property(_getargtypes, _setargtypes)
+ def _check_argtypes_for_fastpath(self):
+ if all([hasattr(argtype, '_ffiargshape') for argtype in self._argtypes_]):
+ fastpath_cls = make_fastpath_subclass(self.__class__)
+ fastpath_cls.enable_fastpath_maybe(self)
def _getparamflags(self):
return self._paramflags
@@ -133,11 +144,11 @@
paramflags = property(_getparamflags, _setparamflags)
+
def _getrestype(self):
return self._restype_
def _setrestype(self, restype):
- self.__restype_set = True
self._ptr = None
if restype is int:
from ctypes import c_int
@@ -146,27 +157,24 @@
callable(restype)):
raise TypeError("restype must be a type, a callable, or None")
self._restype_ = restype
-
+
def _delrestype(self):
self._ptr = None
del self._restype_
-
+
restype = property(_getrestype, _setrestype, _delrestype)
def _geterrcheck(self):
return getattr(self, '_errcheck_', None)
-
More information about the pypy-commit
mailing list