[Python-3000-checkins] r58876 - in python/branches/py3k-pep3137: Doc/library/exceptions.rst Doc/library/warnings.rst Include/pydebug.h Include/pyerrors.h Lib/test/exception_hierarchy.txt Lib/test/test_bytes.py Lib/test/test_unicode.py Misc/NEWS Modules/main.c Objects/bytesobject.c Objects/exceptions.c Objects/stringobject.c Python/pythonrun.c

christian.heimes python-3000-checkins at python.org
Tue Nov 6 12:52:33 CET 2007 

Author: christian.heimes
Date: Tue Nov  6 12:52:32 2007
New Revision: 58876

Modified:
   python/branches/py3k-pep3137/Doc/library/exceptions.rst
   python/branches/py3k-pep3137/Doc/library/warnings.rst
   python/branches/py3k-pep3137/Include/pydebug.h
   python/branches/py3k-pep3137/Include/pyerrors.h
   python/branches/py3k-pep3137/Lib/test/exception_hierarchy.txt
   python/branches/py3k-pep3137/Lib/test/test_bytes.py
   python/branches/py3k-pep3137/Lib/test/test_unicode.py
   python/branches/py3k-pep3137/Misc/NEWS
   python/branches/py3k-pep3137/Modules/main.c
   python/branches/py3k-pep3137/Objects/bytesobject.c
   python/branches/py3k-pep3137/Objects/exceptions.c
   python/branches/py3k-pep3137/Objects/stringobject.c
   python/branches/py3k-pep3137/Python/pythonrun.c
Log:
Applied my patch #1392 to issue warnings or errors for certain operations that involve str and byte/buffer operations like str(bytes()) or str() == bytes(). The code paths are only enabled with the new argument -b.

* Added class BytesWarning(Warning)
* Added -b/-bb command line args and Py_BytesWarningFlag
* issue PyErr_WarnEx(PyExc_BytesWarning) when Py_BytesWarningFlag is set.
* Added warning filter for BytesWarning that raises an exception if
  Py_BytesWarningFlag > 1

Please run the regression tests with -bb to catch problems. I've added filters around some test cases to ignore the operations when it's appropriate.

Modified: python/branches/py3k-pep3137/Doc/library/exceptions.rst
==============================================================================
--- python/branches/py3k-pep3137/Doc/library/exceptions.rst	(original)
+++ python/branches/py3k-pep3137/Doc/library/exceptions.rst	Tue Nov  6 12:52:32 2007
@@ -405,6 +405,10 @@
 
    Base class for warnings related to Unicode.
 
+.. exception:: BytesWarning
+
+   Base class for warnings related to bytes and buffer.
+
 The class hierarchy for built-in exceptions is:
 
 

Modified: python/branches/py3k-pep3137/Doc/library/warnings.rst
==============================================================================
--- python/branches/py3k-pep3137/Doc/library/warnings.rst	(original)
+++ python/branches/py3k-pep3137/Doc/library/warnings.rst	Tue Nov  6 12:52:32 2007
@@ -80,6 +80,10 @@
 | :exc:`UnicodeWarning`            | Base category for warnings related to         |
 |                                  | Unicode.                                      |
 +----------------------------------+-----------------------------------------------+
+| :exc:`BytesWarning`              | Base category for warnings related to         |
+|                                  | bytes and buffer.                             |
++----------------------------------+-----------------------------------------------+
+
 
 While these are technically built-in exceptions, they are documented here,
 because conceptually they belong to the warnings mechanism.

Modified: python/branches/py3k-pep3137/Include/pydebug.h
==============================================================================
--- python/branches/py3k-pep3137/Include/pydebug.h	(original)
+++ python/branches/py3k-pep3137/Include/pydebug.h	Tue Nov  6 12:52:32 2007
@@ -11,6 +11,7 @@
 PyAPI_DATA(int) Py_InspectFlag;
 PyAPI_DATA(int) Py_OptimizeFlag;
 PyAPI_DATA(int) Py_NoSiteFlag;
+PyAPI_DATA(int) Py_BytesWarningFlag;
 PyAPI_DATA(int) Py_UseClassExceptionsFlag;
 PyAPI_DATA(int) Py_FrozenFlag;
 PyAPI_DATA(int) Py_TabcheckFlag;

Modified: python/branches/py3k-pep3137/Include/pyerrors.h
==============================================================================
--- python/branches/py3k-pep3137/Include/pyerrors.h	(original)
+++ python/branches/py3k-pep3137/Include/pyerrors.h	Tue Nov  6 12:52:32 2007
@@ -165,6 +165,7 @@
 PyAPI_DATA(PyObject *) PyExc_FutureWarning;
 PyAPI_DATA(PyObject *) PyExc_ImportWarning;
 PyAPI_DATA(PyObject *) PyExc_UnicodeWarning;
+PyAPI_DATA(PyObject *) PyExc_BytesWarning;
 
 
 /* Convenience functions */

Modified: python/branches/py3k-pep3137/Lib/test/exception_hierarchy.txt
==============================================================================
--- python/branches/py3k-pep3137/Lib/test/exception_hierarchy.txt	(original)
+++ python/branches/py3k-pep3137/Lib/test/exception_hierarchy.txt	Tue Nov  6 12:52:32 2007
@@ -44,5 +44,6 @@
            +-- SyntaxWarning
            +-- UserWarning
            +-- FutureWarning
-	   +-- ImportWarning
-	   +-- UnicodeWarning
+           +-- ImportWarning
+           +-- UnicodeWarning
+           +-- BytesWarning

Modified: python/branches/py3k-pep3137/Lib/test/test_bytes.py
==============================================================================
--- python/branches/py3k-pep3137/Lib/test/test_bytes.py	(original)
+++ python/branches/py3k-pep3137/Lib/test/test_bytes.py	Tue Nov  6 12:52:32 2007
@@ -12,6 +12,7 @@
 import pickle
 import tempfile
 import unittest
+import warnings
 import test.test_support
 import test.string_tests
 import test.buffer_tests
@@ -19,6 +20,12 @@
 
 class BytesTest(unittest.TestCase):
 
+    def setUp(self):
+        self.warning_filters = warnings.filters[:]
+
+    def tearDown(self):
+        warnings.filters = self.warning_filters
+
     def test_basics(self):
         b = buffer()
         self.assertEqual(type(b), buffer)
@@ -87,6 +94,7 @@
         self.assertRaises(ValueError, buffer, [10**100])
 
     def test_repr_str(self):
+        warnings.simplefilter('ignore', BytesWarning)
         for f in str, repr:
             self.assertEqual(f(buffer()), "buffer(b'')")
             self.assertEqual(f(buffer([0])), "buffer(b'\\x00')")
@@ -149,6 +157,8 @@
         self.assertEqual(bytes(b"abc") < b"ab", False)
         self.assertEqual(bytes(b"abc") <= b"ab", False)
 
+    def test_compare_to_str(self):
+        warnings.simplefilter('ignore', BytesWarning)
         # Byte comparisons with unicode should always fail!
         # Test this for all expected byte orders and Unicode character sizes
         self.assertEqual(b"\0a\0b\0c" == "abc", False)
@@ -371,6 +381,7 @@
         self.assertEqual(b, buffer(sample))
 
     def test_to_str(self):
+        warnings.simplefilter('ignore', BytesWarning)
         self.assertEqual(str(b''), "b''")
         self.assertEqual(str(b'x'), "b'x'")
         self.assertEqual(str(b'\x80'), "b'\\x80'")

Modified: python/branches/py3k-pep3137/Lib/test/test_unicode.py
==============================================================================
--- python/branches/py3k-pep3137/Lib/test/test_unicode.py	(original)
+++ python/branches/py3k-pep3137/Lib/test/test_unicode.py	Tue Nov  6 12:52:32 2007
@@ -6,7 +6,11 @@
 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
 
 """#"
-import unittest, sys, struct, codecs, new
+import codecs
+import struct
+import sys
+import unittest
+import warnings
 from test import test_support, string_tests
 
 # Error handling (bad decoder return)
@@ -34,6 +38,12 @@
     ):
     type2test = str
 
+    def setUp(self):
+        self.warning_filters = warnings.filters[:]
+
+    def tearDown(self):
+        warnings.filters = self.warning_filters
+
     def checkequalnofix(self, result, object, methodname, *args):
         method = getattr(object, methodname)
         realresult = method(*args)
@@ -205,6 +215,7 @@
         self.assertRaises(TypeError, 'replace'.replace, "r", 42)
 
     def test_bytes_comparison(self):
+        warnings.simplefilter('ignore', BytesWarning)
         self.assertEqual('abc' == b'abc', False)
         self.assertEqual('abc' != b'abc', True)
         self.assertEqual('abc' == buffer(b'abc'), False)

Modified: python/branches/py3k-pep3137/Misc/NEWS
==============================================================================
--- python/branches/py3k-pep3137/Misc/NEWS	(original)
+++ python/branches/py3k-pep3137/Misc/NEWS	Tue Nov  6 12:52:32 2007
@@ -31,6 +31,9 @@
 - io.open() and _fileio.FileIO have grown a new argument closefd. A false
   value disables the closing of the file descriptor.
 
+- Added a new option -b to issues warnings (-bb for errors) about certain
+  operations between bytes/buffer and str like str(b'') and comparsion.
+
 Extension Modules
 -----------------
 

Modified: python/branches/py3k-pep3137/Modules/main.c
==============================================================================
--- python/branches/py3k-pep3137/Modules/main.c	(original)
+++ python/branches/py3k-pep3137/Modules/main.c	Tue Nov  6 12:52:32 2007
@@ -44,7 +44,7 @@
 static int  orig_argc;
 
 /* command line options */
-#define BASE_OPTS "c:dEhim:OStuvVW:xX?"
+#define BASE_OPTS "bc:dEhim:OStuvVW:xX?"
 
 #define PROGRAM_OPTS BASE_OPTS
 
@@ -55,32 +55,34 @@
 /* Long usage message, split into parts < 512 bytes */
 static char *usage_1 = "\
 Options and arguments (and corresponding environment variables):\n\
+-b     : issue warnings about str(bytes_instance), str(buffer_instance)\n\
+         and comparing bytes/buffer with str. (-bb: issue errors)\n\
 -c cmd : program passed in as string (terminates option list)\n\
 -d     : debug output from parser; also PYTHONDEBUG=x\n\
 -E     : ignore environment variables (such as PYTHONPATH)\n\
 -h     : print this help message and exit (also --help)\n\
--i     : inspect interactively after running script; forces a prompt even\n\
-         if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\
 ";
 static char *usage_2 = "\
+-i     : inspect interactively after running script; forces a prompt even\n\
+         if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\
 -m mod : run library module as a script (terminates option list)\n\
 -O     : optimize generated bytecode slightly; also PYTHONOPTIMIZE=x\n\
 -OO    : remove doc-strings in addition to the -O optimizations\n\
 -S     : don't imply 'import site' on initialization\n\
 -t     : issue warnings about inconsistent tab usage (-tt: issue errors)\n\
--u     : unbuffered binary stdout and stderr; also PYTHONUNBUFFERED=x\n\
 ";
 static char *usage_3 = "\
+-u     : unbuffered binary stdout and stderr; also PYTHONUNBUFFERED=x\n\
          see man page for details on internal buffering relating to '-u'\n\
 -v     : verbose (trace import statements); also PYTHONVERBOSE=x\n\
          can be supplied multiple times to increase verbosity\n\
 -V     : print the Python version number and exit (also --version)\n\
 -W arg : warning control; arg is action:message:category:module:lineno\n\
 -x     : skip first line of source, allowing use of non-Unix forms of #!cmd\n\
-file   : program read from script file\n\
--      : program read from stdin (default; interactive mode if a tty)\n\
 ";
 static char *usage_4 = "\
+file   : program read from script file\n\
+-      : program read from stdin (default; interactive mode if a tty)\n\
 arg ...: arguments passed to program in sys.argv[1:]\n\n\
 Other environment variables:\n\
 PYTHONSTARTUP: file executed on interactive startup (no default)\n\
@@ -252,6 +254,9 @@
 		}
 
 		switch (c) {
+		case 'b':
+			Py_BytesWarningFlag++;
+			break;
 
 		case 'd':
 			Py_DebugFlag++;

Modified: python/branches/py3k-pep3137/Objects/bytesobject.c
==============================================================================
--- python/branches/py3k-pep3137/Objects/bytesobject.c	(original)
+++ python/branches/py3k-pep3137/Objects/bytesobject.c	Tue Nov  6 12:52:32 2007
@@ -917,6 +917,17 @@
 }
 
 static PyObject *
+bytes_str(PyObject *op)
+{
+	if (Py_BytesWarningFlag) {
+		if (PyErr_WarnEx(PyExc_BytesWarning,
+				 "str() on a buffer instance", 1))
+			return NULL;
+	}
+	return bytes_repr((PyBytesObject*)op);
+}
+
+static PyObject *
 bytes_richcompare(PyObject *self, PyObject *other, int op)
 {
     Py_ssize_t self_size, other_size;
@@ -930,6 +941,12 @@
        error, even if the comparison is for equality. */
     if (PyObject_IsInstance(self, (PyObject*)&PyUnicode_Type) ||
         PyObject_IsInstance(other, (PyObject*)&PyUnicode_Type)) {
+        if (Py_BytesWarningFlag && op == Py_EQ) {
+            if (PyErr_WarnEx(PyExc_BytesWarning,
+                            "Comparsion between buffer and string", 1))
+                return NULL;
+        }
+
         Py_INCREF(Py_NotImplemented);
         return Py_NotImplemented;
     }
@@ -3082,7 +3099,7 @@
     &bytes_as_mapping,                  /* tp_as_mapping */
     0,                                  /* tp_hash */
     0,                                  /* tp_call */
-    (reprfunc)bytes_repr,               /* tp_str */
+    bytes_str,                          /* tp_str */
     PyObject_GenericGetAttr,            /* tp_getattro */
     0,                                  /* tp_setattro */
     &bytes_as_buffer,                   /* tp_as_buffer */

Modified: python/branches/py3k-pep3137/Objects/exceptions.c
==============================================================================
--- python/branches/py3k-pep3137/Objects/exceptions.c	(original)
+++ python/branches/py3k-pep3137/Objects/exceptions.c	Tue Nov  6 12:52:32 2007
@@ -1740,6 +1740,14 @@
     "Base class for warnings about Unicode related problems, mostly\n"
     "related to conversion problems.");
 
+/*
+ *    BytesWarning extends Warning
+ */
+SimpleExtendsException(PyExc_Warning, BytesWarning,
+    "Base class for warnings about bytes and buffer related problems, mostly\n"
+    "related to conversion from str or comparing to str.");
+
+
 
 /* Pre-computed MemoryError instance.  Best to create this as early as
  * possible and not wait until a MemoryError is actually raised!
@@ -1839,6 +1847,7 @@
     PRE_INIT(FutureWarning)
     PRE_INIT(ImportWarning)
     PRE_INIT(UnicodeWarning)
+    PRE_INIT(BytesWarning)
 
     bltinmod = PyImport_ImportModule("__builtin__");
     if (bltinmod == NULL)
@@ -1899,6 +1908,7 @@
     POST_INIT(FutureWarning)
     POST_INIT(ImportWarning)
     POST_INIT(UnicodeWarning)
+    POST_INIT(BytesWarning)
 
     PyExc_MemoryErrorInst = BaseException_new(&_PyExc_MemoryError, NULL, NULL);
     if (!PyExc_MemoryErrorInst)

Modified: python/branches/py3k-pep3137/Objects/stringobject.c
==============================================================================
--- python/branches/py3k-pep3137/Objects/stringobject.c	(original)
+++ python/branches/py3k-pep3137/Objects/stringobject.c	Tue Nov  6 12:52:32 2007
@@ -679,6 +679,17 @@
 	return PyString_Repr(op, 1);
 }
 
+static PyObject *
+string_str(PyObject *op)
+{
+	if (Py_BytesWarningFlag) {
+		if (PyErr_WarnEx(PyExc_BytesWarning,
+				 "str() on a bytes instance", 1))
+			return NULL;
+	}
+	return string_repr(op);
+}
+
 static Py_ssize_t
 string_length(PyStringObject *a)
 {
@@ -830,6 +841,15 @@
 
 	/* Make sure both arguments are strings. */
 	if (!(PyString_Check(a) && PyString_Check(b))) {
+		if (Py_BytesWarningFlag && (op == Py_EQ) &&
+		    (PyObject_IsInstance((PyObject*)a,
+					 (PyObject*)&PyUnicode_Type) ||
+		    PyObject_IsInstance((PyObject*)b,
+					 (PyObject*)&PyUnicode_Type))) {
+			if (PyErr_WarnEx(PyExc_BytesWarning,
+				    "Comparsion between bytes and string", 1))
+				return NULL;
+		}
 		result = Py_NotImplemented;
 		goto out;
 	}
@@ -3074,13 +3094,13 @@
 	0,					/* tp_getattr */
 	0,					/* tp_setattr */
 	0,					/* tp_compare */
-	string_repr, 				/* tp_repr */
+	(reprfunc)string_repr, 			/* tp_repr */
 	0,					/* tp_as_number */
 	&string_as_sequence,			/* tp_as_sequence */
 	&string_as_mapping,			/* tp_as_mapping */
 	(hashfunc)string_hash, 			/* tp_hash */
 	0,					/* tp_call */
-	string_repr,				/* tp_str */
+	string_str,				/* tp_str */
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	&string_as_buffer,			/* tp_as_buffer */

Modified: python/branches/py3k-pep3137/Python/pythonrun.c
==============================================================================
--- python/branches/py3k-pep3137/Python/pythonrun.c	(original)
+++ python/branches/py3k-pep3137/Python/pythonrun.c	Tue Nov  6 12:52:32 2007
@@ -75,6 +75,7 @@
 int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */
 int Py_InspectFlag; /* Needed to determine whether to exit at SystemError */
 int Py_NoSiteFlag; /* Suppress 'import site' */
+int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */
 int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */
 int Py_FrozenFlag; /* Needed by getpath.c */
 int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */
@@ -262,8 +263,28 @@
 #endif /* WITH_THREAD */
 
 	warnings_module = PyImport_ImportModule("warnings");
-	if (!warnings_module)
+	if (!warnings_module) {
 		PyErr_Clear();
+	}
+	else {
+		PyObject *o;
+		char *action[8];
+
+		if (Py_BytesWarningFlag > 1)
+			*action = "error";
+		else if (Py_BytesWarningFlag)
+			*action = "default";
+		else
+			*action = "ignore";
+
+		o = PyObject_CallMethod(warnings_module,
+					"simplefilter", "sO",
+					*action, PyExc_BytesWarning);
+		if (o == NULL)
+			Py_FatalError("Py_Initialize: can't initialize"
+				      "warning filter for BytesWarning.");
+		Py_DECREF(o);
+        }
 
 #if defined(HAVE_LANGINFO_H) && defined(CODESET)
 	/* On Unix, set the file system encoding according to the

More information about the Python-3000-checkins mailing list