[Python-checkins] r56728 - in python/branches/release25-maint: Lib/test/test_codecmaps_cn.py Lib/test/test_multibytecodec_support.py Misc/NEWS Modules/cjkcodecs/_codecs_cn.c

[hyeshik.chang]

Author: hyeshik.chang
Date: Sat Aug  4 06:15:04 2007
New Revision: 56728

Modified:
   python/branches/release25-maint/Lib/test/test_codecmaps_cn.py
   python/branches/release25-maint/Lib/test/test_multibytecodec_support.py
   python/branches/release25-maint/Misc/NEWS
   python/branches/release25-maint/Modules/cjkcodecs/_codecs_cn.c
Log:
Backport from trunk r56727:
Fix gb18030 codec's bug that doesn't map two-byte characters on
GB18030 extension in encoding. (bug reported by Bjorn Stabell)


Modified: python/branches/release25-maint/Lib/test/test_codecmaps_cn.py
==============================================================================
--- python/branches/release25-maint/Lib/test/test_codecmaps_cn.py	(original)
+++ python/branches/release25-maint/Lib/test/test_codecmaps_cn.py	Sat Aug  4 06:15:04 2007
@@ -19,10 +19,18 @@
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/' \
                  'MICSFT/WINDOWS/CP936.TXT'
 
+class TestGB18030Map(test_multibytecodec_support.TestBase_Mapping,
+                     unittest.TestCase):
+    encoding = 'gb18030'
+    mapfileurl = 'http://source.icu-project.org/repos/icu/data/' \
+                 'trunk/charset/data/xml/gb-18030-2000.xml'
+
+
 def test_main():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(TestGB2312Map))
     suite.addTest(unittest.makeSuite(TestGBKMap))
+    suite.addTest(unittest.makeSuite(TestGB18030Map))
     test_support.run_suite(suite)
 
 if __name__ == "__main__":

Modified: python/branches/release25-maint/Lib/test/test_multibytecodec_support.py
==============================================================================
--- python/branches/release25-maint/Lib/test/test_multibytecodec_support.py	(original)
+++ python/branches/release25-maint/Lib/test/test_multibytecodec_support.py	Sat Aug  4 06:15:04 2007
@@ -5,7 +5,7 @@
 #
 
 import sys, codecs, os.path
-import unittest
+import unittest, re
 from test import test_support
 from StringIO import StringIO
 
@@ -272,6 +272,12 @@
         return test_support.open_urlresource(self.mapfileurl)
 
     def test_mapping_file(self):
+        if self.mapfileurl.endswith('.xml'):
+            self._test_mapping_file_ucm()
+        else:
+            self._test_mapping_file_plain()
+
+    def _test_mapping_file_plain(self):
         unichrs = lambda s: u''.join(map(unichr, map(eval, s.split('+'))))
         urt_wa = {}
 
@@ -303,6 +309,14 @@
 
             self._testpoint(csetch, unich)
 
+    def _test_mapping_file_ucm(self):
+        ucmdata = self.open_mapping_file().read()
+        uc = re.findall('<a u="([A-F0-9]{4})" b="([0-9A-F ]+)"/>', ucmdata)
+        for uni, coded in uc:
+            unich = unichr(int(uni, 16))
+            codech = ''.join(chr(int(c, 16)) for c in coded.split())
+            self._testpoint(codech, unich)
+
     def test_mapping_supplemental(self):
         for mapping in self.supmaps:
             self._testpoint(*mapping)

Modified: python/branches/release25-maint/Misc/NEWS
==============================================================================
--- python/branches/release25-maint/Misc/NEWS	(original)
+++ python/branches/release25-maint/Misc/NEWS	Sat Aug  4 06:15:04 2007
@@ -26,6 +26,9 @@
 Library
 -------
 
+- GB18030 codec now can encode additional two-byte characters that
+  are missing in GBK.
+
 - Bug #1704793: Raise KeyError if unicodedata.lookup cannot
   represent the result in a single character.
 

Modified: python/branches/release25-maint/Modules/cjkcodecs/_codecs_cn.c
==============================================================================
--- python/branches/release25-maint/Modules/cjkcodecs/_codecs_cn.c	(original)
+++ python/branches/release25-maint/Modules/cjkcodecs/_codecs_cn.c	Sat Aug  4 06:15:04 2007
@@ -197,6 +197,7 @@
 		REQUIRE_OUTBUF(2)
 
 		GBK_ENCODE(c, code)
+		else TRYMAP_ENC(gb18030ext, code, c);
 		else {
 			const struct _gb18030_to_unibmp_ranges *utrrange;