[pypy-commit] pypy default: Fixed up multibytecodec on windows. With much help from armin and amaury.

[alex_gaynor]

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r44108:99f7b410912d
Date: 2011-05-12 15:27 -0400
http://bitbucket.org/pypy/pypy/changeset/99f7b410912d/

Log:	Fixed up multibytecodec on windows. With much help from armin and
	amaury.

diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -18,28 +18,6 @@
 
 srcdir = py.path.local(pypydir).join('translator', 'c')
 
-eci = ExternalCompilationInfo(
-    separate_module_files = [
-        srcdir.join('src', 'cjkcodecs', '_codecs_cn.c'),
-        srcdir.join('src', 'cjkcodecs', '_codecs_hk.c'),
-        srcdir.join('src', 'cjkcodecs', '_codecs_iso2022.c'),
-        srcdir.join('src', 'cjkcodecs', '_codecs_jp.c'),
-        srcdir.join('src', 'cjkcodecs', '_codecs_kr.c'),
-        srcdir.join('src', 'cjkcodecs', '_codecs_tw.c'),
-        srcdir.join('src', 'cjkcodecs', 'multibytecodec.c'),
-    ],
-    includes = ['src/cjkcodecs/multibytecodec.h'],
-    include_dirs = [str(srcdir)],
-)
-
-MBERR_TOOSMALL = -1  # insufficient output buffer space
-MBERR_TOOFEW   = -2  # incomplete input buffer
-MBERR_INTERNAL = -3  # internal runtime error
-MBERR_NOMEMORY = -4  # out of memory
-
-MULTIBYTECODEC_P = rffi.COpaquePtr('struct MultibyteCodec_s',
-                                   compilation_info=eci)
-
 codecs = [
     # _codecs_cn
     'gb2312', 'gbk', 'gb18030', 'hz',
@@ -60,7 +38,38 @@
 
     # _codecs_tw
     'big5', 'cp950',
-    ]
+]
+
+eci = ExternalCompilationInfo(
+    separate_module_files = [
+        srcdir.join('src', 'cjkcodecs', '_codecs_cn.c'),
+        srcdir.join('src', 'cjkcodecs', '_codecs_hk.c'),
+        srcdir.join('src', 'cjkcodecs', '_codecs_iso2022.c'),
+        srcdir.join('src', 'cjkcodecs', '_codecs_jp.c'),
+        srcdir.join('src', 'cjkcodecs', '_codecs_kr.c'),
+        srcdir.join('src', 'cjkcodecs', '_codecs_tw.c'),
+        srcdir.join('src', 'cjkcodecs', 'multibytecodec.c'),
+    ],
+    includes = ['src/cjkcodecs/multibytecodec.h'],
+    include_dirs = [str(srcdir)],
+    export_symbols = [
+        "pypy_cjk_dec_init", "pypy_cjk_dec_free", "pypy_cjk_dec_chunk",
+        "pypy_cjk_dec_outbuf", "pypy_cjk_dec_outlen",
+        "pypy_cjk_dec_inbuf_remaining", "pypy_cjk_dec_inbuf_consumed",
+
+        "pypy_cjk_enc_init", "pypy_cjk_enc_free", "pypy_cjk_enc_chunk",
+        "pypy_cjk_enc_reset", "pypy_cjk_enc_outbuf", "pypy_cjk_enc_outlen",
+        "pypy_cjk_enc_inbuf_remaining", "pypy_cjk_enc_inbuf_consumed",
+    ] + ["pypy_cjkcodec_%s" % codec for codec in codecs],
+)
+
+MBERR_TOOSMALL = -1  # insufficient output buffer space
+MBERR_TOOFEW   = -2  # incomplete input buffer
+MBERR_INTERNAL = -3  # internal runtime error
+MBERR_NOMEMORY = -4  # out of memory
+
+MULTIBYTECODEC_P = rffi.COpaquePtr('struct MultibyteCodec_s',
+                                   compilation_info=eci)
 
 def llexternal(*args, **kwds):
     kwds.setdefault('compilation_info', eci)
@@ -156,7 +165,6 @@
 
 # ____________________________________________________________
 # Encoding
-
 ENCODEBUF_P = rffi.COpaquePtr('struct pypy_cjk_enc_s', compilation_info=eci)
 pypy_cjk_enc_init = llexternal('pypy_cjk_enc_init',
                                [MULTIBYTECODEC_P, rffi.CWCHARP, rffi.SSIZE_T],
@@ -194,7 +202,7 @@
                 assert False
             src = pypy_cjk_enc_outbuf(encodebuf)
             length = pypy_cjk_enc_outlen(encodebuf)
-            return string_from_raw(src, length)
+            return rffi.charpsize2str(src, length)
         #
         finally:
             pypy_cjk_enc_free(encodebuf)
@@ -220,18 +228,3 @@
     end = start + esize
     if 1:  # errors == ERROR_STRICT:
         raise EncodeDecodeError(start, end, reason)
-
-def string_from_raw(src, length):
-    result = lltype.malloc(STR, length)
-    try:
-        str_chars_offset = (rffi.offsetof(STR, 'chars') + \
-                            rffi.itemoffsetof(STR.chars, 0))
-        dest = rffi.cast_ptr_to_adr(result) + str_chars_offset
-        src = rffi.cast_ptr_to_adr(src) + rffi.itemoffsetof(rffi.CCHARP.TO)
-        rffi.raw_memcopy(src, dest,
-                         llmemory.sizeof(lltype.Char) * length)
-        got = hlstr(result)
-    finally:
-        keepalive_until_here(result)
-    assert got is not None
-    return got
diff --git a/pypy/translator/c/src/cjkcodecs/multibytecodec.h b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
@@ -5,12 +5,27 @@
 
 #include <stddef.h>
 #include <stdint.h>
-#include <unistd.h>
 #include <assert.h>
 
+#ifndef _WIN32
+#include <unistd.h>
+#endif
+
+#ifdef _WIN64
+typedef __int64 ssize_t
+#else
+#ifdef _WIN32
+typedef int ssize_t;
+#endif
+#endif
+
 #ifndef Py_UNICODE_SIZE
-#define Py_UNICODE_SIZE  4
-typedef uint32_t Py_UNICODE;
+#ifdef _WIN32
+#define Py_UNICODE_SIZE 2
+#else
+#define Py_UNICODE_SIZE 4
+#endif
+typedef wchar_t Py_UNICODE;
 typedef ssize_t Py_ssize_t;
 #define PY_SSIZE_T_MAX   ((Py_ssize_t)(((size_t) -1) >> 1))
 #endif