[pypy-svn] r68761 - in pypy/branch/logging/pypy/rlib: . test

arigo at codespeak.net arigo at codespeak.net
Mon Oct 26 17:51:41 CET 2009


Author: arigo
Date: Mon Oct 26 17:51:41 2009
New Revision: 68761

Modified:
   pypy/branch/logging/pypy/rlib/rlog.py
   pypy/branch/logging/pypy/rlib/test/test_rlog.py
Log:
Implement the log, for now only on lltype.


Modified: pypy/branch/logging/pypy/rlib/rlog.py
==============================================================================
--- pypy/branch/logging/pypy/rlib/rlog.py	(original)
+++ pypy/branch/logging/pypy/rlib/rlog.py	Mon Oct 26 17:51:41 2009
@@ -1,6 +1,7 @@
 import py, os, time
 from pypy.tool.ansi_print import ansi_log
 from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.rarithmetic import r_uint
 from pypy.rpython.extregistry import ExtRegistryEntry
 
 _log = py.log.Producer("rlog") 
@@ -27,6 +28,7 @@
             logcategories = translator._logcategories
         except AttributeError:
             logcategories = translator._logcategories = {}
+            translator._logwriter = None
         try:
             cat = logcategories[s_category.const]
         except KeyError:
@@ -43,14 +45,17 @@
     def specialize_call(self, hop, **kwds_i):
         from pypy.rpython.lltypesystem import lltype
         translator = self.bookkeeper.annotator.translator
-        logcategories = translator._logcategories
-        cat = logcategories[hop.args_s[0].const]
+        logwriter = translator._logwriter
+        if logwriter is None:
+            logwriter = translator._logwriter = LLLogWriter()
+            logwriter._register(hop.rtyper)
+        cat = translator._logcategories[hop.args_s[0].const]
         args_v = []
         for name, typechar in cat.entries:
             assert typechar == 'd'
             args_v.append(hop.inputarg(lltype.Signed, arg=kwds_i[name]))
         hop.exception_cannot_occur()
-        hop.gendirectcall(cat.call, *args_v)
+        hop.gendirectcall(cat.gen_call(logwriter), *args_v)
 
 # ____________________________________________________________
 
@@ -72,28 +77,33 @@
                 "duplicate name %r in the log message %r" % (name, message))
             seen[name] = True
             self.entries.append((name, typechar))
+        self.call = None
 
-    def get_call(self, logwriter):
-        types = [typechar for name, typechar in self.entries]
-        types = unrolling_iterable(types)
-        #
-        def call(*args):
-            if logwriter.enabled:
-                logwriter.add_entry(self)
-                i = 0
-                for typechar in types:
-                    methname = 'add_subentry_' + typechar
-                    getattr(logwriter, methname)(args[i])
-                    i = i + 1
-        call._always_inline_ = True
-        return call
+    def gen_call(self, logwriter):
+        if self.call is None:
+            self.logwriter = logwriter
+            types = [typechar for name, typechar in self.entries]
+            types = unrolling_iterable(types)
+            #
+            def call(*args):
+                if logwriter.enabled:
+                    logwriter.add_entry(self)
+                    i = 0
+                    for typechar in types:
+                        methname = 'add_subentry_' + typechar
+                        getattr(logwriter, methname)(args[i])
+                        i = i + 1
+            call._always_inline_ = True
+            self.call = call
+        else:
+            assert self.logwriter is logwriter
+        return self.call
 
     def _freeze_(self):
         return True
 
 
 class AbstractLogWriter(object):
-    BUFSIZE = 8192
 
     def __init__(self):
         self.enabled = True
@@ -117,6 +127,8 @@
             self.create_buffer()
             for c in 'RLog\n':
                 self.write_int(ord(c))
+            self.write_int(-1)
+            self.write_int(0)
         self.initialized_file = True
 
     def define_new_category(self, cat):
@@ -126,6 +138,7 @@
         if self.enabled:
             self.write_int(0)
             self.write_int(cat.index)
+            self.write_str(cat.category)
             self.write_str(cat.message)
         self.initialized_index[cat.index] = None
 
@@ -134,32 +147,88 @@
             self.define_new_category(cat)
         if self.enabled:
             self.write_int(cat.index)
-    add_entry._dont_inline_ = True
 
     def add_subentry_d(self, num):
         if self.enabled:
             self.write_int(num)
-    add_subentry_d._dont_inline_ = True
 
     def add_subentry_s(self, str):
         if self.enabled:
             self.write_str(str)
-    add_subentry_d._dont_inline_ = True
+
+# ____________________________________________________________
 
 
 class LLLogWriter(AbstractLogWriter):
+    BUFSIZE = 8192
+
+    def do_write(self, fd, buf, size):
+        "NOT_RPYTHON"
+        l = [buf[i] for i in range(size)]
+        s = ''.join(l)
+        os.write(fd, s)
+        self.writecount += 1
+
+    def _register(self, rtyper):
+        from pypy.rpython.lltypesystem import rffi
+        from pypy.rpython.module.ll_os import underscore_on_windows
+        # attach 'do_write' to self, overwriting the NOT_RPYTHON method
+        def do_write(fd, buf, size):
+            os_write(rffi.cast(rffi.INT, fd),
+                     buf,
+                     rffi.cast(rffi.SIZE_T), size)
+            self.writecount += 1
+        self.do_write = do_write
+        os_write = rffi.llexternal(underscore_on_windows+'write',
+                                   [rffi.INT, rffi.CHARP, rffi.SIZE_T],
+                                   rffi.SIZE_T)
+        # register flush() to be called at program exit
+        def flush_log_cache():
+            if self.initialized_file:
+                self._flush()
+        annhelper = rtyper.getannmixlevel()
+        annhelper.register_atexit(flush_log_cache)
 
     def create_buffer(self):
-        self.buffer = lltype.malloc(rffi.CCHARP.TO, LogWriter.BUFSIZE,
-                                    flavor='raw')
+        from pypy.rpython.lltypesystem import lltype, rffi
+        self.buffer = lltype.malloc(rffi.CCHARP.TO, self.BUFSIZE, flavor='raw')
         self.buffer_position = 0
+        self.writecount = 0
 
     def write_int(self, n):
-        yyy
+        self._write_int_noflush(n)
+        if self.buffer_position > self.BUFSIZE-48:
+            self._flush()
+
+    def _write_int_noflush(self, n):
+        p = self.buffer_position
+        buf = self.buffer
+        n = r_uint(n)
+        while n > 0x7F:
+            buf[p] = chr((n & 0x7F) | 0x80)
+            p += 1
+            n >>= 7
+        buf[p] = chr(n)
+        self.buffer_position = p + 1
 
     def write_str(self, s):
-        zzz
+        self._write_int_noflush(len(s))
+        p = self.buffer_position
+        if p + len(s) > self.BUFSIZE-24:
+            self._flush()
+            os.write(self.fd, s)
+            self.writecount += 1
+        else:
+            buf = self.buffer
+            for i in range(len(s)):
+                buf[p + i] = s[i]
+            self.buffer_position = p + len(s)
+
+    def _flush(self):
+        if self.buffer_position > 0:
+            self.do_write(self.fd, self.buffer, self.buffer_position)
+        self.buffer_position = 0
 
-    def flush(self):
-        if self.initialized_file:
-            xxx
+    def _close(self):
+        self._flush()
+        os.close(self.fd)

Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py
==============================================================================
--- pypy/branch/logging/pypy/rlib/test/test_rlog.py	(original)
+++ pypy/branch/logging/pypy/rlib/test/test_rlog.py	Mon Oct 26 17:51:41 2009
@@ -1,4 +1,5 @@
 from pypy.rlib import rlog
+from pypy.rlib.rarithmetic import intmask
 from pypy.tool.udir import udir
 
 
@@ -40,24 +41,25 @@
 
 def test_logwriter():
     class FakeCategory:
-        def __init__(self, index, message):
+        def __init__(self, index, category, message):
             self.index = index
+            self.category = category
             self.message = message
     #
     logwriter = MyLogWriter()
-    cat5 = FakeCategory(5, "foobar")
-    cat7 = FakeCategory(7, "baz")
+    cat5 = FakeCategory(5, "F5", "foobar")
+    cat7 = FakeCategory(7, "F7", "baz")
     logwriter.add_entry(cat5)
     logwriter.add_entry(cat5)
     logwriter.add_entry(cat7)
     logwriter.add_entry(cat5)
     #
     assert logwriter.content == [
-        ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'),
-        0, 5, "foobar",
+        ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 0,
+        0, 5, "F5", "foobar",
         5,
         5,
-        0, 7, "baz",
+        0, 7, "F7", "baz",
         7,
         5]
 
@@ -65,12 +67,103 @@
     message = "abc%(foo)ddef%(bar)sghi"
     cat = rlog.LogCategory("Aa", message, 17)
     logwriter = MyLogWriter()
-    call = cat.get_call(logwriter)
+    call = cat.gen_call(logwriter)
     call(515, "hellooo")
     call(2873, "woooooorld")
     #
     assert logwriter.content == [
-        ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'),
-        0, 17, message,
+        ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 0,
+        0, 17, "Aa", message,
         17, 515, "hellooo",
         17, 2873, "woooooorld"]
+
+
+class TestLLLogWriter:
+    COUNTER = 0
+
+    def open(self):
+        path = udir.join('test_rlog.lllogwriter%d' % TestLLLogWriter.COUNTER)
+        self.path = path
+        TestLLLogWriter.COUNTER += 1
+        #
+        class MyLLLogWriter(rlog.LLLogWriter):
+            def get_filename(self):
+                return str(path)
+        #
+        logwriter = MyLLLogWriter()
+        logwriter.open_file()
+        return logwriter
+
+    def read_uint(self, f):
+        shift = 0
+        result = 0
+        lastbyte = ord(f.read(1))
+        while lastbyte & 0x80:
+            result |= ((lastbyte & 0x7F) << shift)
+            shift += 7
+            lastbyte = ord(f.read(1))
+        result |= (lastbyte << shift)
+        return result
+
+    def check(self, expected):
+        f = self.path.open('rb')
+        f.seek(0, 2)
+        totalsize = f.tell()
+        f.seek(0, 0)
+        header = f.read(5)
+        assert header == 'RLog\n'
+        for expect in [-1, 0] + expected:
+            if isinstance(expect, int):
+                result = self.read_uint(f)
+                assert intmask(result) == expect
+            elif isinstance(expect, str):
+                length = self.read_uint(f)
+                assert length < totalsize
+                got = f.read(length)
+                assert got == expect
+            else:
+                assert 0, expect
+        moredata = f.read(10)
+        assert not moredata
+
+    def test_write_int(self):
+        logwriter = self.open()
+        for i in range(logwriter.BUFSIZE):
+            logwriter.write_int(i)
+        logwriter._close()
+        self.check(range(logwriter.BUFSIZE))
+        assert logwriter.writecount <= 3
+
+    def test_write_str(self):
+        logwriter = self.open()
+        slist = map(str, range(logwriter.BUFSIZE))
+        for s in slist:
+            logwriter.write_str(s)
+        logwriter._close()
+        self.check(slist)
+        assert logwriter.writecount <= 14
+
+    def test_write_mixed(self):
+        logwriter = self.open()
+        xlist = []
+        for i in range(logwriter.BUFSIZE):
+            if i & 1:
+                i = str(i)
+            xlist.append(i)
+        for x in xlist:
+            if isinstance(x, int):
+                logwriter.write_int(x)
+            else:
+                logwriter.write_str(x)
+        logwriter._close()
+        self.check(xlist)
+        assert logwriter.writecount <= 7
+
+    def test_write_long_str(self):
+        logwriter = self.open()
+        slist = ['abcdefg' * n for n in [10, 100, 1000, 10000]]
+        for s in slist:
+            logwriter.write_str(s)
+        logwriter._close()
+        self.check(slist)
+        assert logwriter.writecount <= 9



More information about the Pypy-commit mailing list