From hpk at codespeak.net Fri Mar 2 12:00:34 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 2 Mar 2007 12:00:34 +0100 (CET) Subject: [py-svn] r39655 - in py/trunk/py/misc: . testing Message-ID: <20070302110034.8C1D210061@code0.codespeak.net> Author: hpk Date: Fri Mar 2 12:00:32 2007 New Revision: 39655 Added: py/trunk/py/misc/killproc.py (contents, props changed) py/trunk/py/misc/testing/test_oskill.py (contents, props changed) Modified: py/trunk/py/misc/conftest-socketgatewayrun.py Log: add a (somewhat clunky) way to kill processes by PID both on windows and unix-ish systems. Modified: py/trunk/py/misc/conftest-socketgatewayrun.py ============================================================================== --- py/trunk/py/misc/conftest-socketgatewayrun.py (original) +++ py/trunk/py/misc/conftest-socketgatewayrun.py Fri Mar 2 12:00:32 2007 @@ -28,8 +28,8 @@ return True class MySession(RemoteTerminalSession): - socketserveradr = ('10.9.4.148', 8888) socketserveradr = ('10.9.2.62', 8888) + socketserveradr = ('10.9.4.148', 8888) def _initslavegateway(self): print "MASTER: initializing remote socket gateway" @@ -59,3 +59,5 @@ assert remotepypath.startswith(topdir), (remotepypath, topdir) #print "remote side has rsynced pythonpath ready: %r" %(topdir,) return gw, topdir + +dist_hosts = ['localhost', 'cobra', 'cobra'] Added: py/trunk/py/misc/killproc.py ============================================================================== --- (empty file) +++ py/trunk/py/misc/killproc.py Fri Mar 2 12:00:32 2007 @@ -0,0 +1,10 @@ + +import py +import os, sys + +def killproc(pid): + if sys.platform == "win32": + py.process.cmdexec("taskkill /F /PID %d" %(pid,)) + else: + os.kill(pid, 15) + Added: py/trunk/py/misc/testing/test_oskill.py ============================================================================== --- (empty file) +++ py/trunk/py/misc/testing/test_oskill.py Fri Mar 2 12:00:32 2007 @@ -0,0 +1,15 @@ + +import py, sys + +from py.__.misc.killproc import killproc + +def test_win_killsubprocess(): + tmp = py.test.ensuretemp("test_win_killsubprocess") + t = tmp.join("t.py") + t.write("import time ; time.sleep(100)") + proc = py.std.subprocess.Popen([sys.executable, str(t)]) + assert proc.poll() is None # no return value yet + killproc(proc.pid) + ret = proc.wait() + assert ret != 0 + From fijal at codespeak.net Tue Mar 6 10:07:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 10:07:55 +0100 (CET) Subject: [py-svn] r39973 - in py/trunk/py/net: . pipe server test Message-ID: <20070306090755.37F6D1006E@code0.codespeak.net> Author: fijal Date: Tue Mar 6 10:07:53 2007 New Revision: 39973 Added: py/trunk/py/net/ (props changed) py/trunk/py/net/__init__.py (contents, props changed) py/trunk/py/net/greenexecnet.py (contents, props changed) py/trunk/py/net/greensock2.py (contents, props changed) py/trunk/py/net/msgstruct.py (contents, props changed) py/trunk/py/net/pipe/ (props changed) py/trunk/py/net/pipe/__init__.py (contents, props changed) py/trunk/py/net/pipe/common.py (contents, props changed) py/trunk/py/net/pipe/fd.py (contents, props changed) py/trunk/py/net/pipe/gsocket.py (contents, props changed) py/trunk/py/net/pipe/mp.py (contents, props changed) py/trunk/py/net/pipelayer.py (contents, props changed) py/trunk/py/net/server/ (props changed) py/trunk/py/net/server/__init__.py py/trunk/py/net/server/httpserver.py (contents, props changed) py/trunk/py/net/test/ (props changed) py/trunk/py/net/test/test_greenexecnet.py (contents, props changed) py/trunk/py/net/test/test_greensock2.py (contents, props changed) py/trunk/py/net/test/test_pipelayer.py (contents, props changed) Log: Add a (stolen almost directly from arigo/hack) network layer based on top of greenlets. Needs some tweaking, so it's not exposed as a py.xxx, but rather py.__.net Added: py/trunk/py/net/__init__.py ============================================================================== Added: py/trunk/py/net/greenexecnet.py ============================================================================== --- (empty file) +++ py/trunk/py/net/greenexecnet.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,203 @@ +import sys, os, py, inspect +from py.__.net import greensock2 +from py.__.net.msgstruct import message, decodemessage + +MSG_REMOTE_EXEC = 'r' +MSG_OBJECT = 'o' +MSG_ERROR = 'e' +MSG_CHAN_CLOSE = 'c' +MSG_FATAL = 'f' +MSG_CHANNEL = 'n' + +class Gateway(object): + + def __init__(self, input, output, is_remote=False): + self.input = input + self.output = output + self.nextchannum = int(is_remote) + self.receivers = {} + self.greenlet = greensock2.autogreenlet(self.serve_forever, is_remote) + + def remote_exec(self, remote_source): + remote_source = py.code.Source(remote_source) + chan = self.newchannel() + msg = message(MSG_REMOTE_EXEC, chan.n, str(remote_source)) + self.output.sendall(msg) + return chan + + def newchannel(self): + n = self.nextchannum + self.nextchannum += 2 + return self.make_channel(n) + + def make_channel(self, n): + giver, accepter = greensock2.meetingpoint() + assert n not in self.receivers + self.receivers[n] = giver + return Channel(self, n, accepter) + + def serve_forever(self, is_remote=False): + try: + buffer = "" + while 1: + msg, buffer = decodemessage(buffer) + if msg is None: + buffer += self.input.recv(16384) + else: + handler = HANDLERS[msg[0]] + handler(self, *msg[1:]) + except greensock2.greenlet.GreenletExit: + raise + except: + if is_remote: + msg = message(MSG_FATAL, format_error(*sys.exc_info())) + self.output.sendall(msg) + else: + raise + + def msg_remote_exec(self, n, source): + def do_execute(channel): + try: + d = {'channel': channel} + exec source in d + except: + channel.report_error(*sys.exc_info()) + else: + channel.close() + greensock2.autogreenlet(do_execute, self.make_channel(n)) + + def msg_object(self, n, objrepr): + obj = eval(objrepr) + if n in self.receivers: + self.receivers[n].give_queued(obj) + + def msg_error(self, n, s): + if n in self.receivers: + self.receivers[n].give_queued(RemoteError(s)) + self.receivers[n].close() + del self.receivers[n] + + def msg_chan_close(self, n): + if n in self.receivers: + self.receivers[n].close() + del self.receivers[n] + + def msg_channel(self, n, m): + if n in self.receivers: + self.receivers[n].give_queued(self.make_channel(m)) + + def msg_fatal(self, s): + raise RemoteError(s) + +HANDLERS = { + MSG_REMOTE_EXEC: Gateway.msg_remote_exec, + MSG_OBJECT: Gateway.msg_object, + MSG_ERROR: Gateway.msg_error, + MSG_CHAN_CLOSE: Gateway.msg_chan_close, + MSG_CHANNEL: Gateway.msg_channel, + MSG_FATAL: Gateway.msg_fatal, + } + + +class Channel(object): + + def __init__(self, gw, n, accepter): + self.gw = gw + self.n = n + self.accepter = accepter + + def send(self, obj): + if isinstance(obj, Channel): + assert obj.gw is self.gw + msg = message(MSG_CHANNEL, self.n, obj.n) + else: + msg = message(MSG_OBJECT, self.n, repr(obj)) + self.gw.output.sendall(msg) + + def receive(self): + obj = self.accepter.accept() + if isinstance(obj, RemoteError): + raise obj + else: + return obj + + def close(self): + try: + self.gw.output.sendall(message(MSG_CHAN_CLOSE, self.n)) + except OSError: + pass + + def report_error(self, exc_type, exc_value, exc_traceback=None): + s = format_error(exc_type, exc_value, exc_traceback) + try: + self.gw.output.sendall(message(MSG_ERROR, self.n, s)) + except OSError: + pass + + +class RemoteError(Exception): + pass + +def format_error(exc_type, exc_value, exc_traceback=None): + import traceback, StringIO + s = StringIO.StringIO() + traceback.print_exception(exc_type, exc_value, exc_traceback, file=s) + return s.getvalue() + + +class PopenCmdGateway(Gateway): + action = "exec input()" + + def __init__(self, cmdline): + from py.__.net.pipe.fd import FDInput, FDOutput + child_in, child_out = os.popen2(cmdline, 't', 0) + fdin = FDInput(child_out.fileno(), child_out.close) + fdout = FDOutput(child_in.fileno(), child_in.close) + fdout.sendall(self.get_bootstrap_code()) + super(PopenCmdGateway, self).__init__(input = fdin, output = fdout) + + def get_bootstrap_code(): + # XXX assumes that the py lib is installed on the remote side + src = [] + src.append('from py.__.net import greenexecnet') + src.append('greenexecnet.PopenCmdGateway.run_server()') + src.append('') + return '%r\n' % ('\n'.join(src),) + get_bootstrap_code = staticmethod(get_bootstrap_code) + + def run_server(): + from py.__.net.pipe.fd import FDInput, FDOutput + gw = Gateway(input = FDInput(os.dup(0)), + output = FDOutput(os.dup(1)), + is_remote = True) + # for now, ignore normal I/O + fd = os.open('/dev/null', os.O_RDWR) + os.dup2(fd, 0) + os.dup2(fd, 1) + os.close(fd) + greensock2.wait(gw.greenlet) + run_server = staticmethod(run_server) + +class PopenGateway(PopenCmdGateway): + def __init__(self, python=sys.executable): + cmdline = '"%s" -u -c "%s"' % (python, self.action) + super(PopenGateway, self).__init__(cmdline) + +class SshGateway(PopenCmdGateway): + def __init__(self, sshaddress, remotepython='python', identity=None): + self.sshaddress = sshaddress + remotecmd = '%s -u -c "%s"' % (remotepython, self.action) + cmdline = [sshaddress, remotecmd] + # XXX Unix style quoting + for i in range(len(cmdline)): + cmdline[i] = "'" + cmdline[i].replace("'", "'\\''") + "'" + cmd = 'ssh -C' + if identity is not None: + cmd += ' -i %s' % (identity,) + cmdline.insert(0, cmd) + super(SshGateway, self).__init__(' '.join(cmdline)) + + +##f = open('LOG', 'a') +##import os; print >> f, '[%d] READY' % (os.getpid(),) +##f.close() Added: py/trunk/py/net/greensock2.py ============================================================================== --- (empty file) +++ py/trunk/py/net/greensock2.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,526 @@ +import os, sys +try: + from stackless import greenlet +except ImportError: + import py + greenlet = py.magic.greenlet +from collections import deque +from select import select as _select +from time import time as _time +from heapq import heappush, heappop, heapify + +TRACE = True + +def meetingpoint(): + senders = deque() # list of senders, or [None] if Giver closed + receivers = deque() # list of receivers, or [None] if Receiver closed + return (MeetingPointGiver(senders, receivers), + MeetingPointAccepter(senders, receivers)) + +def producer(func, *args, **kwds): + iterable = func(*args, **kwds) + giver, accepter = meetingpoint() + def autoproducer(): + try: + giver.wait() + for obj in iterable: + giver.give(obj) + giver.wait() + finally: + giver.close() + autogreenlet(autoproducer) + return accepter + + +class MeetingPointBase(object): + + def __init__(self, senders, receivers): + self.senders = senders + self.receivers = receivers + self.g_active = g_active + + def close(self): + while self.senders: + if self.senders[0] is None: + break + packet = self.senders.popleft() + if packet.g_from is not None: + self.g_active.append(packet.g_from) + else: + self.senders.append(None) + while self.receivers: + if self.receivers[0] is None: + break + other = self.receivers.popleft() + self.g_active.append(other) + else: + self.receivers.append(None) + + __del__ = close + + def closed(self): + return self.receivers and self.receivers[0] is None + + +class MeetingPointGiver(MeetingPointBase): + + def give(self, obj): + if self.receivers: + if self.receivers[0] is None: + raise MeetingPointClosed + other = self.receivers.popleft() + g_active.append(g_getcurrent()) + packet = _Packet() + packet.payload = obj + other.switch(packet) + if not packet.accepted: + raise Interrupted("packet not accepted") + else: + packet = _Packet() + packet.g_from = g_getcurrent() + packet.payload = obj + try: + self.senders.append(packet) + g_dispatcher.switch() + if not packet.accepted: + raise Interrupted("packet not accepted") + except: + remove_by_id(self.senders, packet) + raise + + def give_queued(self, obj): + if self.receivers: + self.give(obj) + else: + packet = _Packet() + packet.g_from = None + packet.payload = obj + self.senders.append(packet) + + def ready(self): + return self.receivers and self.receivers[0] is not None + + def wait(self): + if self.receivers: + if self.receivers[0] is None: + raise MeetingPointClosed + else: + packet = _Packet() + packet.g_from = g_getcurrent() + packet.empty = True + self.senders.append(packet) + try: + g_dispatcher.switch() + if not packet.accepted: + raise Interrupted("no accepter found") + except: + remove_by_id(self.senders, packet) + raise + + def trigger(self): + if self.ready(): + self.give(None) + + +class MeetingPointAccepter(MeetingPointBase): + + def accept(self): + while self.senders: + if self.senders[0] is None: + raise MeetingPointClosed + packet = self.senders.popleft() + packet.accepted = True + if packet.g_from is not None: + g_active.append(packet.g_from) + if not packet.empty: + return packet.payload + g = g_getcurrent() + self.receivers.append(g) + try: + packet = g_dispatcher.switch() + except: + remove_by_id(self.receivers, g) + raise + if type(packet) is not _Packet: + remove_by_id(self.receivers, g) + raise Interrupted("no packet") + packet.accepted = True + return packet.payload + + def ready(self): + for packet in self.senders: + if packet is None: + return False + if not packet.empty: + return True + return False + + def wait_trigger(self, timeout=None, default=None): + if timeout is None: + return self.accept() + else: + timer = Timer(timeout) + try: + try: + return self.accept() + finally: + timer.stop() + except Interrupted: + if timer.finished: + return default + raise + + +class MeetingPointClosed(greenlet.GreenletExit): + pass + +class Interrupted(greenlet.GreenletExit): + pass + +class ConnexionClosed(greenlet.GreenletExit): + pass + +class _Packet(object): + empty = False + accepted = False + +def remove_by_id(d, obj): + lst = [x for x in d if x is not obj] + d.clear() + d.extend(lst) + +# ____________________________________________________________ + +##class Queue(object): + +## def __init__(self): +## self.giver, self.accepter = meetingpoint() +## self.pending = deque() + +## def put(self, item): # preserve the caller's atomicity +## self.pending.append(item) +## if self.accepter.ready(): +## self.accepter.accept() + +## def get(self, block=True): +## if self.pending: +## return self.pending.popleft() +## elif block: +## self.giver.give(None) +## return self.pending.popleft() +## else: +## raise Empty + +##class Empty(Interrupted): +## pass + +##class Event(object): + +## def __init__(self): +## self.giver, self.accepter = meetingpoint() + +## clear = __init__ + +## def isSet(self): +## return self.accepter is None + +## def set(self): # preserve the caller's atomicity +## if self.accepter is not None: +## accepter = self.accepter +## self.giver = self.accepter = None +## while accepter.ready(): # wake up all waiters +## accepter.accept() + +## def wait(self, timeout=None): +## if self.accepter is not None: +## if timeout is None: +## self.giver.give(None) +## else: +## timer = Timer(timeout) +## try: +## try: +## self.giver.give(None) +## except Interrupted: +## pass +## finally: +## timer.stop() + +##class Semaphore(object): + +## def __init__(self, value=1): +## self.giver, self.accepter = meetingpoint() +## for i in range(value): +## self.release() + +## def acquire(self, blocking=True): +## if blocking or self.accepter.ready(): +## return self.accepter.accept() +## else: +## return False + +## def release(self): +## autogreenlet(self.giver.put, True) + +# ____________________________________________________________ + +def wait_input(sock): + _register(g_iwtd, sock) + +def recv(sock, bufsize): + wait_input(sock) + buf = sock.recv(bufsize) + if not buf: + raise ConnexionClosed("inbound connexion closed") + return buf + +def recvall(sock, bufsize): + in_front = False + data = [] + while bufsize > 0: + _register(g_iwtd, sock, in_front=in_front) + buf = sock.recv(bufsize) + if not buf: + raise ConnexionClosed("inbound connexion closed") + data.append(buf) + bufsize -= len(buf) + in_front = True + return ''.join(data) + +def read(fd, bufsize): + assert fd >= 0 + wait_input(fd) + buf = os.read(fd, bufsize) + if not buf: + raise ConnexionClosed("inbound connexion closed") + return buf + +def readall(fd, bufsize): + assert fd >= 0 + in_front = False + data = [] + while bufsize > 0: + _register(g_iwtd, fd, in_front=in_front) + buf = os.read(fd, bufsize) + if not buf: + raise ConnexionClosed("inbound connexion closed") + data.append(buf) + bufsize -= len(buf) + in_front = True + return ''.join(data) + + +def wait_output(sock): + _register(g_owtd, sock) + +def sendall(sock, buffer): + in_front = False + while buffer: + _register(g_owtd, sock, in_front=in_front) + count = sock.send(buffer) + buffer = buffer[count:] + in_front = True + +def writeall(fd, buffer): + assert fd >= 0 + in_front = False + while buffer: + _register(g_owtd, fd, in_front=in_front) + count = os.write(fd, buffer) + if not count: + raise ConnexionClosed("outbound connexion closed") + buffer = buffer[count:] + in_front = True + + +def sleep(duration, *greenlets): + timer = Timer(duration) + try: + wait(*greenlets) + finally: + ok = timer.finished + timer.stop() + if not ok: + raise Interrupted + +def _wait(): + g_dispatcher.switch() + +def wait(*greenlets): + assert greenlets#, "should not wait without events to wait on" + current = g_getcurrent() + for g in greenlets: + if g in g_waiters: + g_waiters[g].append(current) + else: + g_waiters[g] = [current] + g_dispatcher.switch() + +class Timer(object): + started = False + finished = False + + def __init__(self, timeout): + self.g = g_getcurrent() + entry = (_time() + timeout, self) + if g_timers_mixed: + g_timers.append(entry) + else: + heappush(g_timers, entry) + + def stop(self): + global g_timers_mixed + if not self.finished: + for i, (activationtime, timer) in enumerate(g_timers): + if timer is self: + g_timers[i] = g_timers[-1] + g_timers.pop() + g_timers_mixed = True + break + self.finished = True + +# ____________________________________________________________ + +class autogreenlet(greenlet): + def __init__(self, function, *args, **kwds): + self.parent = g_dispatcher + self.function = function + self.args = args + self.kwds = kwds + g_active.append(self) + + def run(self): + self.trace("start") + try: + self.function(*self.args, **self.kwds) + except Exception, e: + self.trace("stop (%s%s)", e.__class__.__name__, + str(e) and (': '+str(e))) + raise + else: + self.trace("done") + + def __repr__(self): +## args = ', '.join([repr(s) for s in self.args] + +## ['%s=%r' % keyvalue for keyvalue in self.kwds.items()]) +## return '' % (self.function.__name__, args) + return '' % (self.function.__name__, + hex(id(self))) + + def trace(self, msg, *args): + if TRACE: + print self, msg % args + + def interrupt(self): + self.throw(Interrupted) + + +g_active = deque() +g_iwtd = {} +g_owtd = {} +g_timers = [] +g_timers_mixed = False + +g_getcurrent = greenlet.getcurrent + +def _register(g_wtd, sock, in_front=False): + d = g_wtd.setdefault(sock, deque()) + g = g_getcurrent() + if in_front: + d.appendleft(g) + else: + d.append(g) + try: + if g_dispatcher.switch() is not g_wtd: + raise Interrupted + except: + remove_by_id(d, g) + raise + +##def _unregister_timer(): +## ... + + +def check_dead_greenlets(mapping): + to_remove = [i for i, v in mapping.items() if not v] + for k in to_remove: + del mapping[k] + +def check_waiters(active): + if active in g_waiters: + for g in g_waiters[active]: + g.switch() + del g_waiters[active] + + +def dispatcher_mainloop(): + global g_timers_mixed + while 1: + try: + while g_active: + print 'active:', g_active[0] + active = g_active.popleft() + active.switch() + if active.dead: + check_waiters(active) + del active + if g_timers: + if g_timers_mixed: + heapify(g_timers) + g_timers_mixed = False + activationtime, timer = g_timers[0] + delay = activationtime - _time() + if delay <= 0.0: + if timer.started: + heappop(g_timers) + #print 'timeout:', g + timer.finished = True + timer.g.switch() + if timer.g.dead: + check_waiters(timer.g) + continue + delay = 0.0 + timer.started = True + else: + check_dead_greenlets(g_iwtd) + check_dead_greenlets(g_owtd) + if not (g_iwtd or g_owtd): + # nothing to do, switch to the main greenlet + g_dispatcher.parent.switch() + continue + delay = None + + print 'selecting...', g_iwtd.keys(), g_owtd.keys(), delay + iwtd, owtd, _ = _select(g_iwtd.keys(), g_owtd.keys(), [], delay) + print 'done' + for s in owtd: + if s in g_owtd: + d = g_owtd[s] + #print 'owtd:', d[0] + g = d.popleft() + if not d: + try: + del g_owtd[s] + except KeyError: + pass + g.switch(g_owtd) + if g.dead: + check_waiters(g) + for s in iwtd: + if s in g_iwtd: + d = g_iwtd[s] + #print 'iwtd:', d[0] + g = d.popleft() + if not d: + try: + del g_iwtd[s] + except KeyError: + pass + g.switch(g_iwtd) + if g.dead: + check_waiters(g) + except: + import sys + g_dispatcher.parent.throw(*sys.exc_info()) + +g_dispatcher = greenlet(dispatcher_mainloop) +g_waiters = {} Added: py/trunk/py/net/msgstruct.py ============================================================================== --- (empty file) +++ py/trunk/py/net/msgstruct.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,29 @@ +from struct import pack, unpack, calcsize + + +def message(tp, *values): + strtype = type('') + typecodes = [''] + for v in values: + if type(v) is strtype: + typecodes.append('%ds' % len(v)) + elif 0 <= v < 256: + typecodes.append('B') + else: + typecodes.append('l') + typecodes = ''.join(typecodes) + assert len(typecodes) < 256 + return pack(("!B%dsc" % len(typecodes)) + typecodes, + len(typecodes), typecodes, tp, *values) + +def decodemessage(data): + if data: + limit = ord(data[0]) + 1 + if len(data) >= limit: + typecodes = "!c" + data[1:limit] + end = limit + calcsize(typecodes) + if len(data) >= end: + return unpack(typecodes, data[limit:end]), data[end:] + #elif end > 1000000: + # raise OverflowError + return None, data Added: py/trunk/py/net/pipe/__init__.py ============================================================================== Added: py/trunk/py/net/pipe/common.py ============================================================================== --- (empty file) +++ py/trunk/py/net/pipe/common.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,38 @@ +import greensock2 +from pypeers.tool import log + + +class BufferedInput(object): + in_buf = '' + + def recv(self, bufsize): + self.wait_input() + buf = self.in_buf[:bufsize] + self.in_buf = self.in_buf[bufsize:] + return buf + + def recvall(self, bufsize): + result = [] + while bufsize > 0: + buf = self.recv(bufsize) + result.append(buf) + bufsize -= len(buf) + return ''.join(result) + +# ____________________________________________________________ + +def forwardpipe(s1, s2): + try: + while 1: + s2.wait_output() + buffer = s1.recv(32768) + log('[%r -> %r] %r', s1, s2, buffer) + s2.sendall(buffer) + del buffer + finally: + s2.shutdown_wr() + s1.shutdown_rd() + +def linkpipes(s1, s2): + greensock2.autogreenlet(forwardpipe, s1, s2) + greensock2.autogreenlet(forwardpipe, s2, s1) Added: py/trunk/py/net/pipe/fd.py ============================================================================== --- (empty file) +++ py/trunk/py/net/pipe/fd.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,69 @@ +import os +from py.__.net import greensock2 + + +class FDInput(object): + + def __init__(self, read_fd, close=True): + self.read_fd = read_fd + self._close = close # a flag or a callback + + def shutdown_rd(self): + fd = self.read_fd + if fd is not None: + self.read_fd = None + close = self._close + if close: + self._close = False + if close == True: + os.close(fd) + else: + close() + + __del__ = shutdown_rd + + def wait_input(self): + greensock2.wait_input(self.read_fd) + + def recv(self, bufsize): +## f = open('LOG', 'a') +## import os; print >> f, '[%d] RECV' % (os.getpid(),) +## f.close() + res = greensock2.read(self.read_fd, bufsize) +## f = open('LOG', 'a') +## import os; print >> f, '[%d] RECV %r' % (os.getpid(), res) +## f.close() + return res + + def recvall(self, bufsize): + return greensock2.readall(self.read_fd, bufsize) + + +class FDOutput(object): + + def __init__(self, write_fd, close=True): + self.write_fd = write_fd + self._close = close # a flag or a callback + + def shutdown_wr(self): + fd = self.write_fd + if fd is not None: + self.write_fd = None + close = self._close + if close: + self._close = False + if close == True: + os.close(fd) + else: + close() + + __del__ = shutdown_wr + + def wait_output(self): + greensock2.wait_output(self.write_fd) + + def sendall(self, buffer): +## f = open('LOG', 'a') +## import os; print >> f, '[%d] %r' % (os.getpid(), buffer) +## f.close() + greensock2.writeall(self.write_fd, buffer) Added: py/trunk/py/net/pipe/gsocket.py ============================================================================== --- (empty file) +++ py/trunk/py/net/pipe/gsocket.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,114 @@ +import greensock2 +import socket, errno, os + +error = socket.error + + +class _delegate(object): + def __init__(self, methname): + self.methname = methname + def __get__(self, obj, typ=None): + result = getattr(obj._s, self.methname) + setattr(obj, self.methname, result) + return result + + +class GreenSocket(object): + + def __init__(self, family = socket.AF_INET, + type = socket.SOCK_STREAM, + proto = 0): + self._s = socket.socket(family, type, proto) + self._s.setblocking(False) + + def fromsocket(cls, s): + if isinstance(s, GreenSocket): + s = s._s + result = GreenSocket.__new__(cls) + result._s = s + s.setblocking(False) + return result + fromsocket = classmethod(fromsocket) + + def accept(self): + while 1: + try: + s, addr = self._s.accept() + break + except error, e: + import pdb;pdb.set_trace() + if e.args[0] not in (errno.EAGAIN, errno.EWOULDBLOCK): + raise + self.wait_input() + return self.fromsocket(s), addr + + bind = _delegate("bind") + close = _delegate("close") + + def connect(self, addr): + err = self.connect_ex(addr) + if err: + raise error(err, os.strerror(err)) + + def connect_ex(self, addr): + err = self._s.connect_ex(addr) + if err == errno.EINPROGRESS: + greensock2.wait_output(self._s) + err = self._s.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + return err + + #XXX dup + fileno = _delegate("fileno") + getpeername = _delegate("getpeername") + getsockname = _delegate("getsockname") + getsockopt = _delegate("getsockopt") + listen = _delegate("listen") + + def makefile(self, mode='r', bufsize=-1): + # hack, but reusing the internal socket._fileobject should just work + return socket._fileobject(self, mode, bufsize) + + def recv(self, bufsize): + return greensock2.recv(self._s, bufsize) + + def recvall(self, bufsize): + return greensock2.recvall(self._s, bufsize) + + def recvfrom(self, bufsize): + self.wait_input() + buf, addr = self._s.recvfrom(bufsize) + if not buf: + raise ConnexionClosed("inbound connexion closed") + return buf, addr + + def send(self, data): + self.wait_output() + return self._s.send(data) + + def sendto(self, data, addr): + self.wait_output() + return self._s.sendto(data, addr) + + def sendall(self, data): + greensock2.sendall(self._s, data) + + setsockopt = _delegate("setsockopt") + shutdown = _delegate("shutdown") + + def shutdown_rd(self): + try: + self._s.shutdown(socket.SHUT_RD) + except error: + pass + + def shutdown_wr(self): + try: + self._s.shutdown(socket.SHUT_WR) + except error: + pass + + def wait_input(self): + greensock2.wait_input(self._s) + + def wait_output(self): + greensock2.wait_output(self._s) Added: py/trunk/py/net/pipe/mp.py ============================================================================== --- (empty file) +++ py/trunk/py/net/pipe/mp.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,29 @@ +from pypeers.stream.common import BufferedInput + + +class MeetingPointInput(BufferedInput): + + def __init__(self, accepter): + self.accepter = accepter + + def wait_input(self): + while not self.in_buf: + self.in_buf = self.accepter.accept() + + def shutdown_rd(self): + self.accepter.close() + + +class MeetingPointOutput(BufferedInput): + + def __init__(self, giver): + self.giver = giver + + def wait_output(self): + self.giver.wait() + + def sendall(self, buffer): + self.giver.give(buffer) + + def shutdown_wr(self): + self.giver.close() Added: py/trunk/py/net/pipelayer.py ============================================================================== --- (empty file) +++ py/trunk/py/net/pipelayer.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,306 @@ +#import os +import struct +from collections import deque + + +class InvalidPacket(Exception): + pass + + +FLAG_NAK1 = 0xE0 +FLAG_NAK = 0xE1 +FLAG_REG = 0xE2 +FLAG_CFRM = 0xE3 + +FLAG_RANGE_START = 0xE0 +FLAG_RANGE_STOP = 0xE4 + +max_old_packets = 256 # must be <= 256 + + +class PipeLayer(object): + timeout = 1 + headersize = 4 + + def __init__(self): + #self.localid = os.urandom(4) + #self.remoteid = None + self.cur_time = 0 + self.out_queue = deque() + self.out_nextseqid = 0 + self.out_nextrepeattime = None + self.in_nextseqid = 0 + self.in_outoforder = {} + self.out_oldpackets = deque() + self.out_flags = FLAG_REG + self.out_resend = 0 + self.out_resend_skip = False + + def queue(self, data): + if data: + self.out_queue.appendleft(data) + + def queue_size(self): + total = 0 + for data in self.out_queue: + total += len(data) + return total + + def in_sync(self): + return not self.out_queue and self.out_nextrepeattime is None + + def settime(self, curtime): + self.cur_time = curtime + if self.out_queue: + if len(self.out_oldpackets) < max_old_packets: + return 0 # more data to send now + if self.out_nextrepeattime is not None: + return max(0, self.out_nextrepeattime - curtime) + else: + return None + + def encode(self, maxlength): + #print ' '*self._dump_indent, '--- OUTQ', self.out_resend, self.out_queue + if len(self.out_oldpackets) >= max_old_packets: + # congestion, stalling + payload = 0 + else: + payload = maxlength - 4 + if payload <= 0: + raise ValueError("encode(): buffer too small") + if (self.out_nextrepeattime is not None and + self.out_nextrepeattime <= self.cur_time): + # no ACK received so far, send a packet (possibly empty) + if not self.out_queue: + payload = 0 + else: + if not self.out_queue: # no more data to send + return None + if payload == 0: # congestion + return None + # prepare a packet + seqid = self.out_nextseqid + flags = self.out_flags + self.out_flags = FLAG_REG # clear out the flags for the next time + if payload > 0: + self.out_nextseqid = (seqid + 1) & 0xFFFF + data = self.out_queue.pop() + packetlength = len(data) + if self.out_resend > 0: + if packetlength > payload: + raise ValueError("XXX need constant buffer size for now") + self.out_resend -= 1 + if self.out_resend_skip: + if self.out_resend > 0: + self.out_queue.pop() + self.out_resend -= 1 + self.out_nextseqid = (seqid + 2) & 0xFFFF + self.out_resend_skip = False + packetpayload = data + else: + packet = [] + while packetlength <= payload: + packet.append(data) + if not self.out_queue: + break + data = self.out_queue.pop() + packetlength += len(data) + else: + rest = len(data) + payload - packetlength + packet.append(data[:rest]) + self.out_queue.append(data[rest:]) + packetpayload = ''.join(packet) + self.out_oldpackets.appendleft(packetpayload) + #print ' '*self._dump_indent, '--- OLDPK', self.out_oldpackets + else: + # a pure ACK packet, no payload + if self.out_oldpackets and flags == FLAG_REG: + flags = FLAG_CFRM + packetpayload = '' + packet = struct.pack("!BBH", flags, + self.in_nextseqid & 0xFF, + seqid) + packetpayload + if self.out_oldpackets: + self.out_nextrepeattime = self.cur_time + self.timeout + else: + self.out_nextrepeattime = None + #self.dump('OUT', packet) + return packet + + def decode(self, rawdata): + if len(rawdata) < 4: + raise InvalidPacket + #print ' '*self._dump_indent, '------ out %d (+%d) in %d' % (self.out_nextseqid, self.out_resend, self.in_nextseqid) + #self.dump('IN ', rawdata) + in_flags, ack_seqid, in_seqid = struct.unpack("!BBH", rawdata[:4]) + if not (FLAG_RANGE_START <= in_flags < FLAG_RANGE_STOP): + raise InvalidPacket + in_diff = (in_seqid - self.in_nextseqid ) & 0xFFFF + ack_diff = (self.out_nextseqid + self.out_resend - ack_seqid) & 0xFF + if in_diff >= max_old_packets: + return '' # invalid, but can occur as a late repetition + if ack_diff != len(self.out_oldpackets): + # forget all acknowledged packets + if ack_diff > len(self.out_oldpackets): + return '' # invalid, but can occur with packet reordering + while len(self.out_oldpackets) > ack_diff: + #print ' '*self._dump_indent, '--- POP', repr(self.out_oldpackets[-1]) + self.out_oldpackets.pop() + if self.out_oldpackets: + self.out_nextrepeattime = self.cur_time + self.timeout + else: + self.out_nextrepeattime = None # all packets ACKed + if in_flags == FLAG_NAK or in_flags == FLAG_NAK1: + # this is a NAK: resend the old packets as far as they've not + # also been ACK'ed in the meantime (can occur with reordering) + while self.out_resend < len(self.out_oldpackets): + self.out_queue.append(self.out_oldpackets[self.out_resend]) + self.out_resend += 1 + self.out_nextseqid = (self.out_nextseqid - 1) & 0xFFFF + #print ' '*self._dump_indent, '--- REP', self.out_nextseqid, repr(self.out_queue[-1]) + self.out_resend_skip = in_flags == FLAG_NAK1 + elif in_flags == FLAG_CFRM: + # this is a CFRM: request for confirmation + self.out_nextrepeattime = self.cur_time + # receive this packet's payload if it is the next in the sequence + if in_diff == 0: + if len(rawdata) > 4: + #print ' '*self._dump_indent, 'RECV ', self.in_nextseqid, repr(rawdata[4:]) + self.in_nextseqid = (self.in_nextseqid + 1) & 0xFFFF + result = [rawdata[4:]] + while self.in_nextseqid in self.in_outoforder: + result.append(self.in_outoforder.pop(self.in_nextseqid)) + self.in_nextseqid = (self.in_nextseqid + 1) & 0xFFFF + return ''.join(result) + else: + # we missed at least one intermediate packet: send a NAK + if len(rawdata) > 4: + self.in_outoforder[in_seqid] = rawdata[4:] + if ((self.in_nextseqid + 1) & 0xFFFF) in self.in_outoforder: + self.out_flags = FLAG_NAK1 + else: + self.out_flags = FLAG_NAK + self.out_nextrepeattime = self.cur_time + return '' + + _dump_indent = 0 + def dump(self, dir, rawdata): + in_flags, ack_seqid, in_seqid = struct.unpack("!BBH", rawdata[:4]) + print ' ' * self._dump_indent, dir, + if in_flags == FLAG_NAK: + print 'NAK', + elif in_flags == FLAG_NAK1: + print 'NAK1', + elif in_flags == FLAG_CFRM: + print 'CFRM', + #print ack_seqid, in_seqid, '(%d bytes)' % (len(rawdata)-4,) + print ack_seqid, in_seqid, repr(rawdata[4:]) + + +def pipe_over_udp(udpsock, send_fd=-1, recv_fd=-1, + timeout=1.0, inactivity_timeout=None): + """Example: send all data showing up in send_fd over the given UDP + socket, and write incoming data into recv_fd. The send_fd and + recv_fd are plain file descriptors. When an EOF is read from + send_fd, this function returns (after making sure that all data was + received by the remote side). + """ + import os + from select import select + from time import time + p = PipeLayer() + p.timeout = timeout + iwtdlist = [udpsock] + if send_fd >= 0: + iwtdlist.append(send_fd) + running = True + while running or not p.in_sync(): + delay = delay1 = p.settime(time()) + if delay is None: + delay = inactivity_timeout + iwtd, owtd, ewtd = select(iwtdlist, [], [], delay) + if iwtd: + if send_fd in iwtd: + data = os.read(send_fd, 1500 - p.headersize) + if not data: + # EOF + iwtdlist.remove(send_fd) + running = False + else: + #print 'queue', len(data) + p.queue(data) + if udpsock in iwtd: + packet = udpsock.recv(65535) + #print 'decode', len(packet) + p.settime(time()) + data = p.decode(packet) + i = 0 + while i < len(data): + i += os.write(recv_fd, data[i:]) + elif delay1 is None: + break # long inactivity + p.settime(time()) + packet = p.encode(1500) + if packet: + #print 'send', len(packet) + #if os.urandom(1) >= '\x08': # emulate packet losses + udpsock.send(packet) + + +class PipeOverUdp(object): + + def __init__(self, udpsock, timeout=1.0): + import thread, os + self.os = os + self.sendpipe = os.pipe() + self.recvpipe = os.pipe() + thread.start_new_thread(pipe_over_udp, (udpsock, + self.sendpipe[0], + self.recvpipe[1], + timeout)) + + def __del__(self): + os = self.os + if self.sendpipe: + os.close(self.sendpipe[0]) + os.close(self.sendpipe[1]) + self.sendpipe = None + if self.recvpipe: + os.close(self.recvpipe[0]) + os.close(self.recvpipe[1]) + self.recvpipe = None + + close = __del__ + + def send(self, data): + if not self.sendpipe: + raise IOError("I/O operation on a closed PipeOverUdp") + return self.os.write(self.sendpipe[1], data) + + def sendall(self, data): + i = 0 + while i < len(data): + i += self.send(data[i:]) + + def recv(self, bufsize): + if not self.recvpipe: + raise IOError("I/O operation on a closed PipeOverUdp") + return self.os.read(self.recvpipe[0], bufsize) + + def recvall(self, bufsize): + buf = [] + while bufsize > 0: + data = self.recv(bufsize) + buf.append(data) + bufsize -= len(data) + return ''.join(buf) + + def fileno(self): + if not self.recvpipe: + raise IOError("I/O operation on a closed PipeOverUdp") + return self.recvpipe[0] + + def ofileno(self): + if not self.sendpipe: + raise IOError("I/O operation on a closed PipeOverUdp") + return self.sendpipe[1] Added: py/trunk/py/net/server/__init__.py ============================================================================== Added: py/trunk/py/net/server/httpserver.py ============================================================================== --- (empty file) +++ py/trunk/py/net/server/httpserver.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,42 @@ +import BaseHTTPServer +from pypeers import greensock2 +from pypeers.pipe.gsocket import GreenSocket + + +class GreenMixIn: + """Mix-in class to handle each request in a new greenlet.""" + + def process_request_greenlet(self, request, client_address): + """Same as in BaseServer but as a greenlet. + In addition, exception handling is done here. + """ + try: + self.finish_request(request, client_address) + self.close_request(request) + except: + self.handle_error(request, client_address) + self.close_request(request) + + def process_request(self, request, client_address): + """Start a new greenlet to process the request.""" + greensock2.autogreenlet(self.process_request_greenlet, + request, client_address) + + +class GreenHTTPServer(GreenMixIn, BaseHTTPServer.HTTPServer): + protocol_version = "HTTP/1.1" + + def server_bind(self): + self.socket = GreenSocket.fromsocket(self.socket) + BaseHTTPServer.HTTPServer.server_bind(self) + + +def test_simple(handler_class=None): + if handler_class is None: + from SimpleHTTPServer import SimpleHTTPRequestHandler + handler_class = SimpleHTTPRequestHandler + server_address = ('', 8000) + httpd = GreenHTTPServer(server_address, handler_class) + sa = httpd.socket.getsockname() + print "Serving HTTP on", sa[0], "port", sa[1], "..." + httpd.serve_forever() Added: py/trunk/py/net/test/test_greenexecnet.py ============================================================================== --- (empty file) +++ py/trunk/py/net/test/test_greenexecnet.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,41 @@ +import py +from py.__.net.greenexecnet import * + +def test_simple(): + gw = PopenGateway() + channel = gw.remote_exec("x = channel.receive(); channel.send(x * 6)") + channel.send(7) + res = channel.receive() + assert res == 42 + +def test_ssh(): + py.test.skip("Bootstrapping") + gw = SshGateway('codespeak.net') + channel = gw.remote_exec(""" + import socket + channel.send(socket.gethostname()) + """) + res = channel.receive() + assert res.endswith('codespeak.net') + +def test_remote_error(): + gw = PopenGateway() + channel = gw.remote_exec("x = channel.receive(); channel.send(x + 1)") + channel.send("hello") + py.test.raises(RemoteError, channel.receive) + +def test_invalid_object(): + class X(object): + pass + gw = PopenGateway() + channel = gw.remote_exec("x = channel.receive(); channel.send(x + 1)") + channel.send(X()) + py.test.raises(RemoteError, channel.receive) + +def test_channel_over_channel(): + gw = PopenGateway() + chan1 = gw.newchannel() + channel = gw.remote_exec("chan1 = channel.receive(); chan1.send(42)") + channel.send(chan1) + res = chan1.receive() + assert res == 42 Added: py/trunk/py/net/test/test_greensock2.py ============================================================================== --- (empty file) +++ py/trunk/py/net/test/test_greensock2.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,213 @@ +import py +from socket import * +from py.__.net.greensock2 import * + +def test_meetingpoint(): + giv1, acc1 = meetingpoint() + giv2, acc2 = meetingpoint() + giv3, acc3 = meetingpoint() + + lst = [] + + def g1(): + lst.append(0) + x = acc2.accept() + assert x == 'hello' + lst.append(2) + giv1.give('world') + lst.append(5) + x = acc3.accept() + assert x == 'middle' + lst.append(6) + giv3.give('done') + + def g2(): + lst.append(1) + giv2.give('hello') + lst.append(3) + y = acc1.accept() + assert y == 'world' + lst.append(4) + + autogreenlet(g1) + autogreenlet(g2) + giv3.give('middle') + tag = acc3.accept() + assert tag == 'done' + assert lst == [0, 1, 2, 3, 4, 5, 6] + + +def test_producer(): + lst = [] + + def prod(n): + lst.append(1) + yield n + lst.append(2) + yield 87 + lst.append(3) + + def cons(): + lst.append(4) + accepter = producer(prod, 145) + lst.append(5) + lst.append(accepter.accept()) + lst.append(6) + lst.append(accepter.accept()) + lst.append(7) + try: + accepter.accept() + except Interrupted: + lst.append(8) + + g = autogreenlet(cons) + wait(g) + assert lst == [4, 5, 1, 145, 6, 2, 87, 7, 3, 8] + + +def test_timer(): + lst = [] + + def g1(): + sleep(0.1, g_1) + lst.append(1) + sleep(0.2, g_1) + lst.append(3) + + def g2(): + lst.append(0) + sleep(0.2, g_2) + lst.append(2) + sleep(0.2, g_2) + lst.append(4) + + g_1 = autogreenlet(g1) + g_2 = autogreenlet(g2) + wait(g_1) + wait(g_2) + assert lst == [0, 1, 2, 3, 4] + + +def test_socket(): + s1 = socket(AF_INET, SOCK_DGRAM) + s2 = socket(AF_INET, SOCK_DGRAM) + s1.bind(('', INADDR_ANY)) + s2.bind(('', INADDR_ANY)) + s1.connect(s2.getsockname()) + s2.connect(s1.getsockname()) + + lst = [] + + def g1(): + lst.append(0) + x = recv(s1, 5) + assert x == 'hello' + lst.append(3) + sendall(s1, 'world') + lst.append(4) + + def g2(): + lst.append(1) + sendall(s2, 'hello') + lst.append(2) + y = recv(s2, 5) + assert y == 'world' + lst.append(5) + + g_1 = autogreenlet(g1) + g_2 = autogreenlet(g2) + wait(g_1) + wait(g_2) + assert lst == [0, 1, 2, 3, 4, 5] + + +##def test_Queue(): + +## def g1(): +## lst.append(5) +## q.put(6) +## lst.append(7) +## q.put(8) +## lst.append(9) +## q.put(10) +## lst.append(11) +## q.put(12) # not used + +## def g2(): +## lst.append(1) +## lst.append(q.get()) +## lst.append(2) +## lst.append(q.get()) +## lst.append(3) +## lst.append(q.get()) +## lst.append(4) + +## q = Queue() +## lst = [] +## autogreenlet(g1) +## autogreenlet(g2) +## wait() +## assert lst == [5, 7, 9, 11, 1, 6, 2, 8, 3, 10, 4] + +## q = Queue() +## lst = [] +## autogreenlet(g2) +## autogreenlet(g1) +## wait() +## assert lst == [1, 5, 7, 9, 11, 6, 2, 8, 3, 10, 4] + + +##def test_Event(): + +## def g1(): +## assert not e.isSet() +## e.wait() +## assert not e.isSet() # no longer set +## lst.append(1) +## e.set() +## e.wait() +## lst.append(2) +## assert e.isSet() +## e.clear() +## assert not e.isSet() +## lst.append(0) +## e.set() +## lst.append(3) +## assert e.isSet() + +## def g2(): +## assert not e.isSet() +## lst.append(4) +## e.set() +## lst.append(7) +## e.clear() +## e.set() +## e.clear() +## assert not e.isSet() +## lst.append(5) +## e.wait() +## assert e.isSet() +## lst.append(6) + +## e = Event() +## lst = [] +## autogreenlet(g1) +## autogreenlet(g2) +## wait() +## assert lst == [4, 7, 5, 1, 2, 0, 3, 6] + + +##def test_Event_timeout(): +## def g1(): +## lst.append(5) +## e.wait(0.1) +## lst.append(e.isSet()) +## e.wait(60.0) +## lst.append(e.isSet()) +## lst = [] +## e = Event() +## autogreenlet(g1) +## sleep(0.5) +## e.set() +## wait() +## assert lst == [5, False, True] Added: py/trunk/py/net/test/test_pipelayer.py ============================================================================== --- (empty file) +++ py/trunk/py/net/test/test_pipelayer.py Tue Mar 6 10:07:53 2007 @@ -0,0 +1,215 @@ +import os, random +from py.__.net.pipelayer import PipeLayer, pipe_over_udp, PipeOverUdp + +def test_simple(): + data1 = os.urandom(1000) + data2 = os.urandom(1000) + p1 = PipeLayer() + p2 = PipeLayer() + p2._dump_indent = 40 + p1.queue(data1) + p1.queue(data2) + recv = '' + while len(recv) < 2000: + raw = p1.encode(64) + assert raw is not None + res = p2.decode(raw) + assert res is not None + recv += res + assert recv == data1 + data2 + raw = p1.encode(64) + assert raw is None + +def test_stabilize(): + data1 = os.urandom(28) + p1 = PipeLayer() + p2 = PipeLayer() + p2._dump_indent = 40 + p1.queue(data1) + recv = '' + t = 0.0 + print + while True: + delay1 = p1.settime(t) + delay2 = p2.settime(t) + t += 0.100000001 + if delay1 is delay2 is None: + break + if delay1 == 0: + raw = p1.encode(10) + p1.dump('OUT', raw) + assert raw is not None + res = p2.decode(raw) + assert res is not None + recv += res + if delay2 == 0: + raw = p2.encode(10) + p2.dump('OUT', raw) + assert raw is not None + res = p1.decode(raw) + assert res == '' + assert recv == data1 + +def test_bidir(): + data1 = os.urandom(1000) + data2 = os.urandom(1000) + p1 = PipeLayer() + p2 = PipeLayer() + p2._dump_indent = 40 + p1.queue(data1) + p2.queue(data2) + recv = ['', ''] + while len(recv[0]) < 1000 or len(recv[1]) < 1000: + progress = False + for (me, other, i) in [(p1, p2, 1), (p2, p1, 0)]: + raw = me.encode(64) + if raw is not None: + res = other.decode(raw) + assert res is not None + recv[i] += res + if res: + progress = True + assert progress + assert recv[0] == data2 + assert recv[1] == data1 + raw = p1.encode(64) + assert raw is None + raw = p2.encode(64) + assert raw is None + +def test_with_loss(): + data1 = os.urandom(10000).encode('hex') + data2 = os.urandom(10000).encode('hex') + #data1 = '0123456789' + #data2 = 'ABCDEFGHIJ' + p1 = PipeLayer() + p2 = PipeLayer() + p2._dump_indent = 40 + p1.queue(data1) + p2.queue(data2) + recv = ['', ''] + time = 0 + active = 1 + while active: + active = 0 + time += 0.2 + #print '.' + exchange = [] + for (me, other, i) in [(p1, p2, 1), (p2, p1, 0)]: + to = me.settime(time) + packet = me.encode(12) + assert (packet is not None) == (to == 0) + if to is not None: + active = 1 + if to == 0: + exchange.append((packet, other, i)) + for (packet, other, i) in exchange: + if random.random() < 0.5: + pass # drop packet + else: + res = other.decode(packet) + assert res is not None + recv[i] += res + assert data2.startswith(recv[0]) + assert data1.startswith(recv[1]) + assert recv[0] == data2 + assert recv[1] == data1 + print time + +def test_massive_reordering(): + data1 = os.urandom(10000).encode('hex') + data2 = os.urandom(10000).encode('hex') + #data1 = '0123456789' + #data2 = 'ABCDEFGHIJ' + p1 = PipeLayer() + p2 = PipeLayer() + p2._dump_indent = 40 + p1.queue(data1) + p2.queue(data2) + recv = ['', ''] + time = 0 + active = 1 + exchange = [] + while active or exchange: + active = 0 + time += 0.2 + #print '.' + for (me, other, i) in [(p1, p2, 1), (p2, p1, 0)]: + to = me.settime(time) + packet = me.encode(12) + assert (packet is not None) == (to == 0) + if to is not None: + active = 1 + if to == 0: + exchange.append((packet, other, i)) + if random.random() < 0.02: + random.shuffle(exchange) + for (packet, other, i) in exchange: + res = other.decode(packet) + assert res is not None + recv[i] += res + exchange = [] + assert data2.startswith(recv[0]) + assert data1.startswith(recv[1]) + assert recv[0] == data2 + assert recv[1] == data1 + print time + +def udpsockpair(): + from socket import socket, AF_INET, SOCK_DGRAM, INADDR_ANY + s1 = socket(AF_INET, SOCK_DGRAM) + s2 = socket(AF_INET, SOCK_DGRAM) + s1.bind(('127.0.0.1', INADDR_ANY)) + s2.bind(('127.0.0.1', INADDR_ANY)) + s2.connect(s1.getsockname()) + s1.connect(s2.getsockname()) + return s1, s2 + +def test_pipe_over_udp(): + import thread + s1, s2 = udpsockpair() + + fd1 = os.open(__file__, os.O_RDONLY) + fd2 = os.open('test_pipelayer.py~copy', os.O_WRONLY|os.O_CREAT|os.O_TRUNC) + + thread.start_new_thread(pipe_over_udp, (s1, fd1)) + pipe_over_udp(s2, recv_fd=fd2, inactivity_timeout=2.5) + os.close(fd1) + os.close(fd2) + f = open(__file__, 'rb') + data1 = f.read() + f.close() + f = open('test_pipelayer.py~copy', 'rb') + data2 = f.read() + f.close() + assert data1 == data2 + os.unlink('test_pipelayer.py~copy') + +def test_PipeOverUdp(): + s1, s2 = udpsockpair() + p1 = PipeOverUdp(s1, timeout=0.2) + p2 = PipeOverUdp(s2, timeout=0.2) + p2.sendall('goodbye') + for k in range(10): + p1.sendall('hello world') + input = p2.recvall(11) + assert input == 'hello world' + input = p1.recvall(7) + assert input == 'goodbye' + + bigchunk1 = os.urandom(500000) + bigchunk2 = os.urandom(500000) + i1 = i2 = 0 + j1 = j2 = 0 + while j1 < len(bigchunk1) or j2 < len(bigchunk2): + i1 += p1.send(bigchunk1[i1:i1+512]) + i2 += p2.send(bigchunk2[i2:i2+512]) + data = p1.recv(512) + assert data == bigchunk2[j2:j2+len(data)] + j2 += len(data) + data = p2.recv(512) + assert data == bigchunk1[j1:j1+len(data)] + j1 += len(data) + #print i1, i2, j1, j2 + p1.close() + p2.close() From fijal at codespeak.net Tue Mar 6 10:41:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 10:41:00 +0100 (CET) Subject: [py-svn] r39974 - in py/trunk/py/net: . pipe Message-ID: <20070306094100.9275510071@code0.codespeak.net> Author: fijal Date: Tue Mar 6 10:40:59 2007 New Revision: 39974 Modified: py/trunk/py/net/greenexecnet.py py/trunk/py/net/greensock2.py py/trunk/py/net/pipe/__init__.py Log: Add some comments Modified: py/trunk/py/net/greenexecnet.py ============================================================================== --- py/trunk/py/net/greenexecnet.py (original) +++ py/trunk/py/net/greenexecnet.py Tue Mar 6 10:40:59 2007 @@ -1,3 +1,17 @@ + +""" This is an implementation of an execnet protocol on top +of a transport layer provided by the greensock2 interface. + +It has the same semantics, but does not use threads at all +(which makes it suitable for specific enviroments, like pypy-c). + +There are some features lacking, most notable: +- callback support for channels +- socket gateway +- bootstrapping (there is assumption of pylib being available + on remote side, which is not always true) +""" + import sys, os, py, inspect from py.__.net import greensock2 from py.__.net.msgstruct import message, decodemessage Modified: py/trunk/py/net/greensock2.py ============================================================================== --- py/trunk/py/net/greensock2.py (original) +++ py/trunk/py/net/greensock2.py Tue Mar 6 10:40:59 2007 @@ -1,3 +1,10 @@ + +""" This is a base implementation of thread-like network programming +on top of greenlets. From API available here it's quite unlikely +that you would like to use anything except wait(). Higher level interface +is available in pipe directory +""" + import os, sys try: from stackless import greenlet Modified: py/trunk/py/net/pipe/__init__.py ============================================================================== --- py/trunk/py/net/pipe/__init__.py (original) +++ py/trunk/py/net/pipe/__init__.py Tue Mar 6 10:40:59 2007 @@ -0,0 +1,8 @@ + +""" This is a higher level network interface based on top +of greensock2. Objects here are ready to use, specific examples +are listed in tests (test_pipelayer and test_greensock2). + +The limitation is that you're not supposed to use threads + blocking +I/O at all. +""" From fijal at codespeak.net Tue Mar 6 10:49:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 10:49:48 +0100 (CET) Subject: [py-svn] r39975 - in py/trunk/py/net: pipe server Message-ID: <20070306094948.6872910074@code0.codespeak.net> Author: fijal Date: Tue Mar 6 10:49:47 2007 New Revision: 39975 Modified: py/trunk/py/net/pipe/common.py py/trunk/py/net/pipe/gsocket.py py/trunk/py/net/pipe/mp.py py/trunk/py/net/server/httpserver.py Log: Fix imports Modified: py/trunk/py/net/pipe/common.py ============================================================================== --- py/trunk/py/net/pipe/common.py (original) +++ py/trunk/py/net/pipe/common.py Tue Mar 6 10:49:47 2007 @@ -1,6 +1,14 @@ -import greensock2 -from pypeers.tool import log +from py.__.net import greensock2 +VERBOSE = True + + +if VERBOSE: + def log(msg, *args): + print '*', msg % args +else: + def log(msg, *args): + pass class BufferedInput(object): in_buf = '' Modified: py/trunk/py/net/pipe/gsocket.py ============================================================================== --- py/trunk/py/net/pipe/gsocket.py (original) +++ py/trunk/py/net/pipe/gsocket.py Tue Mar 6 10:49:47 2007 @@ -1,4 +1,4 @@ -import greensock2 +from py.__.net import greensock2 import socket, errno, os error = socket.error Modified: py/trunk/py/net/pipe/mp.py ============================================================================== --- py/trunk/py/net/pipe/mp.py (original) +++ py/trunk/py/net/pipe/mp.py Tue Mar 6 10:49:47 2007 @@ -1,4 +1,4 @@ -from pypeers.stream.common import BufferedInput +from py.__.net.pipe.common import BufferedInput class MeetingPointInput(BufferedInput): Modified: py/trunk/py/net/server/httpserver.py ============================================================================== --- py/trunk/py/net/server/httpserver.py (original) +++ py/trunk/py/net/server/httpserver.py Tue Mar 6 10:49:47 2007 @@ -1,6 +1,6 @@ import BaseHTTPServer -from pypeers import greensock2 -from pypeers.pipe.gsocket import GreenSocket +from py.__.net import greensock2 +from py.__.net.pipe.gsocket import GreenSocket class GreenMixIn: From fijal at codespeak.net Tue Mar 6 13:38:50 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 13:38:50 +0100 (CET) Subject: [py-svn] r39981 - py/trunk/py/doc/talk Message-ID: <20070306123850.9A84B10078@code0.codespeak.net> Author: fijal Date: Tue Mar 6 13:38:49 2007 New Revision: 39981 Added: py/trunk/py/doc/talk/notes.txt Log: Some random notes about future pylib direction Added: py/trunk/py/doc/talk/notes.txt ============================================================================== --- (empty file) +++ py/trunk/py/doc/talk/notes.txt Tue Mar 6 13:38:49 2007 @@ -0,0 +1,15 @@ + +* Persistant storage layer for storing py.test output, sharing such stuff + and presenting (Presenting mostly means combining tones of hacks here + and there). We need to store test results, revisions and additional + metadata like apigen output + +* Having some kind of pdbplus, which will combine rlcompleter, apigen + information and other various fixes. + +* Improve distributed testing by: + - sharing even more code with normal testing + - using greenexecnet wherever possible (falling back to normal + execnet) + - make test redistribution somehow (in a clean way!) + - C-c support From hpk at codespeak.net Tue Mar 6 13:51:19 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 6 Mar 2007 13:51:19 +0100 (CET) Subject: [py-svn] r39982 - py/trunk/py/execnet Message-ID: <20070306125119.E587A10078@code0.codespeak.net> Author: hpk Date: Tue Mar 6 13:51:18 2007 New Revision: 39982 Modified: py/trunk/py/execnet/channel.py py/trunk/py/execnet/gateway.py Log: introduce gateway._send and have all places route their sending of Messages (or None's) through that method. Modified: py/trunk/py/execnet/channel.py ============================================================================== --- py/trunk/py/execnet/channel.py (original) +++ py/trunk/py/execnet/channel.py Tue Mar 6 13:51:18 2007 @@ -85,7 +85,7 @@ Msg = Message.CHANNEL_LAST_MESSAGE else: Msg = Message.CHANNEL_CLOSE - self.gateway._outgoing.put(Msg(self.id)) + self.gateway._send(Msg(self.id)) def _getremoteerror(self): try: @@ -117,7 +117,7 @@ # state transition "opened/sendonly" --> "closed" # threads warning: the channel might be closed under our feet, # but it's never damaging to send too many CHANNEL_CLOSE messages - put = self.gateway._outgoing.put + put = self.gateway._send if error is not None: put(Message.CHANNEL_CLOSE_ERROR(self.id, str(error))) else: @@ -157,7 +157,7 @@ data = Message.CHANNEL_NEW(self.id, item.id) else: data = Message.CHANNEL_DATA(self.id, item) - self.gateway._outgoing.put(data) + self.gateway._send(data) def receive(self): """receives an item that was sent from the other side, Modified: py/trunk/py/execnet/gateway.py ============================================================================== --- py/trunk/py/execnet/gateway.py (original) +++ py/trunk/py/execnet/gateway.py Tue Mar 6 13:51:18 2007 @@ -111,10 +111,13 @@ self._traceex(exc_info()) break finally: - self._outgoing.put(None) + self._send(None) self._channelfactory._finished_receiving() self._trace('leaving %r' % threading.currentThread()) + def _send(self, msg): + self._outgoing.put(msg) + def _thread_sender(self): """ thread to send Messages over the wire. """ try: @@ -219,8 +222,8 @@ channel = self.newchannel() outid = self._newredirectchannelid(stdout) errid = self._newredirectchannelid(stderr) - self._outgoing.put(Message.CHANNEL_OPEN(channel.id, - (source, outid, errid))) + self._send(Message.CHANNEL_OPEN( + channel.id, (source, outid, errid))) return channel def _remote_redirect(self, stdout=None, stderr=None): @@ -260,7 +263,7 @@ except KeyError: pass else: - self._outgoing.put(None) + self._send(None) def join(self, joinexec=True): """ Wait for all IO (and by default all execution activity) From fijal at codespeak.net Tue Mar 6 14:54:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 14:54:35 +0100 (CET) Subject: [py-svn] r39983 - in py/trunk/py/net: . test Message-ID: <20070306135435.6E1FB10078@code0.codespeak.net> Author: fijal Date: Tue Mar 6 14:54:33 2007 New Revision: 39983 Modified: py/trunk/py/net/greenexecnet.py py/trunk/py/net/greensock2.py py/trunk/py/net/test/test_greensock2.py Log: (arigo, fijal) - * oneof(functions) runs all functions in parallel until one returns a value, then returns that value * allof(functions) same as previous, but returns all return values in a tuple Modified: py/trunk/py/net/greenexecnet.py ============================================================================== --- py/trunk/py/net/greenexecnet.py (original) +++ py/trunk/py/net/greenexecnet.py Tue Mar 6 14:54:33 2007 @@ -189,7 +189,7 @@ os.dup2(fd, 0) os.dup2(fd, 1) os.close(fd) - greensock2.wait(gw.greenlet) + greensock2._suspend_forever() run_server = staticmethod(run_server) class PopenGateway(PopenCmdGateway): Modified: py/trunk/py/net/greensock2.py ============================================================================== --- py/trunk/py/net/greensock2.py (original) +++ py/trunk/py/net/greensock2.py Tue Mar 6 14:54:33 2007 @@ -196,80 +196,6 @@ d.clear() d.extend(lst) -# ____________________________________________________________ - -##class Queue(object): - -## def __init__(self): -## self.giver, self.accepter = meetingpoint() -## self.pending = deque() - -## def put(self, item): # preserve the caller's atomicity -## self.pending.append(item) -## if self.accepter.ready(): -## self.accepter.accept() - -## def get(self, block=True): -## if self.pending: -## return self.pending.popleft() -## elif block: -## self.giver.give(None) -## return self.pending.popleft() -## else: -## raise Empty - -##class Empty(Interrupted): -## pass - -##class Event(object): - -## def __init__(self): -## self.giver, self.accepter = meetingpoint() - -## clear = __init__ - -## def isSet(self): -## return self.accepter is None - -## def set(self): # preserve the caller's atomicity -## if self.accepter is not None: -## accepter = self.accepter -## self.giver = self.accepter = None -## while accepter.ready(): # wake up all waiters -## accepter.accept() - -## def wait(self, timeout=None): -## if self.accepter is not None: -## if timeout is None: -## self.giver.give(None) -## else: -## timer = Timer(timeout) -## try: -## try: -## self.giver.give(None) -## except Interrupted: -## pass -## finally: -## timer.stop() - -##class Semaphore(object): - -## def __init__(self, value=1): -## self.giver, self.accepter = meetingpoint() -## for i in range(value): -## self.release() - -## def acquire(self, blocking=True): -## if blocking or self.accepter.ready(): -## return self.accepter.accept() -## else: -## return False - -## def release(self): -## autogreenlet(self.giver.put, True) - -# ____________________________________________________________ - def wait_input(sock): _register(g_iwtd, sock) @@ -339,28 +265,41 @@ in_front = True -def sleep(duration, *greenlets): +def sleep(duration): timer = Timer(duration) try: - wait(*greenlets) + _suspend_forever() finally: ok = timer.finished timer.stop() if not ok: raise Interrupted -def _wait(): +def _suspend_forever(): g_dispatcher.switch() -def wait(*greenlets): - assert greenlets#, "should not wait without events to wait on" - current = g_getcurrent() +def oneof(*callables): + assert callables + for c in callables: + assert callable(c) + greenlets = [tracinggreenlet(c) for c in callables] + g_active.extend(greenlets) + res = g_dispatcher.switch() for g in greenlets: - if g in g_waiters: - g_waiters[g].append(current) - else: - g_waiters[g] = [current] - g_dispatcher.switch() + g.interrupt() + return res + +def allof(*callables): + for c in callables: + assert callable(c) + greenlets = [tracinggreenlet(lambda i=i, c=c: (i, c())) + for i, c in enumerate(callables)] + g_active.extend(greenlets) + result = [None] * len(callables) + for _ in callables: + num, res = g_dispatcher.switch() + result[num] = res + return tuple(result) class Timer(object): started = False @@ -387,31 +326,31 @@ # ____________________________________________________________ -class autogreenlet(greenlet): +class tracinggreenlet(greenlet): def __init__(self, function, *args, **kwds): - self.parent = g_dispatcher self.function = function self.args = args self.kwds = kwds - g_active.append(self) + + def __repr__(self): +## args = ', '.join([repr(s) for s in self.args] + +## ['%s=%r' % keyvalue for keyvalue in self.kwds.items()]) +## return '' % (self.function.__name__, args) + return '<%s %s at %s>' % (self.__class__.__name__, + self.function.__name__, + hex(id(self))) def run(self): self.trace("start") try: - self.function(*self.args, **self.kwds) + res = self.function(*self.args, **self.kwds) except Exception, e: self.trace("stop (%s%s)", e.__class__.__name__, str(e) and (': '+str(e))) raise else: self.trace("done") - - def __repr__(self): -## args = ', '.join([repr(s) for s in self.args] + -## ['%s=%r' % keyvalue for keyvalue in self.kwds.items()]) -## return '' % (self.function.__name__, args) - return '' % (self.function.__name__, - hex(id(self))) + return res def trace(self, msg, *args): if TRACE: @@ -420,6 +359,11 @@ def interrupt(self): self.throw(Interrupted) +class autogreenlet(tracinggreenlet): + def __init__(self, *args, **kwargs): + super(autogreenlet, self).__init__(*args, **kwargs) + self.parent = g_dispatcher + g_active.append(self) g_active = deque() g_iwtd = {} @@ -452,24 +396,25 @@ for k in to_remove: del mapping[k] -def check_waiters(active): - if active in g_waiters: - for g in g_waiters[active]: - g.switch() - del g_waiters[active] +#def check_waiters(active): +# if active in g_waiters: +# for g in g_waiters[active]: +# g.switch() +# del g_waiters[active] def dispatcher_mainloop(): global g_timers_mixed + GreenletExit = greenlet.GreenletExit while 1: try: while g_active: - print 'active:', g_active[0] - active = g_active.popleft() - active.switch() - if active.dead: - check_waiters(active) - del active + #print 'active:', g_active[0] + g_active.popleft().switch() +# active.switch() +# if active.dead: +# check_waiters(active) +# del active if g_timers: if g_timers_mixed: heapify(g_timers) @@ -482,8 +427,8 @@ #print 'timeout:', g timer.finished = True timer.g.switch() - if timer.g.dead: - check_waiters(timer.g) +# if timer.g.dead: +# check_waiters(timer.g) continue delay = 0.0 timer.started = True @@ -496,9 +441,9 @@ continue delay = None - print 'selecting...', g_iwtd.keys(), g_owtd.keys(), delay + #print 'selecting...', g_iwtd.keys(), g_owtd.keys(), delay iwtd, owtd, _ = _select(g_iwtd.keys(), g_owtd.keys(), [], delay) - print 'done' + #print 'done' for s in owtd: if s in g_owtd: d = g_owtd[s] @@ -510,8 +455,8 @@ except KeyError: pass g.switch(g_owtd) - if g.dead: - check_waiters(g) +# if g.dead: +# check_waiters(g) for s in iwtd: if s in g_iwtd: d = g_iwtd[s] @@ -523,11 +468,13 @@ except KeyError: pass g.switch(g_iwtd) - if g.dead: - check_waiters(g) +# if g.dead: +# check_waiters(g) + except GreenletExit: + raise except: import sys g_dispatcher.parent.throw(*sys.exc_info()) g_dispatcher = greenlet(dispatcher_mainloop) -g_waiters = {} +#g_waiters = {} Modified: py/trunk/py/net/test/test_greensock2.py ============================================================================== --- py/trunk/py/net/test/test_greensock2.py (original) +++ py/trunk/py/net/test/test_greensock2.py Tue Mar 6 14:54:33 2007 @@ -60,8 +60,7 @@ except Interrupted: lst.append(8) - g = autogreenlet(cons) - wait(g) + oneof(cons) assert lst == [4, 5, 1, 145, 6, 2, 87, 7, 3, 8] @@ -69,24 +68,33 @@ lst = [] def g1(): - sleep(0.1, g_1) + sleep(0.1) lst.append(1) - sleep(0.2, g_1) + sleep(0.2) lst.append(3) def g2(): lst.append(0) - sleep(0.2, g_2) + sleep(0.2) lst.append(2) - sleep(0.2, g_2) + sleep(0.2) lst.append(4) - g_1 = autogreenlet(g1) - g_2 = autogreenlet(g2) - wait(g_1) - wait(g_2) - assert lst == [0, 1, 2, 3, 4] + oneof(g1, g2) + assert lst == [0, 1, 2, 3] +def test_kill_other(): + + def g1(): + sleep(.1) + return 1 + + def g2(): + sleep(.2) + return 2 + + res = oneof(g1, g2) + assert res == 1 def test_socket(): s1 = socket(AF_INET, SOCK_DGRAM) @@ -105,6 +113,7 @@ lst.append(3) sendall(s1, 'world') lst.append(4) + return 1 def g2(): lst.append(1) @@ -113,13 +122,12 @@ y = recv(s2, 5) assert y == 'world' lst.append(5) + return 2 - g_1 = autogreenlet(g1) - g_2 = autogreenlet(g2) - wait(g_1) - wait(g_2) + one, two = allof(g1, g2) assert lst == [0, 1, 2, 3, 4, 5] - + assert one == 1 + assert two == 2 ##def test_Queue(): From hpk at codespeak.net Tue Mar 6 15:00:49 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 6 Mar 2007 15:00:49 +0100 (CET) Subject: [py-svn] r39985 - in py/branch/lessthread: . py py/execnet py/execnet/testing Message-ID: <20070306140049.E7C971007E@code0.codespeak.net> Author: hpk Date: Tue Mar 6 15:00:48 2007 New Revision: 39985 Added: py/branch/lessthread/ - copied from r39978, py/trunk/ py/branch/lessthread/py/ - copied from r39981, py/trunk/py/ py/branch/lessthread/py/execnet/channel.py - copied unchanged from r39982, py/trunk/py/execnet/channel.py py/branch/lessthread/py/execnet/gateway.py - copied, changed from r39982, py/trunk/py/execnet/gateway.py Modified: py/branch/lessthread/py/execnet/inputoutput.py py/branch/lessthread/py/execnet/testing/test_gateway.py Log: create branch for reducing of threads in execnet Copied: py/branch/lessthread/py/execnet/gateway.py (from r39982, py/trunk/py/execnet/gateway.py) ============================================================================== --- py/trunk/py/execnet/gateway.py (original) +++ py/branch/lessthread/py/execnet/gateway.py Tue Mar 6 15:00:48 2007 @@ -23,7 +23,6 @@ from py.__.execnet.message import Message ThreadOut = py._thread.ThreadOut WorkerPool = py._thread.WorkerPool - NamedThreadPool = py._thread.NamedThreadPool import os debug = 0 # open('/tmp/execnet-debug-%d' % os.getpid() , 'wa') @@ -38,17 +37,18 @@ inputoutput object and 'execthreads' execution threads. """ - global registered_cleanup + global registered_cleanup, _activegateways self._execpool = WorkerPool(maxthreads=execthreads) self._io = io - self._outgoing = Queue.Queue() self._channelfactory = ChannelFactory(self, _startcount) if not registered_cleanup: atexit.register(cleanup_atexit) registered_cleanup = True - _active_sendqueues[self._outgoing] = True - self._pool = NamedThreadPool(receiver = self._thread_receiver, - sender = self._thread_sender) + _activegateways[self] = True + self._receiverthread = threading.Thread(name="receiver", + target=self._thread_receiver) + self._receiverthread.setDaemon(0) + self._receiverthread.start() def __repr__(self): """ return string representing gateway type and status. """ @@ -58,10 +58,9 @@ else: addr = '' try: - r = (len(self._pool.getstarted('receiver')) - and "receiving" or "not receiving") - s = (len(self._pool.getstarted('sender')) - and "sending" or "not sending") + r = (self._receiverthread.isAlive() and "receiving" or + "not receiving") + s = "sending" # XXX i = len(self._channelfactory.channels()) except AttributeError: r = s = "uninitialized" @@ -116,30 +115,19 @@ self._trace('leaving %r' % threading.currentThread()) def _send(self, msg): - self._outgoing.put(msg) - - def _thread_sender(self): - """ thread to send Messages over the wire. """ - try: - from sys import exc_info - while 1: - msg = self._outgoing.get() - try: - if msg is None: - self._io.close_write() - break - msg.writeto(self._io) - except: - excinfo = exc_info() - self._traceex(excinfo) - if msg is not None: - msg.post_sent(self, excinfo) - break - else: - self._trace('sent -> %r' % msg) - msg.post_sent(self) - finally: - self._trace('leaving %r' % threading.currentThread()) + from sys import exc_info + if msg is None: + self._io.close_write() + else: + try: + msg.writeto(self._io) + except: + excinfo = exc_info() + self._traceex(excinfo) + msg.post_sent(self, excinfo) + else: + msg.post_sent(self) + self._trace('sent -> %r' % msg) def _local_redirect_thread_output(self, outid, errid): l = [] @@ -258,23 +246,16 @@ def exit(self): """ Try to stop all IO activity. """ - try: - del _active_sendqueues[self._outgoing] - except KeyError: - pass - else: - self._send(None) + self._send(None) def join(self, joinexec=True): """ Wait for all IO (and by default all execution activity) to stop. """ current = threading.currentThread() - for x in self._pool.getstarted(): - if x != current: - self._trace("joining %s" % x) - x.join() - self._trace("joining sender/reciver threads finished, current %r" % current) + if self._receiverthread.isAlive(): + self._trace("joining receiver thread") + self._receiverthread.join() if joinexec: self._execpool.join() self._trace("joining execution threads finished, current %r" % current) @@ -288,14 +269,12 @@ return x registered_cleanup = False -_active_sendqueues = weakref.WeakKeyDictionary() +_activegateways = weakref.WeakKeyDictionary() def cleanup_atexit(): if debug: print >>debug, "="*20 + "cleaning up" + "=" * 20 debug.flush() - while True: - try: - queue, ignored = _active_sendqueues.popitem() - except KeyError: - break - queue.put(None) + while _activegateways: + gw, ignored = _activegateways.popitem() + gw.exit() + #gw.join() should work as well Modified: py/branch/lessthread/py/execnet/inputoutput.py ============================================================================== --- py/trunk/py/execnet/inputoutput.py (original) +++ py/branch/lessthread/py/execnet/inputoutput.py Tue Mar 6 15:00:48 2007 @@ -43,11 +43,17 @@ def close_read(self): if self.readable: - self.sock.shutdown(0) + try: + self.sock.shutdown(0) + except socket.error: + pass self.readable = None def close_write(self): if self.writeable: - self.sock.shutdown(1) + try: + self.sock.shutdown(1) + except socket.error: + pass self.writeable = None class Popen2IO: Modified: py/branch/lessthread/py/execnet/testing/test_gateway.py ============================================================================== --- py/trunk/py/execnet/testing/test_gateway.py (original) +++ py/branch/lessthread/py/execnet/testing/test_gateway.py Tue Mar 6 15:00:48 2007 @@ -83,8 +83,7 @@ class BasicRemoteExecution: def test_correct_setup(self): - for x in 'sender', 'receiver': - assert self.gw._pool.getstarted(x) + assert self.gw._receiverthread.isAlive() def test_repr_doesnt_crash(self): assert isinstance(repr(self), str) From hpk at codespeak.net Tue Mar 6 18:35:03 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 6 Mar 2007 18:35:03 +0100 (CET) Subject: [py-svn] r39992 - py/branch/lessthread/py/execnet Message-ID: <20070306173503.54DA310069@code0.codespeak.net> Author: hpk Date: Tue Mar 6 18:35:02 2007 New Revision: 39992 Modified: py/branch/lessthread/py/execnet/gateway.py py/branch/lessthread/py/execnet/register.py Log: removing execution threads by default Modified: py/branch/lessthread/py/execnet/gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/gateway.py (original) +++ py/branch/lessthread/py/execnet/gateway.py Tue Mar 6 18:35:02 2007 @@ -22,7 +22,7 @@ from py.__.execnet.channel import ChannelFactory, Channel from py.__.execnet.message import Message ThreadOut = py._thread.ThreadOut - WorkerPool = py._thread.WorkerPool + WorkerPool = py._thread.WorkerPool import os debug = 0 # open('/tmp/execnet-debug-%d' % os.getpid() , 'wa') @@ -31,20 +31,29 @@ class Gateway(object): _ThreadOut = ThreadOut + _workerpool = None remoteaddress = "" def __init__(self, io, execthreads=None, _startcount=2): """ initialize core gateway, using the given inputoutput object and 'execthreads' execution - threads. + threads. if 'execthreads' is -1, no execution + will be allowed on this side. If it is None, + requests will be queued and _servemain() may + be used to execute them. """ global registered_cleanup, _activegateways - self._execpool = WorkerPool(maxthreads=execthreads) self._io = io self._channelfactory = ChannelFactory(self, _startcount) if not registered_cleanup: atexit.register(cleanup_atexit) registered_cleanup = True _activegateways[self] = True + if execthreads != -1: + if execthreads is not None: + self._workerpool = WorkerPool(maxthreads=execthreads) + else: + self._requestqueue = Queue.Queue() + self._receiverthread = threading.Thread(name="receiver", target=self._thread_receiver) self._receiverthread.setDaemon(0) @@ -110,7 +119,8 @@ self._traceex(exc_info()) break finally: - self._send(None) + self._stopexec() + self._stopsend() self._channelfactory._finished_receiving() self._trace('leaving %r' % threading.currentThread()) @@ -143,10 +153,31 @@ channel.close() return close - def _thread_executor(self, channel, (source, outid, errid)): - """ worker thread to execute source objects from the execution queue. """ + def _local_schedulexec(self, channel, sourcetask): + if self._workerpool: + self._workerpool.dispatch(self._executetask, channel, sourcetask) + elif hasattr(self, '_requestqueue'): + self._requestqueue.put((channel, sourcetask)) + + def _servemain(self, joining=True): from sys import exc_info try: + while 1: + item = self._requestqueue.get() + if item is None: + self._stopsend() + break + self._executetask(item) # could be done in an exec thread + finally: + self._trace("_servemain finished") + if self.joining: + self.join() + + def _executetask(self, item): + """ execute channel/source items. """ + from sys import exc_info + channel, (source, outid, errid) = item + try: loc = { 'channel' : channel } self._trace("execution starts:", repr(source)[:50]) close = self._local_redirect_thread_output(outid, errid) @@ -168,10 +199,6 @@ else: channel.close() - def _local_schedulexec(self, channel, sourcetask): - self._trace("dispatching exec") - self._execpool.dispatch(self._thread_executor, channel, sourcetask) - def _newredirectchannelid(self, callback): if callback is None: return @@ -245,8 +272,18 @@ return Handle() def exit(self): - """ Try to stop all IO activity. """ - self._send(None) + """ Try to stop all exec and IO activity. """ + self._stopexec() + self._stopsend() + + def _stopsend(self): + self._send(None) + + def _stopexec(self): + if hasattr(self, '_requestqueue'): + self._requestqueue.put(None) + if self._workerpool: + self._workerpool.shutdown() def join(self, joinexec=True): """ Wait for all IO (and by default all execution activity) @@ -257,8 +294,9 @@ self._trace("joining receiver thread") self._receiverthread.join() if joinexec: - self._execpool.join() - self._trace("joining execution threads finished, current %r" % current) + if self._workerpool: + self._workerpool.join() + self._trace("joining execution threads finished, current %r" % current) def getid(gw, cache={}): name = gw.__class__.__name__ Modified: py/branch/lessthread/py/execnet/register.py ============================================================================== --- py/branch/lessthread/py/execnet/register.py (original) +++ py/branch/lessthread/py/execnet/register.py Tue Mar 6 18:35:02 2007 @@ -41,7 +41,7 @@ bootstrap = [extra] bootstrap += [getsource(x) for x in startup_modules] bootstrap += [io.server_stmt, - "Gateway(io=io, _startcount=2).join(joinexec=False)", + "Gateway(io=io, _startcount=2)._servemain()", ] source = "\n".join(bootstrap) self._trace("sending gateway bootstrap code") From hpk at codespeak.net Tue Mar 6 18:51:34 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 6 Mar 2007 18:51:34 +0100 (CET) Subject: [py-svn] r39993 - py/branch/lessthread/py/execnet Message-ID: <20070306175134.CA5CC1006E@code0.codespeak.net> Author: hpk Date: Tue Mar 6 18:51:31 2007 New Revision: 39993 Modified: py/branch/lessthread/py/execnet/gateway.py py/branch/lessthread/py/execnet/register.py Log: strike support for exec threads, and disallow execution from remote by default. Modified: py/branch/lessthread/py/execnet/gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/gateway.py (original) +++ py/branch/lessthread/py/execnet/gateway.py Tue Mar 6 18:51:31 2007 @@ -22,7 +22,6 @@ from py.__.execnet.channel import ChannelFactory, Channel from py.__.execnet.message import Message ThreadOut = py._thread.ThreadOut - WorkerPool = py._thread.WorkerPool import os debug = 0 # open('/tmp/execnet-debug-%d' % os.getpid() , 'wa') @@ -31,15 +30,11 @@ class Gateway(object): _ThreadOut = ThreadOut - _workerpool = None + _requestqueue = None remoteaddress = "" - def __init__(self, io, execthreads=None, _startcount=2): + def __init__(self, io, allowexec=False, _startcount=2): """ initialize core gateway, using the given - inputoutput object and 'execthreads' execution - threads. if 'execthreads' is -1, no execution - will be allowed on this side. If it is None, - requests will be queued and _servemain() may - be used to execute them. + inputoutput object. """ global registered_cleanup, _activegateways self._io = io @@ -48,12 +43,8 @@ atexit.register(cleanup_atexit) registered_cleanup = True _activegateways[self] = True - if execthreads != -1: - if execthreads is not None: - self._workerpool = WorkerPool(maxthreads=execthreads) - else: - self._requestqueue = Queue.Queue() - + if allowexec: + self._requestqueue = Queue.Queue() self._receiverthread = threading.Thread(name="receiver", target=self._thread_receiver) self._receiverthread.setDaemon(0) @@ -154,9 +145,7 @@ return close def _local_schedulexec(self, channel, sourcetask): - if self._workerpool: - self._workerpool.dispatch(self._executetask, channel, sourcetask) - elif hasattr(self, '_requestqueue'): + if self._requestqueue is not None: self._requestqueue.put((channel, sourcetask)) def _servemain(self, joining=True): @@ -280,23 +269,17 @@ self._send(None) def _stopexec(self): - if hasattr(self, '_requestqueue'): + if self._requestqueue is not None: self._requestqueue.put(None) - if self._workerpool: - self._workerpool.shutdown() def join(self, joinexec=True): """ Wait for all IO (and by default all execution activity) - to stop. + to stop. the joinexec parameter is obsolete. """ current = threading.currentThread() if self._receiverthread.isAlive(): self._trace("joining receiver thread") self._receiverthread.join() - if joinexec: - if self._workerpool: - self._workerpool.join() - self._trace("joining execution threads finished, current %r" % current) def getid(gw, cache={}): name = gw.__class__.__name__ Modified: py/branch/lessthread/py/execnet/register.py ============================================================================== --- py/branch/lessthread/py/execnet/register.py (original) +++ py/branch/lessthread/py/execnet/register.py Tue Mar 6 18:51:31 2007 @@ -11,7 +11,6 @@ startup_modules = [ 'py.__.thread.io', - 'py.__.thread.pool', 'py.__.execnet.inputoutput', 'py.__.execnet.gateway', 'py.__.execnet.message', @@ -41,7 +40,7 @@ bootstrap = [extra] bootstrap += [getsource(x) for x in startup_modules] bootstrap += [io.server_stmt, - "Gateway(io=io, _startcount=2)._servemain()", + "Gateway(io=io, allowexec=True, _startcount=2)._servemain()", ] source = "\n".join(bootstrap) self._trace("sending gateway bootstrap code") From fijal at codespeak.net Tue Mar 6 19:02:29 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 19:02:29 +0100 (CET) Subject: [py-svn] r39994 - py/trunk/py/doc/talk Message-ID: <20070306180229.B53CD1006E@code0.codespeak.net> Author: fijal Date: Tue Mar 6 19:02:28 2007 New Revision: 39994 Modified: py/trunk/py/doc/talk/notes.txt Log: fix rest Modified: py/trunk/py/doc/talk/notes.txt ============================================================================== --- py/trunk/py/doc/talk/notes.txt (original) +++ py/trunk/py/doc/talk/notes.txt Tue Mar 6 19:02:28 2007 @@ -8,6 +8,7 @@ information and other various fixes. * Improve distributed testing by: + - sharing even more code with normal testing - using greenexecnet wherever possible (falling back to normal execnet) From fijal at codespeak.net Tue Mar 6 19:06:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 19:06:38 +0100 (CET) Subject: [py-svn] r39995 - py/trunk/py/test/terminal Message-ID: <20070306180638.E7FA61006E@code0.codespeak.net> Author: fijal Date: Tue Mar 6 19:06:37 2007 New Revision: 39995 Modified: py/trunk/py/test/terminal/terminal.py Log: Possibly controversial checkin. Kill a hack for explicit isinstance(). We've got just two methods, so doing full getmro doesn't seem to make too much sense. Anyway this changes semantics slightly, so now we rely on exact inheritance rather than name (I don't have good answer for those) Modified: py/trunk/py/test/terminal/terminal.py ============================================================================== --- py/trunk/py/test/terminal/terminal.py (original) +++ py/trunk/py/test/terminal/terminal.py Tue Mar 6 19:06:37 2007 @@ -42,11 +42,15 @@ cls = getattr(colitem, '__class__', None) if cls is None: return - for typ in py.std.inspect.getmro(cls): - meth = getattr(self, 'start_%s' % typ.__name__, None) - if meth: - meth(colitem) - break + if issubclass(cls, py.test.collect.Module): + self.start_Module(colitem) + elif issubclass(cls, py.test.collect.Item): + self.start_Item(colitem) + #for typ in py.std.inspect.getmro(cls): + # meth = getattr(self, 'start_%s' % typ.__name__, None) + # if meth: + # meth(colitem) + # break colitem.start = py.std.time.time() def start_Module(self, colitem): From hpk at codespeak.net Tue Mar 6 22:06:32 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 6 Mar 2007 22:06:32 +0100 (CET) Subject: [py-svn] r40000 - in py/branch/lessthread/py/execnet: . testing Message-ID: <20070306210632.7AB8610069@code0.codespeak.net> Author: hpk Date: Tue Mar 6 22:06:30 2007 New Revision: 40000 Modified: py/branch/lessthread/py/execnet/gateway.py py/branch/lessthread/py/execnet/testing/test_gateway.py Log: re-adding thread support as a method "remote_init_threads" on top of the normal (by default non-thread) mechanisms Modified: py/branch/lessthread/py/execnet/gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/gateway.py (original) +++ py/branch/lessthread/py/execnet/gateway.py Tue Mar 6 22:06:30 2007 @@ -27,6 +27,8 @@ debug = 0 # open('/tmp/execnet-debug-%d' % os.getpid() , 'wa') sysex = (KeyboardInterrupt, SystemExit) +class StopExecLoop(Exception): + pass class Gateway(object): _ThreadOut = ThreadOut @@ -156,12 +158,32 @@ if item is None: self._stopsend() break - self._executetask(item) # could be done in an exec thread + stop = self._executetask(item) # could be done in an exec thread + if stop: + break finally: self._trace("_servemain finished") if self.joining: self.join() + def remote_init_threads(self, num=None): + """ start up to 'num' threads for subsequent + remote_exec() invocations to allow concurrent + execution. + """ + from py.__.thread import pool + source = py.code.Source(pool, """ + execpool = WorkerPool(maxthreads=%r) + gw = channel.gateway + while 1: + task = gw._requestqueue.get() + if task is None: + execpool.shutdown() + raise StopExecLoop + execpool.dispatch(gw._executetask, task) + """ % num) + self.remote_exec(source) + def _executetask(self, item): """ execute channel/source items. """ from sys import exc_info @@ -179,6 +201,9 @@ self._trace("execution finished:", repr(source)[:50]) except (KeyboardInterrupt, SystemExit): pass + except StopExecLoop: + channel.close() + return True except: excinfo = exc_info() l = traceback.format_exception(*excinfo) Modified: py/branch/lessthread/py/execnet/testing/test_gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/testing/test_gateway.py (original) +++ py/branch/lessthread/py/execnet/testing/test_gateway.py Tue Mar 6 22:06:30 2007 @@ -485,3 +485,19 @@ # now it did py.test.raises(IOError, gw.remote_exec, "...") +def test_threads(): + gw = py.execnet.PopenGateway() + gw.remote_init_threads(3) + c1 = gw.remote_exec("channel.send(channel.receive())") + c2 = gw.remote_exec("channel.send(channel.receive())") + c2.send(1) + res = c2.receive() + assert res == 1 + c1.send(42) + res = c1.receive() + assert res == 42 + gw.exit() + + + + From fijal at codespeak.net Tue Mar 6 22:11:14 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 22:11:14 +0100 (CET) Subject: [py-svn] r40001 - py/trunk/py Message-ID: <20070306211114.6C6BE1006E@code0.codespeak.net> Author: fijal Date: Tue Mar 6 22:11:12 2007 New Revision: 40001 Modified: py/trunk/py/__init__.py Log: Encoding (let's suppose UTF-8 will not hurt) Modified: py/trunk/py/__init__.py ============================================================================== --- py/trunk/py/__init__.py (original) +++ py/trunk/py/__init__.py Tue Mar 6 22:11:12 2007 @@ -1,3 +1,5 @@ + +# -*- coding: utf-8 -*- """ the py lib is a development support library featuring py.test, ad-hoc distributed execution, micro-threads From fijal at codespeak.net Tue Mar 6 22:13:34 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Mar 2007 22:13:34 +0100 (CET) Subject: [py-svn] r40002 - py/trunk/py/misc/testing Message-ID: <20070306211334.ADE151006E@code0.codespeak.net> Author: fijal Date: Tue Mar 6 22:13:33 2007 New Revision: 40002 Modified: py/trunk/py/misc/testing/test_initpkg.py Log: Re-enable url check test Modified: py/trunk/py/misc/testing/test_initpkg.py ============================================================================== --- py/trunk/py/misc/testing/test_initpkg.py (original) +++ py/trunk/py/misc/testing/test_initpkg.py Tue Mar 6 22:13:33 2007 @@ -253,7 +253,7 @@ # #assert False def test_url_of_version(): - py.test.skip("FAILING! - provide a proper URL or upload pylib tgz") + #py.test.skip("FAILING! - provide a proper URL or upload pylib tgz") from urllib import URLopener URLopener().open(py.__package__.download_url) From hpk at codespeak.net Tue Mar 6 22:23:30 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 6 Mar 2007 22:23:30 +0100 (CET) Subject: [py-svn] r40003 - in py/branch/lessthread/py/execnet: . testing Message-ID: <20070306212330.E05DA10063@code0.codespeak.net> Author: hpk Date: Tue Mar 6 22:23:30 2007 New Revision: 40003 Modified: py/branch/lessthread/py/execnet/gateway.py py/branch/lessthread/py/execnet/testing/test_gateway.py Log: refine thread support a bit Modified: py/branch/lessthread/py/execnet/gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/gateway.py (original) +++ py/branch/lessthread/py/execnet/gateway.py Tue Mar 6 22:23:30 2007 @@ -158,8 +158,9 @@ if item is None: self._stopsend() break - stop = self._executetask(item) # could be done in an exec thread - if stop: + try: + self._executetask(item) + except StopExecLoop: break finally: self._trace("_servemain finished") @@ -171,6 +172,8 @@ remote_exec() invocations to allow concurrent execution. """ + if hasattr(self, '_remotechannelthread'): + raise IOError("remote threads already running") from py.__.thread import pool source = py.code.Source(pool, """ execpool = WorkerPool(maxthreads=%r) @@ -182,7 +185,7 @@ raise StopExecLoop execpool.dispatch(gw._executetask, task) """ % num) - self.remote_exec(source) + self._remotechannelthread = self.remote_exec(source) def _executetask(self, item): """ execute channel/source items. """ @@ -203,7 +206,7 @@ pass except StopExecLoop: channel.close() - return True + raise except: excinfo = exc_info() l = traceback.format_exception(*excinfo) Modified: py/branch/lessthread/py/execnet/testing/test_gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/testing/test_gateway.py (original) +++ py/branch/lessthread/py/execnet/testing/test_gateway.py Tue Mar 6 22:23:30 2007 @@ -498,6 +498,10 @@ assert res == 42 gw.exit() - +def test_threads_twice(): + gw = py.execnet.PopenGateway() + gw.remote_init_threads(3) + py.test.raises(IOError, gw.remote_init_threads, 3) + gw.exit() From hpk at codespeak.net Wed Mar 7 18:10:58 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 7 Mar 2007 18:10:58 +0100 (CET) Subject: [py-svn] r40039 - in py/branch/lessthread/py/execnet: . testing Message-ID: <20070307171058.63E9F1006E@code0.codespeak.net> Author: hpk Date: Wed Mar 7 18:10:57 2007 New Revision: 40039 Modified: py/branch/lessthread/py/execnet/gateway.py py/branch/lessthread/py/execnet/testing/test_gateway.py Log: providing a nice error/message when you try to execute on a remote gateway that does not allow it (which is the default now) Modified: py/branch/lessthread/py/execnet/gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/gateway.py (original) +++ py/branch/lessthread/py/execnet/gateway.py Wed Mar 7 18:10:57 2007 @@ -24,7 +24,7 @@ ThreadOut = py._thread.ThreadOut import os -debug = 0 # open('/tmp/execnet-debug-%d' % os.getpid() , 'wa') +debug = open('/tmp/execnet-debug-%d' % os.getpid() , 'wa') sysex = (KeyboardInterrupt, SystemExit) class StopExecLoop(Exception): @@ -149,6 +149,10 @@ def _local_schedulexec(self, channel, sourcetask): if self._requestqueue is not None: self._requestqueue.put((channel, sourcetask)) + else: + # we will not execute, let's send back an error + # to inform the other side + channel.close("execution disallowed") def _servemain(self, joining=True): from sys import exc_info Modified: py/branch/lessthread/py/execnet/testing/test_gateway.py ============================================================================== --- py/branch/lessthread/py/execnet/testing/test_gateway.py (original) +++ py/branch/lessthread/py/execnet/testing/test_gateway.py Wed Mar 7 18:10:57 2007 @@ -372,6 +372,18 @@ res = channel.receive() assert res == 42 + def test_non_reverse_execution(self): + gw = self.gw + c1 = gw.remote_exec(""" + c = channel.gateway.remote_exec("pass") + try: + c.waitclose() + except c.RemoteError, e: + channel.send(str(e)) + """) + text = c1.receive() + assert text.find("execution disallowed") != -1 + #class TestBlockingIssues: # def test_join_blocked_execution_gateway(self): # gateway = py.execnet.PopenGateway() @@ -497,7 +509,7 @@ res = c1.receive() assert res == 42 gw.exit() - + def test_threads_twice(): gw = py.execnet.PopenGateway() gw.remote_init_threads(3) From arigo at codespeak.net Sun Mar 18 16:59:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 18 Mar 2007 16:59:47 +0100 (CET) Subject: [py-svn] r40702 - in py/trunk/py/test: . testing Message-ID: <20070318155947.0A7401007D@code0.codespeak.net> Author: arigo Date: Sun Mar 18 16:59:45 2007 New Revision: 40702 Modified: py/trunk/py/test/collect.py py/trunk/py/test/item.py py/trunk/py/test/testing/test_setup_nested.py Log: (pedronis, arigo) Add setup/teardown calls around generators, with the same semantics as around functions and methods. Modified: py/trunk/py/test/collect.py ============================================================================== --- py/trunk/py/test/collect.py (original) +++ py/trunk/py/test/collect.py Sun Mar 18 16:59:45 2007 @@ -442,7 +442,9 @@ Collector.Function.__get__(self)) # XXX for python 2.2 Function = property(Function) -class Generator(PyCollectorMixin, Collector): +from py.__.test.item import FunctionMixin # XXX import order issues :-( + +class Generator(FunctionMixin, PyCollectorMixin, Collector): def run(self): self._prepare() itemlist = self._name2items @@ -468,13 +470,6 @@ call, args = obj, () return call, args - def _getpathlineno(self): - code = py.code.Code(self.obj) - return code.path, code.firstlineno - - def _getsortvalue(self): - return self._getpathlineno() - class DoctestFile(PyCollectorMixin, FSCollector): def run(self): return [self.fspath.basename] Modified: py/trunk/py/test/item.py ============================================================================== --- py/trunk/py/test/item.py (original) +++ py/trunk/py/test/item.py Sun Mar 18 16:59:45 2007 @@ -37,38 +37,15 @@ def finishcapture(self): self._config._finishcapture(self) -class Function(Item): - """ a Function Item is responsible for setting up - and executing a Python callable test object. +class FunctionMixin(object): + """ mixin for the code common to Function and Generator. """ - _state = SetupState() - def __init__(self, name, parent, args=(), obj=_dummy, sort_value = None): - super(Function, self).__init__(name, parent) - self._args = args - if obj is not _dummy: - self._obj = obj - self._sort_value = sort_value - - def __repr__(self): - return "<%s %r>" %(self.__class__.__name__, self.name) - def _getpathlineno(self): code = py.code.Code(self.obj) return code.path, code.firstlineno def _getsortvalue(self): - if self._sort_value is None: - return self._getpathlineno() - return self._sort_value - - def run(self): - """ setup and execute the underlying test function. """ - self._state.prepare(self) - self.execute(self.obj, *self._args) - - def execute(self, target, *args): - """ execute the given test function. """ - target(*args) + return self._getpathlineno() def setup(self): """ perform setup for this test function. """ @@ -92,6 +69,35 @@ if meth is not None: return meth(self.obj) +class Function(FunctionMixin, Item): + """ a Function Item is responsible for setting up + and executing a Python callable test object. + """ + _state = SetupState() + def __init__(self, name, parent, args=(), obj=_dummy, sort_value = None): + super(Function, self).__init__(name, parent) + self._args = args + if obj is not _dummy: + self._obj = obj + self._sort_value = sort_value + + def __repr__(self): + return "<%s %r>" %(self.__class__.__name__, self.name) + + def _getsortvalue(self): + if self._sort_value is None: + return self._getpathlineno() + return self._sort_value + + def run(self): + """ setup and execute the underlying test function. """ + self._state.prepare(self) + self.execute(self.obj, *self._args) + + def execute(self, target, *args): + """ execute the given test function. """ + target(*args) + # # triggering specific outcomes while executing Items # Modified: py/trunk/py/test/testing/test_setup_nested.py ============================================================================== --- py/trunk/py/test/testing/test_setup_nested.py (original) +++ py/trunk/py/test/testing/test_setup_nested.py Sun Mar 18 16:59:45 2007 @@ -42,14 +42,23 @@ class TestSetupTeardownOnInstance(TestSimpleClassSetup): def setup_method(self, method): - self.clslevel.append(17) + self.clslevel.append(method.__name__) def teardown_method(self, method): x = self.clslevel.pop() - assert x == 17 + assert x == method.__name__ def test_setup(self): - assert self.clslevel[-1] == 17 + assert self.clslevel[-1] == 'test_setup' + + def test_generate(self): + assert self.clslevel[-1] == 'test_generate' + yield self.generated, 5 + assert self.clslevel[-1] == 'test_generate' + + def generated(self, value): + assert value == 5 + assert self.clslevel[-1] == 'test_generate' def test_teardown_method_worked(): assert not TestSetupTeardownOnInstance.clslevel From hpk at codespeak.net Sun Mar 18 17:28:06 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 18 Mar 2007 17:28:06 +0100 (CET) Subject: [py-svn] r40705 - in py/trunk/py: green green/pipe green/server green/test net Message-ID: <20070318162806.729771007F@code0.codespeak.net> Author: hpk Date: Sun Mar 18 17:28:04 2007 New Revision: 40705 Added: py/trunk/py/green/ - copied from r40102, py/trunk/py/net/ Removed: py/trunk/py/net/ Modified: py/trunk/py/green/greenexecnet.py py/trunk/py/green/pipe/common.py py/trunk/py/green/pipe/fd.py py/trunk/py/green/pipe/gsocket.py py/trunk/py/green/pipe/mp.py py/trunk/py/green/server/httpserver.py py/trunk/py/green/test/test_greenexecnet.py py/trunk/py/green/test/test_greensock2.py py/trunk/py/green/test/test_pipelayer.py Log: move greenlet support objects to the "py/green" directory. Modified: py/trunk/py/green/greenexecnet.py ============================================================================== --- py/trunk/py/net/greenexecnet.py (original) +++ py/trunk/py/green/greenexecnet.py Sun Mar 18 17:28:04 2007 @@ -13,8 +13,8 @@ """ import sys, os, py, inspect -from py.__.net import greensock2 -from py.__.net.msgstruct import message, decodemessage +from py.__.green import greensock2 +from py.__.green.msgstruct import message, decodemessage MSG_REMOTE_EXEC = 'r' MSG_OBJECT = 'o' @@ -163,7 +163,7 @@ action = "exec input()" def __init__(self, cmdline): - from py.__.net.pipe.fd import FDInput, FDOutput + from py.__.green.pipe.fd import FDInput, FDOutput child_in, child_out = os.popen2(cmdline, 't', 0) fdin = FDInput(child_out.fileno(), child_out.close) fdout = FDOutput(child_in.fileno(), child_in.close) @@ -173,14 +173,14 @@ def get_bootstrap_code(): # XXX assumes that the py lib is installed on the remote side src = [] - src.append('from py.__.net import greenexecnet') + src.append('from py.__.green import greenexecnet') src.append('greenexecnet.PopenCmdGateway.run_server()') src.append('') return '%r\n' % ('\n'.join(src),) get_bootstrap_code = staticmethod(get_bootstrap_code) def run_server(): - from py.__.net.pipe.fd import FDInput, FDOutput + from py.__.green.pipe.fd import FDInput, FDOutput gw = Gateway(input = FDInput(os.dup(0)), output = FDOutput(os.dup(1)), is_remote = True) Modified: py/trunk/py/green/pipe/common.py ============================================================================== --- py/trunk/py/net/pipe/common.py (original) +++ py/trunk/py/green/pipe/common.py Sun Mar 18 17:28:04 2007 @@ -1,4 +1,4 @@ -from py.__.net import greensock2 +from py.__.green import greensock2 VERBOSE = True Modified: py/trunk/py/green/pipe/fd.py ============================================================================== --- py/trunk/py/net/pipe/fd.py (original) +++ py/trunk/py/green/pipe/fd.py Sun Mar 18 17:28:04 2007 @@ -1,5 +1,5 @@ import os -from py.__.net import greensock2 +from py.__.green import greensock2 class FDInput(object): Modified: py/trunk/py/green/pipe/gsocket.py ============================================================================== --- py/trunk/py/net/pipe/gsocket.py (original) +++ py/trunk/py/green/pipe/gsocket.py Sun Mar 18 17:28:04 2007 @@ -1,4 +1,4 @@ -from py.__.net import greensock2 +from py.__.green import greensock2 import socket, errno, os error = socket.error Modified: py/trunk/py/green/pipe/mp.py ============================================================================== --- py/trunk/py/net/pipe/mp.py (original) +++ py/trunk/py/green/pipe/mp.py Sun Mar 18 17:28:04 2007 @@ -1,4 +1,4 @@ -from py.__.net.pipe.common import BufferedInput +from py.__.green.pipe.common import BufferedInput class MeetingPointInput(BufferedInput): Modified: py/trunk/py/green/server/httpserver.py ============================================================================== --- py/trunk/py/net/server/httpserver.py (original) +++ py/trunk/py/green/server/httpserver.py Sun Mar 18 17:28:04 2007 @@ -1,6 +1,6 @@ import BaseHTTPServer -from py.__.net import greensock2 -from py.__.net.pipe.gsocket import GreenSocket +from py.__.green import greensock2 +from py.__.green.pipe.gsocket import GreenSocket class GreenMixIn: Modified: py/trunk/py/green/test/test_greenexecnet.py ============================================================================== --- py/trunk/py/net/test/test_greenexecnet.py (original) +++ py/trunk/py/green/test/test_greenexecnet.py Sun Mar 18 17:28:04 2007 @@ -1,5 +1,5 @@ import py -from py.__.net.greenexecnet import * +from py.__.green.greenexecnet import * def test_simple(): gw = PopenGateway() Modified: py/trunk/py/green/test/test_greensock2.py ============================================================================== --- py/trunk/py/net/test/test_greensock2.py (original) +++ py/trunk/py/green/test/test_greensock2.py Sun Mar 18 17:28:04 2007 @@ -1,6 +1,6 @@ import py from socket import * -from py.__.net.greensock2 import * +from py.__.green.greensock2 import * def test_meetingpoint(): giv1, acc1 = meetingpoint() Modified: py/trunk/py/green/test/test_pipelayer.py ============================================================================== --- py/trunk/py/net/test/test_pipelayer.py (original) +++ py/trunk/py/green/test/test_pipelayer.py Sun Mar 18 17:28:04 2007 @@ -1,5 +1,5 @@ import os, random -from py.__.net.pipelayer import PipeLayer, pipe_over_udp, PipeOverUdp +from py.__.green.pipelayer import PipeLayer, pipe_over_udp, PipeOverUdp def test_simple(): data1 = os.urandom(1000) From hpk at codespeak.net Sun Mar 18 17:33:15 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 18 Mar 2007 17:33:15 +0100 (CET) Subject: [py-svn] r40706 - py/trunk/py/green/test Message-ID: <20070318163315.61B2B1007F@code0.codespeak.net> Author: hpk Date: Sun Mar 18 17:33:13 2007 New Revision: 40706 Added: py/trunk/py/green/test/__init__.py (contents, props changed) Log: test/__init__.py is needed as well (for determining test paths etc.) Added: py/trunk/py/green/test/__init__.py ============================================================================== --- (empty file) +++ py/trunk/py/green/test/__init__.py Sun Mar 18 17:33:13 2007 @@ -0,0 +1 @@ +# From hpk at codespeak.net Sun Mar 18 17:37:55 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 18 Mar 2007 17:37:55 +0100 (CET) Subject: [py-svn] r40707 - py/dist Message-ID: <20070318163755.6F3771007F@code0.codespeak.net> Author: hpk Date: Sun Mar 18 17:37:53 2007 New Revision: 40707 Added: py/dist/ - copied from r40706, py/trunk/ Log: merging py lib trunk to dist From fijal at codespeak.net Sun Mar 18 19:47:29 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Mar 2007 19:47:29 +0100 (CET) Subject: [py-svn] r40719 - py/trunk/py/green/pipe Message-ID: <20070318184729.355F71008D@code0.codespeak.net> Author: fijal Date: Sun Mar 18 19:47:26 2007 New Revision: 40719 Modified: py/trunk/py/green/pipe/gsocket.py Log: Argh. There should be post-commit hook which warns about every import pdb in your code. Modified: py/trunk/py/green/pipe/gsocket.py ============================================================================== --- py/trunk/py/green/pipe/gsocket.py (original) +++ py/trunk/py/green/pipe/gsocket.py Sun Mar 18 19:47:26 2007 @@ -36,7 +36,6 @@ s, addr = self._s.accept() break except error, e: - import pdb;pdb.set_trace() if e.args[0] not in (errno.EAGAIN, errno.EWOULDBLOCK): raise self.wait_input() From fijal at codespeak.net Sun Mar 18 19:49:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Mar 2007 19:49:48 +0100 (CET) Subject: [py-svn] r40720 - py/trunk/py/green Message-ID: <20070318184948.8CFA41008D@code0.codespeak.net> Author: fijal Date: Sun Mar 18 19:49:45 2007 New Revision: 40720 Modified: py/trunk/py/green/greensock2.py Log: Tracing defaults to false Modified: py/trunk/py/green/greensock2.py ============================================================================== --- py/trunk/py/green/greensock2.py (original) +++ py/trunk/py/green/greensock2.py Sun Mar 18 19:49:45 2007 @@ -16,7 +16,7 @@ from time import time as _time from heapq import heappush, heappop, heapify -TRACE = True +TRACE = False def meetingpoint(): senders = deque() # list of senders, or [None] if Giver closed From fijal at codespeak.net Sun Mar 18 23:39:03 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Mar 2007 23:39:03 +0100 (CET) Subject: [py-svn] r40737 - in py/trunk/py/test/rsession: . testing Message-ID: <20070318223903.080D310092@code0.codespeak.net> Author: fijal Date: Sun Mar 18 23:38:57 2007 New Revision: 40737 Modified: py/trunk/py/test/rsession/reporter.py py/trunk/py/test/rsession/testing/test_reporter.py Log: A fix that allows (for command line reporter) to report failure in case when FAILED TO LOAD MODULE Modified: py/trunk/py/test/rsession/reporter.py ============================================================================== --- py/trunk/py/test/rsession/reporter.py (original) +++ py/trunk/py/test/rsession/reporter.py Sun Mar 18 23:38:57 2007 @@ -280,6 +280,8 @@ def report_FailedTryiter(self, event): self.out.line("FAILED TO LOAD MODULE: %s\n" % "/".join(event.item.listnames())) self.failed_tests_outcome.append(event) + # argh! bad hack, need to fix it + self.failed[self.hosts[0]] += 1 def report_SkippedTryiter(self, event): self.out.line("Skipped (%s) %s\n" % (str(event.excinfo.value), "/". @@ -301,6 +303,7 @@ #self.show_item(event.item, False) self.out.write("- FAILED TO LOAD MODULE") self.failed_tests_outcome.append(event) + self.failed[self.hosts[0]] += 1 def report_ReceivedItemOutcome(self, event): host = self.hosts[0] Modified: py/trunk/py/test/rsession/testing/test_reporter.py ============================================================================== --- py/trunk/py/test/rsession/testing/test_reporter.py (original) +++ py/trunk/py/test/rsession/testing/test_reporter.py Sun Mar 18 23:38:57 2007 @@ -135,11 +135,13 @@ r.report(repevent.RsyncFinished()) list(rootcol._tryiter(reporterror=lambda x : AbstractSession.reporterror(r.report, x))) r.report(repevent.TestFinished()) + return r cap = py.io.StdCaptureFD() - boxfun() + r = boxfun() out, err = cap.reset() assert not err + assert out.find("1 failed in") != -1 assert out.find("NameError: name 'sadsadsa' is not defined") != -1 def _test_still_to_go(self): @@ -187,7 +189,7 @@ repmod/test_one.py[1] repmod/test_three.py[0] - FAILED TO LOAD MODULE repmod/test_two.py[0] - skipped (reason)""" - assert received.find(expected) != -1 + assert received.find(expected) != -1 class TestRemoteReporter(AbstractTestReporter): reporter = RemoteReporter From fijal at codespeak.net Sun Mar 18 23:44:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Mar 2007 23:44:57 +0100 (CET) Subject: [py-svn] r40738 - py/trunk/py/test/rsession Message-ID: <20070318224457.2460910092@code0.codespeak.net> Author: fijal Date: Sun Mar 18 23:44:54 2007 New Revision: 40738 Modified: py/trunk/py/test/rsession/web.py Log: Count also FAILED TO LOAD MODULE failures in web interface Modified: py/trunk/py/test/rsession/web.py ============================================================================== --- py/trunk/py/test/rsession/web.py (original) +++ py/trunk/py/test/rsession/web.py Sun Mar 18 23:44:54 2007 @@ -286,6 +286,11 @@ def report_ReceivedItemOutcome(self, event): self.all += 1 self.pending_events.put(event) + + def report_FailedTryiter(self, event): + fullitemname = "/".join(event.item.listnames()) + self.fail_reasons[fullitemname] = '' + self.pending_events.put(event) def report_ItemStart(self, event): if isinstance(event.item, py.test.collect.Module): From fijal at codespeak.net Sun Mar 18 23:53:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Mar 2007 23:53:16 +0100 (CET) Subject: [py-svn] r40739 - in py/trunk/py/test/rsession: . webdata Message-ID: <20070318225316.3D04E10092@code0.codespeak.net> Author: fijal Date: Sun Mar 18 23:53:14 2007 New Revision: 40739 Modified: py/trunk/py/test/rsession/web.py py/trunk/py/test/rsession/webdata/source.js py/trunk/py/test/rsession/webjs.py Log: Make FAILED TO LOAD MODULE clickable Modified: py/trunk/py/test/rsession/web.py ============================================================================== --- py/trunk/py/test/rsession/web.py (original) +++ py/trunk/py/test/rsession/web.py Sun Mar 18 23:53:14 2007 @@ -276,7 +276,7 @@ def repr_source(self, relline, source): lines = [] - for num, line in enumerate(source.split("\n")): + for num, line in enumerate(str(source).split("\n")): if num == relline: lines.append(">>>>" + line) else: @@ -289,7 +289,10 @@ def report_FailedTryiter(self, event): fullitemname = "/".join(event.item.listnames()) - self.fail_reasons[fullitemname] = '' + self.fail_reasons[fullitemname] = self.repr_failure_tblong( + event.item, event.excinfo, event.excinfo.traceback) + self.stdout[fullitemname] = '' + self.stderr[fullitemname] = '' self.pending_events.put(event) def report_ItemStart(self, event): Modified: py/trunk/py/test/rsession/webdata/source.js ============================================================================== Binary files. No diff available. Modified: py/trunk/py/test/rsession/webjs.py ============================================================================== --- py/trunk/py/test/rsession/webjs.py (original) +++ py/trunk/py/test/rsession/webjs.py Sun Mar 18 23:53:14 2007 @@ -212,10 +212,16 @@ return True tr = create_elem("tr") td = create_elem("td") + a = create_elem("a") + a.setAttribute("href", "javascript:show_traceback('%s')" % ( + msg['fullitemname'],)) txt = create_text_elem("- FAILED TO LOAD MODULE") - td.appendChild(txt) + a.appendChild(txt) + td.appendChild(a) tr.appendChild(td) module_part.appendChild(tr) + item_name = msg['fullitemname'] + exported_methods.show_fail(item_name, fail_come_back) elif msg['type'] == 'SkippedTryiter': module_part = get_elem(msg['fullitemname']) if not module_part: From fijal at codespeak.net Mon Mar 19 11:43:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Mar 2007 11:43:53 +0100 (CET) Subject: [py-svn] r40753 - in py/trunk/py/test/rsession: . webdata Message-ID: <20070319104353.76B331006F@code0.codespeak.net> Author: fijal Date: Mon Mar 19 11:43:49 2007 New Revision: 40753 Modified: py/trunk/py/test/rsession/webdata/source.js py/trunk/py/test/rsession/webjs.py Log: Make MAX_COUNTER small enough. It's an obscure hack, but at least works Modified: py/trunk/py/test/rsession/webdata/source.js ============================================================================== Binary files. No diff available. Modified: py/trunk/py/test/rsession/webjs.py ============================================================================== --- py/trunk/py/test/rsession/webjs.py (original) +++ py/trunk/py/test/rsession/webjs.py Mon Mar 19 11:43:49 2007 @@ -25,7 +25,7 @@ max_items = {} short_item_names = {} -MAX_COUNTER = 50 # Maximal size of one-line table +MAX_COUNTER = 30 # Maximal size of one-line table class Globals(object): def __init__(self): From hpk at codespeak.net Tue Mar 20 13:10:57 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 20 Mar 2007 13:10:57 +0100 (CET) Subject: [py-svn] r40831 - py/trunk/py Message-ID: <20070320121057.6537510060@code0.codespeak.net> Author: hpk Date: Tue Mar 20 13:10:56 2007 New Revision: 40831 Modified: py/trunk/py/__init__.py Log: bump version number, fixes Modified: py/trunk/py/__init__.py ============================================================================== --- py/trunk/py/__init__.py (original) +++ py/trunk/py/__init__.py Tue Mar 20 13:10:56 2007 @@ -7,7 +7,7 @@ """ from initpkg import initpkg -version = "0.9.0" +version = "0.9.1-alpha" initpkg(__name__, description = "py lib: agile development and test support library", @@ -15,9 +15,9 @@ lastchangedate = '$LastChangedDate$', version = version, url = "http://codespeak.net/py", - download_url = "http://codespeak.net/download/py/py-%s.tar.gz" %(version,), + download_url = "xxx" # "http://codespeak.net/download/py/py-%s.tar.gz" %(version,), license = "MIT license", - platforms = ['unix', 'linux', 'cygwin'], + platforms = ['unix', 'linux', 'cygwin', 'win32'], author = "holger krekel, Carl Friedrich Bolz, Guido Wesdorp, Maciej Fijalkowski, Armin Rigo & others", author_email = "py-dev at codespeak.net", long_description = globals()['__doc__'], From hpk at codespeak.net Tue Mar 20 13:11:31 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 20 Mar 2007 13:11:31 +0100 (CET) Subject: [py-svn] r40832 - py/trunk/py Message-ID: <20070320121131.EB88210060@code0.codespeak.net> Author: hpk Date: Tue Mar 20 13:11:31 2007 New Revision: 40832 Modified: py/trunk/py/__init__.py Log: fixing syntax error Modified: py/trunk/py/__init__.py ============================================================================== --- py/trunk/py/__init__.py (original) +++ py/trunk/py/__init__.py Tue Mar 20 13:11:31 2007 @@ -15,7 +15,7 @@ lastchangedate = '$LastChangedDate$', version = version, url = "http://codespeak.net/py", - download_url = "xxx" # "http://codespeak.net/download/py/py-%s.tar.gz" %(version,), + download_url = "xxx", # "http://codespeak.net/download/py/py-%s.tar.gz" %(version,), license = "MIT license", platforms = ['unix', 'linux', 'cygwin', 'win32'], author = "holger krekel, Carl Friedrich Bolz, Guido Wesdorp, Maciej Fijalkowski, Armin Rigo & others", From hpk at codespeak.net Tue Mar 20 13:20:34 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 20 Mar 2007 13:20:34 +0100 (CET) Subject: [py-svn] r40834 - in py/trunk/py: . misc/testing Message-ID: <20070320122034.D82CA10050@code0.codespeak.net> Author: hpk Date: Tue Mar 20 13:20:33 2007 New Revision: 40834 Modified: py/trunk/py/__init__.py py/trunk/py/misc/testing/test_initpkg.py Log: skip test for XXX'ed download urls if version contains "alpha" Modified: py/trunk/py/__init__.py ============================================================================== --- py/trunk/py/__init__.py (original) +++ py/trunk/py/__init__.py Tue Mar 20 13:20:33 2007 @@ -15,7 +15,7 @@ lastchangedate = '$LastChangedDate$', version = version, url = "http://codespeak.net/py", - download_url = "xxx", # "http://codespeak.net/download/py/py-%s.tar.gz" %(version,), + download_url = "XXX", # "http://codespeak.net/download/py/py-%s.tar.gz" %(version,), license = "MIT license", platforms = ['unix', 'linux', 'cygwin', 'win32'], author = "holger krekel, Carl Friedrich Bolz, Guido Wesdorp, Maciej Fijalkowski, Armin Rigo & others", Modified: py/trunk/py/misc/testing/test_initpkg.py ============================================================================== --- py/trunk/py/misc/testing/test_initpkg.py (original) +++ py/trunk/py/misc/testing/test_initpkg.py Tue Mar 20 13:20:33 2007 @@ -255,5 +255,10 @@ def test_url_of_version(): #py.test.skip("FAILING! - provide a proper URL or upload pylib tgz") from urllib import URLopener - URLopener().open(py.__package__.download_url) + url = py.__package__.download_url + if url.lower() == "xxx": + assert py.__package__.version.find("alpha") != -1 + else: + URLopener().open(url) + From hpk at codespeak.net Tue Mar 20 13:22:41 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 20 Mar 2007 13:22:41 +0100 (CET) Subject: [py-svn] r40835 - py/dist Message-ID: <20070320122241.AC13C10050@code0.codespeak.net> Author: hpk Date: Tue Mar 20 13:22:40 2007 New Revision: 40835 Added: py/dist/ - copied from r40834, py/trunk/ Log: remerge py trunk From fijal at codespeak.net Wed Mar 21 15:49:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Mar 2007 15:49:51 +0100 (CET) Subject: [py-svn] r40943 - py/trunk/py/test/rsession/webdata Message-ID: <20070321144951.BA8C91009F@code0.codespeak.net> Author: fijal Date: Wed Mar 21 15:49:49 2007 New Revision: 40943 Modified: py/trunk/py/test/rsession/webdata/source.js Log: Regenerate js Modified: py/trunk/py/test/rsession/webdata/source.js ============================================================================== Binary files. No diff available. From fijal at codespeak.net Wed Mar 21 22:22:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Mar 2007 22:22:57 +0100 (CET) Subject: [py-svn] r40982 - py/trunk/py/green Message-ID: <20070321212257.4AF0910072@code0.codespeak.net> Author: fijal Date: Wed Mar 21 22:22:56 2007 New Revision: 40982 Modified: py/trunk/py/green/greensock2.py Log: Possible problem-point (exploded at least once there) Modified: py/trunk/py/green/greensock2.py ============================================================================== --- py/trunk/py/green/greensock2.py (original) +++ py/trunk/py/green/greensock2.py Wed Mar 21 22:22:56 2007 @@ -448,6 +448,7 @@ if s in g_owtd: d = g_owtd[s] #print 'owtd:', d[0] + # XXX: Check if d is non-empty g = d.popleft() if not d: try: @@ -461,6 +462,7 @@ if s in g_iwtd: d = g_iwtd[s] #print 'iwtd:', d[0] + # XXX: Check if d is non-empty g = d.popleft() if not d: try: From guido at codespeak.net Thu Mar 22 16:06:43 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 22 Mar 2007 16:06:43 +0100 (CET) Subject: [py-svn] r41080 - in py/trunk/py: apigen test/rsession/webdata Message-ID: <20070322150643.A5DFE10074@code0.codespeak.net> Author: guido Date: Thu Mar 22 16:06:41 2007 New Revision: 41080 Modified: py/trunk/py/apigen/apigen.py py/trunk/py/apigen/htmlgen.py py/trunk/py/test/rsession/webdata/index.html Log: Added support for filtering listdir() calls in the SourcePageBuilder, using it to filter out hidden files and the 'build' subdir of greenlet. Modified: py/trunk/py/apigen/apigen.py ============================================================================== --- py/trunk/py/apigen/apigen.py (original) +++ py/trunk/py/apigen/apigen.py Thu Mar 22 16:06:41 2007 @@ -31,6 +31,11 @@ Channel.__apigen_hide_from_nav__ = True return pkgname, pkgdict +def sourcedirfilter(p): + return ('.svn' not in str(p).split(p.sep) and + not p.basename.startswith('.') and + str(p).find('c-extension%sgreenlet%sbuild' % (p.sep, p.sep)) == -1) + def build(pkgdir, dsa, capture): # create a linker (link database) for cross-linking l = linker.TempLinker() @@ -51,7 +56,7 @@ apb = htmlgen.ApiPageBuilder(targetdir, l, dsa, pkgdir, namespace_tree, proj, capture, LayoutPage) spb = htmlgen.SourcePageBuilder(targetdir, l, pkgdir, proj, capture, - LayoutPage) + LayoutPage, dirfilter=sourcedirfilter) apb.build_namespace_pages() class_names = dsa.get_class_names() Modified: py/trunk/py/apigen/htmlgen.py ============================================================================== --- py/trunk/py/apigen/htmlgen.py (original) +++ py/trunk/py/apigen/htmlgen.py Thu Mar 22 16:06:41 2007 @@ -81,7 +81,7 @@ return inspect.formatargspec(*inspect.getargspec(func)) # some helper functionality -def source_dirs_files(fspath): +def source_dirs_files(fspath, fil=None): """ returns a tuple (dirs, files) for fspath dirs are all the subdirs, files are the files which are interesting @@ -93,7 +93,7 @@ """ dirs = [] files = [] - for child in fspath.listdir(): + for child in fspath.listdir(fil=fil): if child.basename.startswith('.'): continue if child.check(dir=True): @@ -203,13 +203,14 @@ class SourcePageBuilder(AbstractPageBuilder): """ builds the html for a source docs page """ def __init__(self, base, linker, projroot, project, capture=None, - pageclass=LayoutPage): + pageclass=LayoutPage, dirfilter=None): self.base = base self.linker = linker self.projroot = projroot self.project = project self.capture = capture self.pageclass = pageclass + self.dirfilter = dirfilter def build_navigation(self, fspath): nav = H.Navigation(class_='sidebar') @@ -240,7 +241,7 @@ else: # we're a file, build our parent's children only dirpath = fspath.dirpath() - diritems, fileitems = source_dirs_files(dirpath) + diritems, fileitems = source_dirs_files(dirpath, self.dirfilter) for dir in diritems: nav.append(H.NavigationItem(self.linker, dir.strpath, dir.basename, indent, False)) @@ -265,7 +266,7 @@ return tag, nav def build_dir_page(self, fspath): - dirs, files = source_dirs_files(fspath) + dirs, files = source_dirs_files(fspath, self.dirfilter) dirs = [(p.basename, self.linker.get_lazyhref(str(p))) for p in dirs] files = [(p.basename, self.linker.get_lazyhref(str(p))) for p in files] tag = H.DirList(dirs, files) @@ -281,7 +282,15 @@ return tag, nav def build_pages(self, base): - for fspath in [base] + list(base.visit()): + def visit(p): + dirs, files = source_dirs_files(p, self.dirfilter) + for d in dirs: + yield d + for sp in visit(d): + yield sp + for f in files: + yield f + for fspath in [base] + list(visit(base)): if fspath.ext in ['.pyc', '.pyo']: continue if self.capture: Modified: py/trunk/py/test/rsession/webdata/index.html ============================================================================== --- py/trunk/py/test/rsession/webdata/index.html (original) +++ py/trunk/py/test/rsession/webdata/index.html Thu Mar 22 16:06:41 2007 @@ -62,10 +62,6 @@ border: 0px; } - #hosts { - width: 100%; - } - @@ -114,12 +110,10 @@
Data [hide]: - -
- + +
- -
- + +
From fijal at codespeak.net Sat Mar 24 10:20:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Mar 2007 10:20:27 +0100 (CET) Subject: [py-svn] r41224 - py/trunk/py/code/testing Message-ID: <20070324092027.2B93710070@code0.codespeak.net> Author: fijal Date: Sat Mar 24 10:20:25 2007 New Revision: 41224 Modified: py/trunk/py/code/testing/test_source.py Log: Added failing test, which I would suppose to pass Modified: py/trunk/py/code/testing/test_source.py ============================================================================== --- py/trunk/py/code/testing/test_source.py (original) +++ py/trunk/py/code/testing/test_source.py Sat Mar 24 10:20:25 2007 @@ -281,3 +281,15 @@ """ lines = deindent(source.splitlines()) assert lines == ['', 'def f():', ' def g():', ' pass', ' '] + +def test_write_read(): + py.test.skip("Failing") + tmpdir = py.test.ensuretemp("source_write_read") + source = py.code.Source(''' + class A(object): + def method(self): + x = 1 + ''') + tmpdir.ensure("a.py").write(source) + s2 = py.code.Source(tmpdir.join("a.py").pyimport().A) + assert source == s2 From arigo at codespeak.net Tue Mar 27 15:28:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Mar 2007 15:28:21 +0200 (CEST) Subject: [py-svn] r41480 - py/trunk/py/test Message-ID: <20070327132821.05E9910097@code0.codespeak.net> Author: arigo Date: Tue Mar 27 15:28:18 2007 New Revision: 41480 Modified: py/trunk/py/test/collect.py py/trunk/py/test/item.py Log: Move the FunctionMixin to collect.py, as an attempt to avoid circular imports. Modified: py/trunk/py/test/collect.py ============================================================================== --- py/trunk/py/test/collect.py (original) +++ py/trunk/py/test/collect.py Tue Mar 27 15:28:18 2007 @@ -442,7 +442,38 @@ Collector.Function.__get__(self)) # XXX for python 2.2 Function = property(Function) -from py.__.test.item import FunctionMixin # XXX import order issues :-( + +class FunctionMixin(object): + """ mixin for the code common to Function and Generator. + """ + def _getpathlineno(self): + code = py.code.Code(self.obj) + return code.path, code.firstlineno + + def _getsortvalue(self): + return self._getpathlineno() + + def setup(self): + """ perform setup for this test function. """ + if getattr(self.obj, 'im_self', None): + name = 'setup_method' + else: + name = 'setup_function' + obj = self.parent.obj + meth = getattr(obj, name, None) + if meth is not None: + return meth(self.obj) + + def teardown(self): + """ perform teardown for this test function. """ + if getattr(self.obj, 'im_self', None): + name = 'teardown_method' + else: + name = 'teardown_function' + obj = self.parent.obj + meth = getattr(obj, name, None) + if meth is not None: + return meth(self.obj) class Generator(FunctionMixin, PyCollectorMixin, Collector): def run(self): Modified: py/trunk/py/test/item.py ============================================================================== --- py/trunk/py/test/item.py (original) +++ py/trunk/py/test/item.py Tue Mar 27 15:28:18 2007 @@ -2,6 +2,7 @@ from inspect import isclass, ismodule from py.__.test.outcome import Skipped, Failed, Passed +from py.__.test.collect import FunctionMixin _dummy = object() @@ -37,38 +38,6 @@ def finishcapture(self): self._config._finishcapture(self) -class FunctionMixin(object): - """ mixin for the code common to Function and Generator. - """ - def _getpathlineno(self): - code = py.code.Code(self.obj) - return code.path, code.firstlineno - - def _getsortvalue(self): - return self._getpathlineno() - - def setup(self): - """ perform setup for this test function. """ - if getattr(self.obj, 'im_self', None): - name = 'setup_method' - else: - name = 'setup_function' - obj = self.parent.obj - meth = getattr(obj, name, None) - if meth is not None: - return meth(self.obj) - - def teardown(self): - """ perform teardown for this test function. """ - if getattr(self.obj, 'im_self', None): - name = 'teardown_method' - else: - name = 'teardown_function' - obj = self.parent.obj - meth = getattr(obj, name, None) - if meth is not None: - return meth(self.obj) - class Function(FunctionMixin, Item): """ a Function Item is responsible for setting up and executing a Python callable test object. From fijal at codespeak.net Thu Mar 29 10:47:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Mar 2007 10:47:01 +0200 (CEST) Subject: [py-svn] r41606 - py/trunk/py/green Message-ID: <20070329084701.5F5451007B@code0.codespeak.net> Author: fijal Date: Thu Mar 29 10:46:59 2007 New Revision: 41606 Modified: py/trunk/py/green/greensock2.py Log: Rather strange hack to make sure that queue is not empty. Needed nowadays for passing test_session in js/examples/console Modified: py/trunk/py/green/greensock2.py ============================================================================== --- py/trunk/py/green/greensock2.py (original) +++ py/trunk/py/green/greensock2.py Thu Mar 29 10:46:59 2007 @@ -449,13 +449,17 @@ d = g_owtd[s] #print 'owtd:', d[0] # XXX: Check if d is non-empty - g = d.popleft() + try: + g = d.popleft() + except IndexError: + g = None if not d: try: del g_owtd[s] except KeyError: pass - g.switch(g_owtd) + if g: + g.switch(g_owtd) # if g.dead: # check_waiters(g) for s in iwtd: @@ -463,13 +467,17 @@ d = g_iwtd[s] #print 'iwtd:', d[0] # XXX: Check if d is non-empty - g = d.popleft() + try: + g = d.popleft() + except IndexError: + g = None if not d: try: del g_iwtd[s] except KeyError: pass - g.switch(g_iwtd) + if g: + g.switch(g_iwtd) # if g.dead: # check_waiters(g) except GreenletExit: From guido at codespeak.net Thu Mar 29 13:56:17 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 29 Mar 2007 13:56:17 +0200 (CEST) Subject: [py-svn] r41620 - in py/trunk/py/apigen: . source testing Message-ID: <20070329115617.0F0381007B@code0.codespeak.net> Author: guido Date: Thu Mar 29 13:56:14 2007 New Revision: 41620 Modified: py/trunk/py/apigen/html.py py/trunk/py/apigen/htmlgen.py py/trunk/py/apigen/linker.py py/trunk/py/apigen/source/html.py py/trunk/py/apigen/testing/test_apigen_example.py py/trunk/py/apigen/testing/test_htmlgen.py py/trunk/py/apigen/testing/test_linker.py Log: Re-added anchors to the (full) source files, and using them from the API documentation. Modified: py/trunk/py/apigen/html.py ============================================================================== --- py/trunk/py/apigen/html.py (original) +++ py/trunk/py/apigen/html.py Thu Mar 29 13:56:14 2007 @@ -61,6 +61,7 @@ infoid = 'info_%s' % (localname.replace('.', '_dot_'),) docstringid = 'docstring_%s' % (localname.replace('.', '_dot_'),) fd = H.FunctionDef(localname, argdesc, + title='click to view details', onclick=('showhideel(' 'document.getElementById("%s")); ' % (infoid,))) Modified: py/trunk/py/apigen/htmlgen.py ============================================================================== --- py/trunk/py/apigen/htmlgen.py (original) +++ py/trunk/py/apigen/htmlgen.py Thu Mar 29 13:56:14 2007 @@ -18,6 +18,25 @@ REDUCE_CALLSITES = True +def find_method_origin(meth): + cls = getattr(meth, 'im_class', None) + if cls is None: + return None # XXX unknown origin (built-in function or method or sth) + name = meth.im_func.func_name + origin = cls + # XXX old-style classes support required? :| + mro = inspect.getmro(cls) + for base in mro: + m = getattr(base, name, None) + if m is None: + continue + if not hasattr(m, 'im_func'): + # builtin + return None + if m.im_func is meth.im_func: + origin = base + return origin + def is_navigateable(name): return (not is_private(name) and name != '__doc__') @@ -135,10 +154,17 @@ source_html.prepare_line([line], tokenizer, enc)) except py.error.ENOENT: # error reading source code, giving up - snippet = org + snippet = codelines break return snippet +def enumerate_and_color_module(path, enc): + snippet = H.SourceBlock() + tokenizer = source_color.Tokenizer(source_color.PythonSchema) + for i, text in enumerate(source_html.prepare_module(path, tokenizer, enc)): + snippet.add_line(i + 1, text) + return snippet + _get_obj_cache = {} def get_obj(dsa, pkg, dotted_name): full_dotted_name = '%s.%s' % (pkg.__name__, dotted_name) @@ -258,9 +284,16 @@ # XXX two reads of the same file here... not very bad (disk caches # and such) but also not very nice... enc = source_html.get_module_encoding(fspath.strpath) - source = fspath.read() - sep = get_linesep(source) - colored = [enumerate_and_color(source.split(sep), 0, enc)] + try: + colored = [enumerate_and_color_module(fspath, enc)] + except (KeyboardInterrupt, SystemExit): + raise + except Exception, e: + #self.capture.err.writeorg('\ncompilation exception: %s\n' % (e,)) + # problem building HTML with anchors; let's try without... + source = fspath.read() + sep = get_linesep(source) + colored = [enumerate_and_color(source.split(sep), 0, enc)] tag = H.PythonSource(colored) nav = self.build_navigation(fspath) return tag, nav @@ -316,7 +349,7 @@ if fspath.check(ext='.py'): try: tag, nav = self.build_python_page(fspath) - except (KeyboardInterrupt, SystemError): + except (KeyboardInterrupt, SystemExit): raise except: # XXX strange stuff going wrong at times... need to fix raise @@ -398,8 +431,8 @@ relpath = get_rel_sourcepath(self.projroot, sourcefile, sourcefile) text = 'source: %s' % (relpath,) if is_in_pkg: - href = self.linker.get_lazyhref(sourcefile) - + href = self.linker.get_lazyhref(sourcefile, + self.get_anchor(func)) csource = H.SourceSnippet(text, href, colored) cslinks = self.build_callsites(dotted_name) snippet = H.FunctionDescription(localname, argdesc, docstring, @@ -432,7 +465,8 @@ if sourcefile[-1] in ['o', 'c']: sourcefile = sourcefile[:-1] sourcelink = H.div(H.a('view source', - href=self.linker.get_lazyhref(sourcefile))) + href=self.linker.get_lazyhref(sourcefile, + self.get_anchor(cls)))) snippet = H.ClassDescription( # XXX bases HTML @@ -651,6 +685,10 @@ return lst name, _desc_type, is_degenerated = data if not is_degenerated: + try: + obj = self.dsa.get_obj(name) + except KeyError: + obj = None linktarget = self.linker.get_lazyhref(name) lst.append(H.a(str(_type), href=linktarget)) else: @@ -695,6 +733,7 @@ _reg_source = py.std.re.compile(r'([^>]*)<(.*)>') def gen_traceback(self, dotted_name, call_site): tbtag = H.CallStackDescription() + obj = self.dsa.get_obj(dotted_name) for frame in call_site: lineno = frame.lineno - frame.firstlineno source = frame.source @@ -772,3 +811,20 @@ self._revcache[dotted_name] = rev return rev + def get_anchor(self, obj): + # XXX may not always return the right results... + anchor = None + if hasattr(obj, 'im_func'): + # method + origin = find_method_origin(obj) + if origin: + anchor = '%s.%s' % (origin.__name__, + obj.im_func.func_name) + elif hasattr(obj, 'func_name'): + anchor = obj.func_name + elif hasattr(obj, '__name__'): + anchor = obj.__name__ + elif hasattr(obj, '__class__'): + anchor = obj.__class__.__name__ + return anchor + Modified: py/trunk/py/apigen/linker.py ============================================================================== --- py/trunk/py/apigen/linker.py (original) +++ py/trunk/py/apigen/linker.py Thu Mar 29 13:56:14 2007 @@ -63,8 +63,11 @@ def __init__(self): self._linkid2target = {} - def get_lazyhref(self, linkid): - return '%s://%s' % (TEMPLINK_PROTO, linkid) + def get_lazyhref(self, linkid, anchor=None): + href = '%s://%s' % (TEMPLINK_PROTO, linkid) + if anchor: + href += '#' + anchor + return href def set_link(self, linkid, target): assert linkid not in self._linkid2target @@ -72,13 +75,18 @@ def get_target(self, tempurl, fromlocation=None): assert tempurl.startswith('%s://' % (TEMPLINK_PROTO,)) - linkid = '://'.join(tempurl.split('://')[1:]) + anchor = None + if '#' in tempurl: + tempurl, anchor = tempurl.split('#', 1) + linkid = tempurl.split('://', 1)[1] linktarget = self._linkid2target[linkid] if fromlocation is not None: linktarget = relpath(fromlocation, linktarget) + if anchor is not None: + linktarget += '#' + anchor return linktarget - _reg_tempurl = py.std.re.compile('["\'](%s:\/\/[^"\s]*)["\']' % ( + _reg_tempurl = py.std.re.compile('(["\'])(%s:\/\/[^"\'\s#]*)(["\'#])' % ( TEMPLINK_PROTO,)) def replace_dirpath(self, dirpath, stoponerrors=True): """ replace temporary links in all html files in dirpath and below """ @@ -88,16 +96,19 @@ match = self._reg_tempurl.search(html) if not match: break - tempurl = match.group(1) + tempurl = match.group(2) + pre = match.group(1) + post = match.group(3) try: - html = html.replace('"' + tempurl + '"', - '"' + self.get_target(tempurl, - fpath.relto(dirpath)) + '"') + html = html.replace(match.group(0), pre + + self.get_target(tempurl, + fpath.relto(dirpath)) + post) except KeyError: if stoponerrors: raise - html = html.replace('"' + tempurl + '"', - '"apigen.notfound://%s"' % (tempurl,)) + html = html.replace(match.group(0), pre + + 'apigen.notfound://%s' % (tempurl,) + + post) fpath.write(html) Modified: py/trunk/py/apigen/source/html.py ============================================================================== --- py/trunk/py/apigen/source/html.py (original) +++ py/trunk/py/apigen/source/html.py Thu Mar 29 13:56:14 2007 @@ -7,6 +7,10 @@ from compiler import ast import time from py.__.apigen.source.color import Tokenizer, PythonSchema +from py.__.apigen.source.browser import parse_path + +class CompilationException(Exception): + """ raised when something goes wrong while importing a module """ class HtmlEnchanter(object): def __init__(self, mod): @@ -59,6 +63,30 @@ ret.append(item) return ret +def prepare_module(path, tokenizer, encoding): + path = py.path.local(path) + try: + mod = parse_path(path) + except: + # XXX don't try to catch SystemExit: it's actually raised by one + # of the modules in the py lib on import :( + exc, e, tb = py.std.sys.exc_info() + del tb + raise CompilationException('while compiling %s: %s - %s' % ( + path, e.__class__.__name__, e)) + lines = [unicode(l, encoding) for l in path.readlines()] + + enchanter = HtmlEnchanter(mod) + ret = [] + for i, line in enumerate(lines): + text = enchanter.enchant_row(i + 1, line) + if text == ['']: + text = [raw(' ')] + else: + text = prepare_line(text, tokenizer, encoding) + ret.append(text) + return ret + class HTMLDocument(object): def __init__(self, encoding, tokenizer=None): self.encoding = encoding Modified: py/trunk/py/apigen/testing/test_apigen_example.py ============================================================================== --- py/trunk/py/apigen/testing/test_apigen_example.py (original) +++ py/trunk/py/apigen/testing/test_apigen_example.py Thu Mar 29 13:56:14 2007 @@ -198,12 +198,10 @@ self.apb.build_class_pages(['main.SomeSubClass', 'main.SomeClass']) self.apb.build_namespace_pages() - # fake some stuff that would be built from other methods self.linker.replace_dirpath(self.base, False) clsfile = self.base.join('api/main.SomeClass.html') assert clsfile.check() html = clsfile.read() - print html run_string_sequence_test(html, [ 'href="../style.css"', 'href="../apigen_style.css"', @@ -237,7 +235,8 @@ self.spb.build_pages(self.fs_root) self.linker.replace_dirpath(self.base, False) funchtml = self.base.join('api/main.SomeClass.html').read() - assert funchtml.find('href="../source/pkg/someclass.py.html"') > -1 + print funchtml + assert funchtml.find('href="../source/pkg/someclass.py.html#SomeClass"') > -1 _checkhtml(funchtml) def test_build_namespace_pages(self): @@ -433,7 +432,8 @@ assert funcsource.check(file=True) html = funcsource.read() print html - assert ('def func(arg1)') in html + assert ('def ' + 'func(arg1)') in html def test_build_navigation_root(self): self.spb.build_pages(self.fs_root) Modified: py/trunk/py/apigen/testing/test_htmlgen.py ============================================================================== --- py/trunk/py/apigen/testing/test_htmlgen.py (original) +++ py/trunk/py/apigen/testing/test_htmlgen.py Thu Mar 29 13:56:14 2007 @@ -165,3 +165,30 @@ assert (htmlgen.get_rel_sourcepath(projpath, py.path.local('')) is None) +def test_find_method_origin(): + class Foo(object): + def foo(self): + pass + class Bar(Foo): + def bar(self): + pass + class Baz(Bar): + pass + assert htmlgen.find_method_origin(Baz.bar) is Bar + assert htmlgen.find_method_origin(Baz.foo) is Foo + assert htmlgen.find_method_origin(Bar.bar) is Bar + assert htmlgen.find_method_origin(Baz.__init__) is None + +def test_find_method_origin_old_style(): + class Foo: + def foo(self): + pass + class Bar(Foo): + def bar(self): + pass + class Baz(Bar): + pass + assert htmlgen.find_method_origin(Baz.bar) is Bar + assert htmlgen.find_method_origin(Baz.foo) is Foo + assert htmlgen.find_method_origin(Bar.bar) is Bar + Modified: py/trunk/py/apigen/testing/test_linker.py ============================================================================== --- py/trunk/py/apigen/testing/test_linker.py (original) +++ py/trunk/py/apigen/testing/test_linker.py Thu Mar 29 13:56:14 2007 @@ -47,6 +47,13 @@ l.replace_dirpath(temp) assert bar.read() == 'baz' + def test_with_anchor(self): + linker = TempLinker() + temphref = linker.get_lazyhref('py.path.local', 'LocalPath.join') + linker.set_link('py.path.local', 'py/path/local.html') + relpath = linker.get_target(temphref) + assert relpath == 'py/path/local.html#LocalPath.join' + def gen_check(frompath, topath, sep, expected): result = relpath(frompath, topath, sep=sep) assert result == expected From guido at codespeak.net Thu Mar 29 22:57:17 2007 From: guido at codespeak.net (guido at codespeak.net) Date: Thu, 29 Mar 2007 22:57:17 +0200 (CEST) Subject: [py-svn] r41655 - py/trunk/py/apigen Message-ID: <20070329205717.DE09B10090@code0.codespeak.net> Author: guido Date: Thu Mar 29 22:57:16 2007 New Revision: 41655 Modified: py/trunk/py/apigen/html.py py/trunk/py/apigen/htmlgen.py Log: Added list of possible exceptions for callables. Modified: py/trunk/py/apigen/html.py ============================================================================== --- py/trunk/py/apigen/html.py (original) +++ py/trunk/py/apigen/html.py Thu Mar 29 22:57:16 2007 @@ -56,8 +56,8 @@ pass class FunctionDescription(Description): - def __init__(self, localname, argdesc, docstring, valuedesc, csource, - callstack): + def __init__(self, localname, argdesc, docstring, valuedesc, excdesc, + csource, callstack): infoid = 'info_%s' % (localname.replace('.', '_dot_'),) docstringid = 'docstring_%s' % (localname.replace('.', '_dot_'),) fd = H.FunctionDef(localname, argdesc, @@ -68,7 +68,7 @@ infodiv = H.div( H.Docstring(docstring or '*no docstring available*', id=docstringid), - H.FunctionInfo(valuedesc, csource, callstack, + H.FunctionInfo(valuedesc, excdesc, csource, callstack, id=infoid, style="display: none"), class_='funcdocinfo') super(H.FunctionDescription, self).__init__(fd, infodiv) @@ -81,8 +81,9 @@ class_=class_, **kwargs) class FunctionInfo(html.div): - def __init__(self, valuedesc, csource, callstack, **kwargs): - super(H.FunctionInfo, self).__init__(valuedesc, H.br(), csource, + def __init__(self, valuedesc, excdesc, csource, callstack, **kwargs): + super(H.FunctionInfo, self).__init__(valuedesc, H.br(), excdesc, + H.br(), csource, callstack, class_='funcinfo', **kwargs) @@ -183,6 +184,13 @@ def __init__(self, *args, **kwargs): super(H.ValueDescList, self).__init__(*args, **kwargs) + class ExceptionDescList(html.ul): + def __init__(self, *args, **kwargs): + super(H.ExceptionDescList, self).__init__(*args, **kwargs) + + def append(self, t): + super(H.ExceptionDescList, self).append(html.li(t)) + class ValueDescItem(html.li): pass Modified: py/trunk/py/apigen/htmlgen.py ============================================================================== --- py/trunk/py/apigen/htmlgen.py (original) +++ py/trunk/py/apigen/htmlgen.py Thu Mar 29 22:57:16 2007 @@ -412,6 +412,7 @@ docstring = deindent(docstring) localname = func.__name__ argdesc = get_param_htmldesc(self.linker, func) + excdesc = self.build_exception_description(dotted_name) valuedesc = self.build_callable_signature_description(dotted_name) sourcefile = inspect.getsourcefile(func) @@ -423,11 +424,9 @@ colored = [] if sourcefile and callable_source: enc = source_html.get_module_encoding(sourcefile) - tokenizer = source_color.Tokenizer(source_color.PythonSchema) - firstlineno = func.func_code.co_firstlineno sep = get_linesep(callable_source) - org = callable_source.split(sep) - colored = [enumerate_and_color(org, firstlineno, enc)] + colored = [enumerate_and_color(callable_source.split(sep), + func.func_code.co_firstlineno, enc)] relpath = get_rel_sourcepath(self.projroot, sourcefile, sourcefile) text = 'source: %s' % (relpath,) if is_in_pkg: @@ -436,7 +435,7 @@ csource = H.SourceSnippet(text, href, colored) cslinks = self.build_callsites(dotted_name) snippet = H.FunctionDescription(localname, argdesc, docstring, - valuedesc, csource, cslinks) + valuedesc, excdesc, csource, cslinks) return snippet def build_class_view(self, dotted_name): @@ -697,6 +696,14 @@ lst.append(name) return lst + def build_exception_description(self, dotted_name): + excs = self.dsa.get_function_exceptions(dotted_name) + excdesc = H.ExceptionDescList() + for exc in excs: + excdesc.append(exc) + ret = H.div(H.div('possible exceptions:'), excdesc) + return ret + def is_in_pkg(self, sourcefile): return py.path.local(sourcefile).relto(self.projpath) From py-svn at codespeak.net Fri Mar 30 16:03:51 2007 From: py-svn at codespeak.net (Canadian Pharmacy) Date: Fri, 30 Mar 2007 16:03:51 +0200 (CEST) Subject: [py-svn] st.Doctor Jewell Message-ID: <20070330090346.4101.qmail@piglet3.sbor.net> An HTML attachment was scrubbed... URL: