[pypy-svn] r35679 - in pypy/dist/pypy/tool/build: . bin test
guido at codespeak.net
guido at codespeak.net
Wed Dec 13 15:50:00 CET 2006
Author: guido
Date: Wed Dec 13 15:49:47 2006
New Revision: 35679
Added:
pypy/dist/pypy/tool/build/build.py
pypy/dist/pypy/tool/build/test/repo.py
pypy/dist/pypy/tool/build/test/test_build.py
pypy/dist/pypy/tool/build/tooloption.py
Removed:
pypy/dist/pypy/tool/build/test/test_request_storage.py
Modified:
pypy/dist/pypy/tool/build/bin/client
pypy/dist/pypy/tool/build/bin/startcompile
pypy/dist/pypy/tool/build/client.py
pypy/dist/pypy/tool/build/config.py
pypy/dist/pypy/tool/build/server.py
pypy/dist/pypy/tool/build/test/fake.py
pypy/dist/pypy/tool/build/test/test_client.py
pypy/dist/pypy/tool/build/test/test_pypybuilder.py
pypy/dist/pypy/tool/build/test/test_server.py
Log:
Relatively large refactoring to implement support for choosing a revision (or
a range of revisions) to build. The client still doesn't actually update to
the revision, so it's not entirely functional yet, but it's all there.
I changed the way the data is passed around: instead of passing a tuple with
info from the startcompile script to the server, which passes it to the client,
a special object (BuildRequest) is passed around. This object contains info
about the email address, the revision and revision range, etc. When done
compiling, this object is serialized and stored on the BuildPath along with
the rest of the data.
The RequestStorage is removed, since the caching it did is no longer possible
(revision range spoils this).
Modified: pypy/dist/pypy/tool/build/bin/client
==============================================================================
--- pypy/dist/pypy/tool/build/bin/client (original)
+++ pypy/dist/pypy/tool/build/bin/client Wed Dec 13 15:49:47 2006
@@ -3,10 +3,13 @@
import path
import sys
import random
+import traceback
from pypy.tool.build import config as buildconfig
+from pypy.interpreter.error import OperationError
from py.execnet import SshGateway, PopenGateway
from pypy.tool.build.client import init, zip_result, OutputBuffer
+from pypy.tool.build import build
from pypy.config.config import to_optparse, Config
from pypy.config import pypyoption
@@ -28,18 +31,27 @@
try:
try:
while 1:
- data = channel.receive()
- if isinstance(data, str):
- continue
- if not isinstance(data, tuple): # needs more checks here
+ # receive compile requests
+ request = channel.receive()
+ if isinstance(request, str):
+ try:
+ request = build.BuildRequest.fromstring(request)
+ except (KeyError, SyntaxError), e:
+ print ('exception occurred when trying to interpret the '
+ 'following request:')
+ print request
+ print
+ print 'going to continue'
+ continue
+ else:
raise ValueError(
'received wrong unexpected data of type %s' % (
- type(data),)
+ type(request),)
)
- sysinfo, compileinfo = data
accepting = True
for checker in buildconfig.client_checkers:
- if not checker(sysinfo, compileinfo):
+ if not checker(request.sysinfo, request.compileinfo):
+ print 'request refused by checker', checker.func_name
accepting = False
break
channel.send(accepting)
@@ -47,25 +59,43 @@
print 'refusing compilation'
continue
# XXX we should compile here, using data dict for info
- print 'compilation requested for info %r\n\nbuilding...' % (
- data,)
+ print 'compilation requested for %s\nbuilding...\n' % (request,)
config = pypyoption.get_pypy_config()
- config.override(compileinfo)
+ config.override(request.compileinfo)
buffer = OutputBuffer(sys.__stderr__)
sys.stdout = buffer
sys.stderr = buffer
+ exception_occurred = False
try:
- driver = TranslationDriver.from_targetspec(
- targetpypystandalone.__dict__, config=config,
- default_goal='compile')
- driver.proceed(['compile'])
+ try:
+ driver = TranslationDriver.from_targetspec(
+ targetpypystandalone.__dict__, config=config,
+ default_goal='compile')
+ driver.proceed(['compile'])
+ except Exception, e:
+ if isinstance(e, OperationError):
+ print dir(e)
+ print repr(e.msg)
+ exception_occurred = True
+ exc, e, tb = sys.exc_info()
+ print '=' * 79
+ print 'Exception during compilation:'
+ print '%s: %s' % (exc, e)
+ print
+ print '\n'.join(traceback.format_tb(tb))
+ print '=' * 79
+ del tb
finally:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
- # send over zip data, end with a None
- zip_result(udir, channel)
+ if not exception_occurred:
+ # send over zip data, end with a None
+ zip_result(udir, channel)
+ else:
+ # just send the None
+ channel.send(None)
# send over logs
channel.send(buffer.getvalue())
@@ -76,3 +106,4 @@
finally:
channel.close()
gw.exit()
+
Modified: pypy/dist/pypy/tool/build/bin/startcompile
==============================================================================
--- pypy/dist/pypy/tool/build/bin/startcompile (original)
+++ pypy/dist/pypy/tool/build/bin/startcompile Wed Dec 13 15:49:47 2006
@@ -4,14 +4,17 @@
import sys
import random
from pypy.tool.build import config
+from pypy.tool.build import build
+from pypy.tool.build.tooloption import tool_config
-def parse_options(config):
+def parse_options(config, tool_config):
# merge system + compile options into one optionparser
from py.compat.optparse import OptionParser, OptionGroup
from pypy.config.config import to_optparse
optparser = to_optparse(config.system_config)
to_optparse(config.compile_config, parser=optparser)
+ to_optparse(tool_config, parser=optparser)
(options, args) = optparser.parse_args()
@@ -26,7 +29,8 @@
try:
from pypy.tool.build import ppbserver
- ret = ppbserver.compile(%r, (%r, %r))
+ from pypy.tool.build import build
+ ret = ppbserver.compile(%r)
channel.send(ret)
channel.close()
except:
@@ -37,22 +41,27 @@
channel.send(line[:-1])
del tb
"""
-def init(gw, sysinfo, compileinfo, path, email, port=12321):
+def init(gw, request, path, port=12321):
from pypy.tool.build import execnetconference
conference = execnetconference.conference(gw, port, False)
- channel = conference.remote_exec(initcode % (path, email, sysinfo,
- compileinfo))
+ channel = conference.remote_exec(initcode % (path, request))
return channel
if __name__ == '__main__':
from py.execnet import SshGateway, PopenGateway
from pypy.config.config import make_dict
- optparser, options, args = parse_options(config)
+ optparser, options, args = parse_options(config, tool_config)
sysinfo = make_dict(config.system_config)
compileinfo = make_dict(config.compile_config)
+
+ buildrequest = build.BuildRequest(args[0], sysinfo, compileinfo,
+ config.svnpath_to_url(
+ tool_config.svnpath),
+ tool_config.svnrev,
+ tool_config.revrange)
print 'going to start compile job with info:'
for k, v in sysinfo.items():
@@ -66,8 +75,8 @@
gw = PopenGateway()
else:
gw = SshGateway(config.server)
- channel = init(gw, sysinfo, compileinfo, config.path, args[0],
- port=config.port)
+
+ channel = init(gw, buildrequest, config.path, port=config.port)
data = channel.receive()
if type(data) == str:
print data
@@ -85,3 +94,4 @@
print 'you will be mailed once it\'s ready'
channel.close()
gw.exit()
+
Added: pypy/dist/pypy/tool/build/build.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/tool/build/build.py Wed Dec 13 15:49:47 2006
@@ -0,0 +1,136 @@
+import py
+from py.__.path.local.local import LocalPath
+
+def normalize_revision(svnurl, rev, highest_if_error=True):
+ """ return the actual revision of a certain repo state
+
+ if the string HEAD is provided, this looks up the trunk revision, if
+ the provided revision is an int (or string containing an int), the
+ revision id of the last revision before rev is returned, including (so
+ if rev itself had changes, rev is returned)
+ """
+ if rev == 'HEAD':
+ u = py.path.svnurl(svnurl)
+ else:
+ rev = int(rev)
+ u = py.path.svnurl(svnurl, rev=rev)
+ try:
+ return int(u.info().created_rev)
+ except py.error.Error:
+ if not highest_if_error:
+ raise
+ u = py.path.svnurl(svnurl)
+ return int(u.info().created_rev)
+
+class BuildPath(LocalPath):
+ def _request(self):
+ req = self.join('request')
+ if not req.check():
+ return None
+ return BuildRequest.fromstring(req.read())
+
+ def _set_request(self, request):
+ self.ensure('request', file=True).write(request.serialize())
+
+ request = property(_request, _set_request)
+
+ def _zipfile(self):
+ return self.ensure('data.zip', file=True)
+
+ def _set_zipfile(self, iterable):
+ # XXX not in use right now...
+ fp = self._zipfile().open('w')
+ try:
+ for chunk in iterable:
+ fp.write(chunk)
+ finally:
+ fp.close()
+
+ zipfile = property(_zipfile, _set_zipfile)
+
+ def _log(self):
+ log = self.join('log')
+ if not log.check():
+ return ''
+ return log.read()
+
+ def _set_log(self, data):
+ self.ensure('log', file=True).write(data)
+
+ log = property(_log, _set_log)
+
+ def _done(self):
+ return not not self.log
+ done = property(_done)
+
+class Build(object):
+ """ build data """
+ def __init__(self, buildrequest, buildpath):
+ self.request = buildrequest
+ self.path = buildpath
+
+class BuildRequest(object):
+ """ build request data
+
+ holds information about a build request, and some functionality to
+ serialize and unserialize itself
+ """
+ def __init__(self, email, sysinfo, compileinfo, svnurl, svnrev, revrange):
+ self.email = email
+ self.sysinfo = sysinfo
+ self.compileinfo = compileinfo
+ self.svnurl = svnurl
+ self.svnrev = svnrev
+ self.revrange = revrange
+
+ def __repr__(self):
+ return 'build.BuildRequest(%r, %r, %r, %r, %r, %r)' % (
+ self.email, self.sysinfo, self.compileinfo, self.svnurl,
+ self.svnrev, self.revrange)
+
+ def serialize(self):
+ data = {'normalized_rev': self.normalized_rev}
+ data.update(self.__dict__)
+ return """\
+email: %(email)s
+sysinfo: %(sysinfo)r
+compileinfo: %(compileinfo)r
+svnurl: %(svnurl)s
+svnrev: %(svnrev)s
+revrange: %(revrange)s
+normalized_rev: %(normalized_rev)s
+""" % data
+
+ def _fromstring(cls, s):
+ data = {}
+ for line in s.strip().split('\n'):
+ try:
+ key, value = line.split(':', 1)
+ except ValueError:
+ raise SyntaxError('line %r not in the right format' % (line,))
+ data[key.strip()] = value.strip()
+ ret = cls(data['email'], eval(data['sysinfo']),
+ eval(data['compileinfo']), data['svnurl'], data['svnrev'],
+ int(data['revrange']))
+ ret._nr = int(data['normalized_rev'])
+ return ret
+ fromstring = classmethod(_fromstring)
+
+ def has_satisfying_data(self, other):
+ """ return True if other request's data satisfies our needs """
+ return (self.sysinfo == other.sysinfo and
+ self.compileinfo == other.compileinfo and
+ self.svnurl == other.svnurl and
+ self.rev_in_range(other.normalized_rev))
+
+ _nr = None
+ def _normalized_rev(self):
+ if self._nr is None:
+ self._nr = normalize_revision(self.svnurl, self.svnrev)
+ return self._nr
+ normalized_rev = property(_normalized_rev)
+
+ def rev_in_range(self, comparerev):
+ return (self.normalized_rev >= comparerev - self.revrange and
+ self.normalized_rev <= comparerev + self.revrange)
+
Modified: pypy/dist/pypy/tool/build/client.py
==============================================================================
--- pypy/dist/pypy/tool/build/client.py (original)
+++ pypy/dist/pypy/tool/build/client.py Wed Dec 13 15:49:47 2006
@@ -20,22 +20,24 @@
self.channel.waitclose()
self.channel.close()
- def compile(self, info):
+ def compile(self, request):
"""send a compile job to the client side"""
- self.channel.send(info)
+ self.channel.send(request.serialize())
accepted = self.channel.receive()
if accepted:
- self.busy_on = info
- thread.start_new_thread(self.wait_until_done, (info,))
+ self.busy_on = request
+ thread.start_new_thread(self.wait_until_done, (request,))
else:
- self.refused.append(info)
+ self.refused.append(request)
return accepted
- def wait_until_done(self, info):
- buildpath = self.server.get_new_buildpath(info)
+ def wait_until_done(self, request):
+ buildpath = self.server.get_new_buildpath(request)
+ open('/tmp/tessie', 'w').write('foo')
if not self.testing:
fp = buildpath.zipfile.open('w')
+ gotdata = False
try:
while True:
# read data in chunks
@@ -47,13 +49,14 @@
# end of data is marked by sending a None
if chunk is None:
break
+ gotdata = True
fp.write(chunk)
finally:
fp.close()
# write the log (process stdout/stderr) to the buildpath
buildpath.log = self.channel.receive()
- self.server.compilation_done(info, buildpath)
+ self.server.compilation_done(buildpath)
self.busy_on = None
initcode = """
@@ -142,3 +145,4 @@
def isatty(self):
return False
+
Modified: pypy/dist/pypy/tool/build/config.py
==============================================================================
--- pypy/dist/pypy/tool/build/config.py (original)
+++ pypy/dist/pypy/tool/build/config.py Wed Dec 13 15:49:47 2006
@@ -37,3 +37,8 @@
# sysinfo, compileinfo), if one of them returns False the compilation will
# not be accepted
client_checkers = []
+
+# function to turn SVN paths into full URLs
+def svnpath_to_url(p):
+ return 'http://codespeak.net/svn/pypy/%s' % (p,)
+
Modified: pypy/dist/pypy/tool/build/server.py
==============================================================================
--- pypy/dist/pypy/tool/build/server.py (original)
+++ pypy/dist/pypy/tool/build/server.py Wed Dec 13 15:49:47 2006
@@ -3,6 +3,7 @@
import thread
import smtplib
import py
+from pypy.tool.build.build import BuildPath
def issubdict(d1, d2):
"""sees whether a dict is a 'subset' of another dict
@@ -24,129 +25,6 @@
return False
return True
-class RequestStorage(object):
- """simple registry that manages information"""
- def __init__(self, info_to_path=[]):
- self._id_to_info = {} # id -> info dict
- self._id_to_emails = {} # id -> requestor email address
- self._id_to_path = {} # id -> filepath
-
- self._last_id = 0
- self._id_lock = thread.allocate_lock()
-
- self._build_initial(info_to_path)
-
- def request(self, email, info):
- """place a request
-
- this either returns a path to the binary (if it's available
- already) or an id for the info
- """
- infoid = self.get_info_id(info)
- path = self._id_to_path.get(infoid)
- if path is not None:
- return path
- self._id_to_emails.setdefault(infoid, []).append(email)
-
- def get_info_id(self, info):
- """retrieve or create an id for an info dict"""
- self._id_lock.acquire()
- try:
- for k, v in self._id_to_info.iteritems():
- if v == info:
- return k
- self._last_id += 1
- id = self._last_id
- self._id_to_info[id] = info
- return id
- finally:
- self._id_lock.release()
-
- def add_build(self, info, path):
- """store the data for a build and make it available
-
- returns a list of email addresses for the people that should be
- warned
- """
- infoid = self.get_info_id(info)
- emails = self._id_to_emails.pop(infoid)
- self._id_to_path[infoid] = path
- return emails
-
- def _build_initial(self, info_to_path):
- """fill the dicts with info about files that are already built"""
- for info, path in info_to_path:
- id = self.get_info_id(info)
- self._id_to_path[id] = path
-
-from py.__.path.local.local import LocalPath
-class BuildPath(LocalPath):
- def _info(self):
- info = getattr(self, '_info_value', [])
- if info:
- return info
- for name in ['system', 'compile']:
- currinfo = {}
- infopath = self.join('%s_info.txt' % (name,))
- if not infopath.check():
- return ({}, {})
- for line in infopath.readlines():
- line = line.strip()
- if not line:
- continue
- chunks = line.split(':')
- key = chunks.pop(0)
- value = ':'.join(chunks)
- currinfo[key] = eval(value)
- info.append(currinfo)
- info = tuple(info)
- self._info_value = info
- return info
-
- def _set_info(self, info):
- self._info_value = info
- assert len(info) == 2, 'not a proper info tuple'
- for i, name in enumerate(['system', 'compile']):
- infopath = self.join('%s_info.txt' % (name,))
- infopath.ensure()
- fp = infopath.open('w')
- try:
- for key, value in info[i].iteritems():
- fp.write('%s: %r\n' % (key, value))
- finally:
- fp.close()
-
- info = property(_info, _set_info)
-
- def _zipfile(self):
- return py.path.local(self / 'data.zip')
-
- def _set_zipfile(self, iterable):
- # XXX not in use right now...
- fp = self._zipfile().open('w')
- try:
- for chunk in iterable:
- fp.write(chunk)
- finally:
- fp.close()
-
- zipfile = property(_zipfile, _set_zipfile)
-
- def _log(self):
- log = self.join('log')
- if not log.check():
- return ''
- return log.read()
-
- def _set_log(self, data):
- self.join('log').write(data)
-
- log = property(_log, _set_log)
-
- def _done(self):
- return not not self.log
- done = property(_done)
-
class PPBServer(object):
retry_interval = 10
@@ -161,15 +39,18 @@
self._buildpath = py.path.local(builddir)
self._clients = []
- info_to_path = []
+
+ done = []
for bp in self._get_buildpaths(builddir):
if bp.done:
- info_to_path.append((bp.info, str(bp)))
+ done.append(bp)
else:
# throw away half-done builds...
bp.remove()
- self._requeststorage = RequestStorage(info_to_path)
+
+ self._done = done
self._queued = []
+ self._waiting = []
self._queuelock = thread.allocate_lock()
self._namelock = thread.allocate_lock()
@@ -180,7 +61,7 @@
client, client.sysinfo))
client.channel.send('welcome')
- def compile(self, requester_email, info):
+ def compile(self, request):
"""start a compilation
requester_email is an email address of the person requesting the
@@ -197,47 +78,53 @@
in any case, if the first item of the tuple returned is False,
an email will be sent once the build is available
"""
- path = self._requeststorage.request(requester_email, info)
- if path is not None:
- pathstr = str(path)
- self._channel.send('already a build for this info available')
- return (True, pathstr)
+ # store the request, if there's already a build available the
+ # storage will return that path
+ for bp in self._done:
+ if request.has_satisfying_data(bp.request):
+ path = str(bp)
+ self._channel.send('already a build for this info available')
+ return (True, path)
for client in self._clients:
- if client.busy_on == info:
- self._channel.send('build for %r currently in progress' %
- (info,))
+ if client.busy_on and request.has_satisfying_data(client.busy_on):
+ self._channel.send('build for %s currently in progress' %
+ (request,))
+ self._waiting.append(request)
return (False, 'this build is already in progress')
# we don't have a build for this yet, find a client to compile it
- if self.run(info):
+ if self.run(request):
return (False, 'found a suitable client, going to build')
self._queuelock.acquire()
try:
- self._queued.append(info)
+ self._queued.append(request)
finally:
self._queuelock.release()
return (False, 'no suitable client found; your request is queued')
- def run(self, info):
+ def run(self, request):
"""find a suitable client and run the job if possible"""
clients = self._clients[:]
# XXX shuffle should be replaced by something smarter obviously ;)
random.shuffle(clients)
for client in clients:
- if (client.busy_on or not issubdict(info[0], client.sysinfo) or
- info in client.refused):
+ # if client is busy, or sysinfos don't match, refuse rightaway,
+ # else ask client to build it
+ if (client.busy_on or
+ not issubdict(request.sysinfo, client.sysinfo) or
+ request in client.refused):
continue
else:
self._channel.send(
- 'going to send compile job with info %r to %s' % (
- info, client
+ 'going to send compile job for request %s to %s' % (
+ request, client
)
)
- accepted = client.compile(info)
+ accepted = client.compile(request)
if accepted:
return True
self._channel.send(
- 'no suitable client available for compilation with info %r' % (
- info,
+ 'no suitable client available for compilation of %s' % (
+ request,
)
)
@@ -247,20 +134,32 @@
while 1:
time.sleep(self.retry_interval)
self._cleanup_clients()
+ self._test_waiting()
self._try_queued()
- def get_new_buildpath(self, info):
+ def get_new_buildpath(self, request):
path = BuildPath(str(self._buildpath / self._create_filename()))
- path.info = info
+ path.request = request
return path
- def compilation_done(self, info, path):
+ def compilation_done(self, buildpath):
"""client is done with compiling and sends data"""
- self._channel.send('compilation done for %r, written to %s' % (
- info, path))
- emails = self._requeststorage.add_build(info, path)
- for emailaddr in emails:
- self._send_email(emailaddr, info, path)
+ self._queuelock.acquire()
+ try:
+ self._channel.send('compilation done for %s, written to %s' % (
+ buildpath.request, buildpath))
+ emails = [buildpath.request.email]
+ self._done.append(buildpath)
+ waiting = self._waiting[:]
+ for req in waiting:
+ if req.has_satisfying_data(buildpath.request):
+ self._waiting.remove(req)
+ emails.append(req.email)
+ for emailaddr in emails:
+ print 'sending mail to %s' % (emailaddr,)
+ self._send_email(emailaddr, buildpath)
+ finally:
+ self._queuelock.release()
def _cleanup_clients(self):
self._queuelock.acquire()
@@ -274,15 +173,34 @@
finally:
self._queuelock.release()
+ def _test_waiting(self):
+ """ for each waiting request, see if the compilation is still alive
+
+ if the compilation is dead, the request is moved to self._queued
+ """
+ self._queuelock.acquire()
+ try:
+ waiting = self._waiting[:]
+ for request in waiting:
+ for client in self._clients:
+ if request.has_satisfying_data(client.busy_on):
+ break
+ else:
+ self._waiting.remove(request)
+ self._queued.append(request)
+ continue
+ finally:
+ self._queuelock.release()
+
def _try_queued(self):
self._queuelock.acquire()
try:
toremove = []
- for info in self._queued:
- if self.run(info):
- toremove.append(info)
- for info in toremove:
- self._queued.remove(info)
+ for request in self._queued:
+ if self.run(request):
+ toremove.append(request)
+ for request in toremove:
+ self._queued.remove(request)
finally:
self._queuelock.release()
@@ -305,7 +223,7 @@
finally:
self._namelock.release()
- def _send_email(self, addr, info, path):
+ def _send_email(self, addr, buildpath):
self._channel.send('going to send email to %s' % (addr,))
if self._mailhost is not None:
msg = '\r\n'.join([
@@ -314,7 +232,9 @@
'Subject: %s compilation done' % (self._projname,),
'',
'The compilation you requested is done. You can find it at',
- str(path),
+ str(build.path),
+ '',
+ buildpath.log,
])
server = smtplib.SMTP(self._mailhost, self._mailport)
server.set_debuglevel(0)
Modified: pypy/dist/pypy/tool/build/test/fake.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/fake.py (original)
+++ pypy/dist/pypy/tool/build/test/fake.py Wed Dec 13 15:49:47 2006
@@ -1,4 +1,4 @@
-from pypy.tool.build.server import BuildPath
+from pypy.tool.build.build import BuildPath
class FakeChannel(object):
def __init__(self):
@@ -23,11 +23,10 @@
self.busy_on = None
self.refused = []
- def compile(self, info):
- for k, v in info[0].items():
- self.channel.send('%s: %r' % (k, v))
+ def compile(self, request):
+ self.channel.send(request.serialize())
self.channel.send(None)
- self.busy_on = info
+ self.busy_on = request
return True
class FakeServer(object):
@@ -41,15 +40,15 @@
def register(self, client):
self._clients.append(client)
- def compilation_done(self, info, data):
- self._done.append((info, data))
+ def compilation_done(self, ret):
+ self._done.append(ret)
i = 0
- def get_new_buildpath(self, info):
+ def get_new_buildpath(self, request):
name = 'build-%s' % (self.i,)
self.i += 1
bp = BuildPath(str(self._builddirpath / name))
- bp.info = info
+ bp.request = request
bp.ensure(dir=1)
return bp
Added: pypy/dist/pypy/tool/build/test/repo.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/tool/build/test/repo.py Wed Dec 13 15:49:47 2006
@@ -0,0 +1,10 @@
+import py
+
+def create_temp_repo(reponame):
+ t = py.test.ensuretemp('build-svnrepo')
+ repo = t.join(reponame)
+ ret = py.std.os.system('svnadmin create %r' % (str(repo),))
+ if ret:
+ py.test.skip('could not create temporary svn repository')
+ return py.path.svnurl('file://%s' % (repo,))
+
Added: pypy/dist/pypy/tool/build/test/test_build.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/tool/build/test/test_build.py Wed Dec 13 15:49:47 2006
@@ -0,0 +1,153 @@
+import py
+from pypy.tool.build import build
+from py.__.misc.cache import AgingCache
+from py.__.path.svn import urlcommand
+from repo import create_temp_repo
+
+def setup_module(mod):
+ # remove nasty cache from py.path.svnurl to allow this to work...
+ mod._oldcache = urlcommand.SvnCommandPath._lsnorevcache
+ urlcommand.SvnCommandPath._lsnorevcache = AgingCache(maxentries=1,
+ maxseconds=0)
+
+def teardown_module(mod):
+ urlcommand.SvnCommandPath._lsnorevcache = mod._oldcache
+
+def test_normalize_revision():
+ if py.std.sys.platform.startswith('win'):
+ py.test.skip('win32 escaping problems with file:// urls')
+ repo = create_temp_repo('normalize')
+ repo.ensure('foo', dir=True)
+ foourl = str(repo.join('foo'))
+ wc = py.path.svnwc(py.test.ensuretemp('build-svnwc').join('wc-foo'))
+ wc.checkout(foourl)
+ ret = build.normalize_revision(foourl, 'HEAD')
+ assert ret == 1
+
+ f1 = wc.ensure('file1', file=True, versioned=True)
+ f1.write('foo')
+ print wc.status().added
+ wc.commit('test something')
+ wc.update()
+ assert int(wc.status().rev) == 2
+ ret = build.normalize_revision(foourl, 'HEAD')
+ assert ret == 2
+
+ f2 = wc.ensure('file2', file=True, versioned=True)
+ f2.write('foo')
+ wc.commit('test something')
+ wc.update()
+ ret = build.normalize_revision(foourl, 'HEAD')
+ assert ret == 3
+
+ ret = build.normalize_revision(foourl, 1234)
+ assert ret == 3
+
+def test_buildpath():
+ tempdir = py.test.ensuretemp('pypybuilder-buildpath')
+ bp = build.BuildPath(str(tempdir / 'test'))
+ assert not bp.check()
+ bp.log = 'foo'
+ assert bp.check()
+
+def test_buildpath_request():
+ tempdir = py.test.ensuretemp('pypybuilder-buildpath')
+ temprepo = create_temp_repo('request')
+ repodir = temprepo.mkdir('foo')
+ print str(tempdir)
+ bp = build.BuildPath(str(tempdir / 'test_request'))
+ assert bp.request is None
+ br = build.BuildRequest('foo at bar.com', {'foo': 1}, {'bar': 1},
+ str(repodir), 'HEAD', 0)
+ bp.request = br
+ assert bp.join('request').check()
+ assert bp.request.serialize() == br.serialize()
+
+def test_buildpath_zip():
+ tempdir = py.test.ensuretemp('pypybuilder-buildpath')
+ bp = build.BuildPath(str(tempdir / 'test_zip'))
+ assert isinstance(bp.zipfile, py.path.local)
+ bp.zipfile = ['foo', 'bar', 'baz']
+ assert bp.zipfile.read() == 'foobarbaz'
+
+def test_buildpath_log_and_done():
+ tempdir = py.test.ensuretemp('pypybuilder-buildpath')
+ bp = build.BuildPath(str(tempdir / 'test_log'))
+ log = bp.log
+ assert not log
+ assert not bp.done
+ bp.log = 'log data'
+ assert bp.log == 'log data'
+ assert bp.done
+
+def test_buildrequest_serialize():
+ br = build.BuildRequest('foo at bar.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ 'file:///foo/bar', 'HEAD', 0)
+ br._nr = 1
+ ser = br.serialize()
+ assert ser == """\
+email: foo at bar.com
+sysinfo: {'foo': 'bar'}
+compileinfo: {'spam': 'eggs'}
+svnurl: file:///foo/bar
+svnrev: HEAD
+revrange: 0
+normalized_rev: 1
+"""
+ assert build.BuildRequest.fromstring(ser).serialize() == ser
+
+ py.test.raises(SyntaxError, 'build.BuildRequest.fromstring("foo")')
+ py.test.raises(KeyError, 'build.BuildRequest.fromstring("foo: bar")')
+
+def test_buildrequest_has_satisfying_data():
+ if py.std.sys.platform.startswith('win'):
+ py.test.skip('win32 escaping problems with file:// urls')
+
+ # note that this implicitly tests rev_in_range() too...
+
+ repo = create_temp_repo('satisfying')
+ testproj = repo.ensure('testproj', dir=True)
+
+ testurl = str(testproj)
+
+ wc = py.path.svnwc(py.test.ensuretemp('satisfying-svnwc'))
+ wc.checkout(testurl)
+
+ br1 = build.BuildRequest('foo at bar.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ testurl, 'HEAD', 0)
+
+ br2 = build.BuildRequest('foo at baz.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ testurl, 'HEAD', 0)
+ assert br2.has_satisfying_data(br1)
+
+ br3 = build.BuildRequest('foo at baz.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ testurl, 1, 0)
+ assert br3.has_satisfying_data(br1)
+
+ # this actually succeeds: because there's no revision 2 yet,
+ # normalize_revision will return the highest rev (1), which matches
+ br4 = build.BuildRequest('foo at baz.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ testurl, 2, 0)
+ assert br4.has_satisfying_data(br1)
+
+ foo = wc.ensure('foo', file=True)
+ foo.add()
+ wc.commit('commit message')
+
+ # now it should fail...
+ br5 = build.BuildRequest('foo at baz.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ testurl, 2, 0)
+ assert not br5.has_satisfying_data(br1)
+
+ br6 = build.BuildRequest('foo at baz.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ testurl, 2, 1)
+ assert br6.has_satisfying_data(br1)
+
+ br7 = build.BuildRequest('foo at baz.com', {'foo': 'baz'}, {'spam': 'eggs'},
+ testurl, 1, 0)
+ assert not br7.has_satisfying_data(br1)
+
+ br8 = build.BuildRequest('foo at baz.com', {'foo': 'bar'}, {'spam': 'eggs'},
+ testurl + '/baz', 1, 0)
+ assert not br8.has_satisfying_data(br1)
+
Modified: pypy/dist/pypy/tool/build/test/test_client.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_client.py (original)
+++ pypy/dist/pypy/tool/build/test/test_client.py Wed Dec 13 15:49:47 2006
@@ -1,5 +1,6 @@
import path
from pypy.tool.build import client
+from pypy.tool.build import build
import py
import time
import sys
@@ -9,7 +10,10 @@
def __init__(self, *args, **kwargs):
super(ClientForTests, self).__init__(*args, **kwargs)
self._done = []
-
+
+class BuildRequestForTests(build.BuildRequest):
+ normalized_rev = 1
+
def setup_module(mod):
mod.temp = temp = py.test.ensuretemp('pypybuilder-client')
mod.svr = svr = FakeServer(temp)
@@ -26,27 +30,28 @@
svr.register(c2)
def test_compile():
- info = ({'foo': 1}, {'bar': 2})
+ nfo = ({'foo': 1}, {'bar': 2})
+ br = BuildRequestForTests('foo at bar.com', {'foo': 1}, {'bar': 1},
+ 'file:///foo', 'HEAD', 0)
c1c.send(True) # notifying we 'accept' the compile
- accepted = c1.compile(info)
+ accepted = c1.compile(br)
assert accepted
ret = c1.channel.receive()
- assert ret == info # this was still in the buffer
- assert c1.busy_on == info
+ assert ret == br.serialize() # this was still in the buffer
+ assert c1.busy_on.serialize() == br.serialize()
c1.channel.send('foo bar')
c1.channel.send(None)
c1.channel.send('log')
# meanwhile the client starts a thread that waits until there's data
# available on its own channel, with our FakeChannel it has data rightaway,
- # though (the channel out and in are the same, and we just sent 'info'
+ # though (the channel out and in are the same, and we just sent 'ret'
# over the out one)
time.sleep(1)
done = svr._done.pop()
- assert done[0] == info
- assert done[1] == (temp / 'build-0')
+ assert str(done) == str(temp / 'build-0')
assert temp.join('build-0/log').read() == 'log'
def test_channelwrapper():
@@ -66,11 +71,13 @@
assert c.buffer == ['foo', 'bar', 'baz', None]
def test_failed_checker():
- info = ({'foo': 1}, {'bar': 2})
+ br = build.BuildRequest('foo at bar.com', {'foo': 1}, {'bar': 2},
+ 'file:///foo', 'HEAD', 0)
+ br._nr = 1
c1c.send(False) # notifying we _don't_ 'accept' the compile
- accepted = c1.compile(info)
+ accepted = c1.compile(br)
assert not accepted
- assert info in c1.refused
+ assert br in c1.refused
assert c1.busy_on == None
def test_output_buffer():
Modified: pypy/dist/pypy/tool/build/test/test_pypybuilder.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_pypybuilder.py (original)
+++ pypy/dist/pypy/tool/build/test/test_pypybuilder.py Wed Dec 13 15:49:47 2006
@@ -1,8 +1,10 @@
import path
from pypy.tool.build import client, server, execnetconference
from pypy.tool.build import config
+from pypy.tool.build import build
from pypy.config import config as pypyconfig
import py
+from repo import create_temp_repo
# XXX NOTE: if you encounter failing tests on a slow system, you may want to
# increase the sleep interval a bit to see if that helps...
@@ -20,11 +22,15 @@
def test_functional_1():
if not py.test.pypybuilder_option.functional:
py.test.skip('skipping functional test, use --functional to run it')
-
+
# XXX this one is a bit messy, it's a quick functional test for the whole
# system, but for instance contains time.sleep()s to make sure all threads
# get the time to perform tasks and such...
+ repo = create_temp_repo('functional')
+ repo.mkdir('foo')
+ foourl = str(repo.join('foo'))
+
# first initialize a server
sgw = py.execnet.PopenGateway()
temppath = py.test.ensuretemp('pypybuilder-functional')
@@ -55,7 +61,8 @@
sys.path += %r
from pypy.tool.build import ppbserver
- channel.send(ppbserver.compile(%r, (%r, {})))
+ from pypy.tool.build import build
+ channel.send(ppbserver.compile(%r))
channel.close()
"""
compgw = py.execnet.PopenGateway()
@@ -65,8 +72,9 @@
# freezes (from the app waiting for input)
# this one should fail because there's no client found for foo = 3
- compc = compconf.remote_exec(code % (config.testpath, 'foo1 at bar.com',
- {'foo': 3}))
+ br = build.BuildRequest('foo1 at bar.com', {'foo': 3}, {}, foourl,
+ 1, 0)
+ compc = compconf.remote_exec(code % (config.testpath, br))
# sorry...
py.std.time.sleep(SLEEP_INTERVAL)
@@ -76,8 +84,9 @@
assert ret[1].find('no suitable client found') > -1
# this one should be handled by client 1
- compc = compconf.remote_exec(code % (config.testpath, 'foo2 at bar.com',
- {'foo': 1}))
+ br = build.BuildRequest('foo2 at bar.com', {'foo': 1}, {}, foourl,
+ 1, 0)
+ compc = compconf.remote_exec(code % (config.testpath, br))
# client 1 will now send a True to the server to tell it wants to compile
cc1.send(True)
@@ -95,7 +104,8 @@
# client 1 should by now have received the info to build for
ret = cc1.receive()
- assert ret == ({'foo': 1}, {})
+ request = build.BuildRequest.fromstring(ret)
+ assert request.sysinfo == {'foo': 1}
# this should have created a package in the temp dir
assert len(temppath.listdir()) == 1
@@ -122,7 +132,7 @@
ppbserver._try_queued()
# give the server some time, the clients 'compile' in threads
time.sleep(%s)
- channel.send(ppbserver._requeststorage._id_to_emails)
+ channel.send(ppbserver._waiting)
channel.close()
"""
compgw2 = py.execnet.PopenGateway()
@@ -134,7 +144,7 @@
# we check whether all emails are now sent, since after adding the third
# client, and calling _try_queued(), both jobs should have been processed
ret = compc2.receive()
- assert ret.values() == []
+ assert ret == []
# this should also have created another package in the temp dir
assert len(temppath.listdir()) == 2
@@ -152,3 +162,4 @@
compgw.exit()
compgw2.exit()
sgw.exit()
+
Modified: pypy/dist/pypy/tool/build/test/test_server.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_server.py (original)
+++ pypy/dist/pypy/tool/build/test/test_server.py Wed Dec 13 15:49:47 2006
@@ -2,9 +2,9 @@
from pypy.tool.build import server
import py
from fake import FakeChannel, FakeClient
-from pypy.tool.build.server import RequestStorage
-from pypy.tool.build.server import BuildPath
+from pypy.tool.build import build
import time
+from repo import create_temp_repo
def setup_module(mod):
mod.temppath = temppath = py.test.ensuretemp('pypybuilder-server')
@@ -45,44 +45,55 @@
py.test.raises(IndexError, 'svr._channel.receive()')
def test_compile():
- # XXX this relies on the output not changing... quite scary
- info = {'foo': 1}
- ret = svr.compile('test at domain.com', (info, None))
+ repo = create_temp_repo('compile')
+ repodir = repo.mkdir('foo')
+
+ br = build.BuildRequest('foo at bar.com', {'foo': 1}, {},
+ str(repodir), 'HEAD', 0)
+ ret = svr.compile(br)
assert not ret[0]
assert ret[1].find('found a suitable client') > -1
ret = svr._channel.receive()
assert ret.find('going to send compile job') > -1
ret = c1.channel.receive()
- assert ret == 'foo: 1'
- ret = c1.channel.receive()
- assert ret is None
+ assert ret == br.serialize()
+ none = c1.channel.receive()
+ assert none is None
py.test.raises(IndexError, "c2.channel.receive()")
- svr.compile('test at domain.com', ({'foo': 3}, None))
+ br2 = build.BuildRequest('foo at baz.com', {'foo': 3}, {},
+ str(repodir), 'HEAD', 0)
+ svr.compile(br2)
ret = svr._channel.receive()
assert ret.find('no suitable client available') > -1
- info = {'bar': [3]}
- svr.compile('test at domain.com', (info, None))
+ br3 = build.BuildRequest('foo at qux.com', {'bar': [3]}, {},
+ str(repodir), 'HEAD', 0)
+ svr.compile(br3)
ret = svr._channel.receive()
assert ret.find('going to send') > -1
- assert c2.channel.receive() == 'bar: [3]'
+ assert c2.channel.receive() == br3.serialize()
assert c2.channel.receive() is None
py.test.raises(IndexError, "c1.channel.receive()")
- info = {'foo': 1}
- ret = svr.compile('test at domain.com', (info, None))
+ br4 = build.BuildRequest('foo at spam.com', {'foo': 1}, {},
+ str(repodir), 'HEAD', 0)
+ ret = svr.compile(br4)
assert not ret[0]
assert ret[1].find('this build is already') > -1
assert svr._channel.receive().find('currently in progress') > -1
c1.busy_on = None
- bp = BuildPath(str(temppath / 'foo'))
- svr.compilation_done((info, None), bp)
- ret = svr.compile('test at domain.com', (info, None))
+ bp = build.BuildPath(str(temppath / 'foo'))
+ print br
+ bp.request = br
+ svr.compilation_done(bp)
+ clone = build.BuildRequest.fromstring(bp.request.serialize())
+ clone.email = 'test at domain.com'
+ ret = svr.compile(clone)
assert ret[0]
assert isinstance(ret[1], str)
- assert BuildPath(ret[1]) == bp
+ assert build.BuildPath(ret[1]) == bp
ret = svr._channel.receive()
assert ret.find('compilation done for') > -1
for i in range(2):
@@ -90,22 +101,6 @@
assert ret.find('going to send email to') > -1
ret = svr._channel.receive()
assert ret.find('already a build for this info') > -1
-
-def test_buildpath():
- tempdir = py.test.ensuretemp('pypybuilder-buildpath')
- # grmbl... local.__new__ checks for class equality :(
- bp = BuildPath(str(tempdir / 'test1'))
- assert not bp.check()
- assert bp.info == ({}, {})
-
- bp.info = ({'foo': 1, 'bar': [1,2]}, {'baz': 1})
- assert bp.info == ({'foo': 1, 'bar': [1,2]}, {'baz': 1})
- assert (sorted((bp / 'system_info.txt').readlines()) ==
- ['bar: [1, 2]\n', 'foo: 1\n'])
-
- assert isinstance(bp.zipfile, py.path.local)
- bp.zipfile = ['foo', 'bar', 'baz']
- assert bp.zipfile.read() == 'foobarbaz'
def test__create_filename():
svr._i = 0 # reset counter
@@ -113,37 +108,42 @@
name1 = svr._create_filename()
assert name1 == 'pypytest-%s-0' % (today,)
assert svr._create_filename() == ('pypytest-%s-1' % (today,))
- bp = BuildPath(str(temppath / ('pypytest-%s-2' % (today,))))
+ bp = build.BuildPath(str(temppath / ('pypytest-%s-2' % (today,))))
try:
bp.ensure()
assert svr._create_filename() == 'pypytest-%s-3'% (today,)
finally:
bp.remove()
-
+
def test_get_new_buildpath():
+ repo = create_temp_repo('get_new_buildpath')
+ repodir = repo.mkdir('foo')
+
svr._i = 0
today = time.strftime('%Y%m%d')
+ br = build.BuildRequest('foo at bar.com', {'foo': 'bar'}, {'baz': 'qux'},
+ str(repodir), 'HEAD', 0)
- path1 = svr.get_new_buildpath(({'foo': 'bar'}, {'baz': 'qux'}))
+ bp1 = svr.get_new_buildpath(br)
+ bp1.log = ['foo']
try:
- assert isinstance(path1, BuildPath)
- assert path1.info == ({'foo': 'bar'}, {'baz': 'qux'})
- assert path1.basename == 'pypytest-%s-0' % (today,)
+ assert isinstance(bp1, build.BuildPath)
+ assert bp1.basename == 'pypytest-%s-0' % (today,)
try:
- path2 = svr.get_new_buildpath(({'foo': 'baz'}, {'bar': 'qux'}))
- assert path2.info == ({'foo': 'baz'}, {'bar': 'qux'})
- assert path2.basename == 'pypytest-%s-1' % (today,)
+ bp2 = svr.get_new_buildpath(br)
+ bp2.log = ['bar']
+ assert bp2.basename == 'pypytest-%s-1' % (today,)
finally:
- path2.remove()
+ bp2.remove()
finally:
- path1.remove()
+ bp1.remove()
def test_cleanup_old_builds():
temppath = py.test.ensuretemp('cleanup_old_builds')
- bp1 = server.BuildPath(temppath.join('bp1'))
+ bp1 = build.BuildPath(temppath.join('bp1'))
bp1.ensure(dir=True)
- bp2 = server.BuildPath(temppath.join('bp2'))
+ bp2 = build.BuildPath(temppath.join('bp2'))
bp2.ensure(dir=True)
bp2.log = 'log'
svr = server.PPBServer('test', FakeChannel(), str(temppath))
Added: pypy/dist/pypy/tool/build/tooloption.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/tool/build/tooloption.py Wed Dec 13 15:49:47 2006
@@ -0,0 +1,14 @@
+import py
+from pypy.config.config import StrOption, IntOption
+from pypy.config.config import OptionDescription, Config
+
+import sys
+tool_optiondescription = OptionDescription('tool', '', [
+ IntOption('svnrev', 'Subversion revision', default='HEAD'),
+ StrOption('svnpath', 'Subversion path (relative to the project root)',
+ default='dist'),
+ StrOption('revrange', 'Revision range (max difference in revision between '
+ 'requested build and result)', default=0),
+])
+
+tool_config = Config(tool_optiondescription)
More information about the Pypy-commit
mailing list