[pypy-svn] r40773 - in pypy/dist/pypy: bin tool/build tool/build/bin tool/build/test tool/build/testproject tool/build/web/templates tool/build/web/test
guido at codespeak.net
guido at codespeak.net
Mon Mar 19 15:12:00 CET 2007
Author: guido
Date: Mon Mar 19 15:11:56 2007
New Revision: 40773
Added:
pypy/dist/pypy/tool/build/compile.py
pypy/dist/pypy/tool/build/test/test_compile.py
Modified:
pypy/dist/pypy/bin/startcompile.py
pypy/dist/pypy/tool/build/bin/metaserver
pypy/dist/pypy/tool/build/config.py
pypy/dist/pypy/tool/build/metaserver.py
pypy/dist/pypy/tool/build/systemoption.py
pypy/dist/pypy/tool/build/test/test_pypybuilder.py
pypy/dist/pypy/tool/build/testproject/compileoption.py
pypy/dist/pypy/tool/build/web/templates/buildersinfo.html
pypy/dist/pypy/tool/build/web/test/test_app.py
Log:
Added --foreground switch to startcompile.py. In order to support this, and to
be able to test it a little bit, I did some refactorings (like moving most of
the startcompile to a seperate file and splitting it up a bit, moving some
vars to config.py). Also fixed the busy_on links in buildersinfo.html, fixed
a test in test_app.py, and did several small cleanups to more easily allow
re-use of the project (issues that popped up while adding a test project) and
fixed some issues in the metaserver queue handling.
Modified: pypy/dist/pypy/bin/startcompile.py
==============================================================================
--- pypy/dist/pypy/bin/startcompile.py (original)
+++ pypy/dist/pypy/bin/startcompile.py Mon Mar 19 15:11:56 2007
@@ -1,95 +1,9 @@
#!/usr/bin/env python
import autopath
-from pypy.tool.build.bin import path
-import sys
-import random
from pypy.tool.build import config
-from pypy.tool.build import build
-from pypy.tool.build.tooloption import tool_config
-
-def parse_options(config, tool_config):
- # merge system + compile options into one optionparser
- from py.compat.optparse import OptionParser, OptionGroup
- from pypy.config.config import to_optparse
-
- optparser = to_optparse(config.system_config)
- to_optparse(config.compile_config, parser=optparser)
- to_optparse(tool_config, parser=optparser)
-
- (options, args) = optparser.parse_args()
-
- if not args or len(args) != 1:
- optparser.error('please provide an email address')
-
- return optparser, options, args
-
-initcode = """
- import sys
- sys.path += %r
-
- try:
- from pypy.tool.build import metaserver_instance
- from pypy.tool.build import build
- ret = metaserver_instance.compile(%r)
- channel.send(ret)
- channel.close()
- except:
- import sys, traceback
- exc, e, tb = sys.exc_info()
- channel.send(str(exc) + ' - ' + str(e))
- for line in traceback.format_tb(tb):
- channel.send(line[:-1])
- del tb
-"""
-def init(gw, request, path, port=12321):
- from pypy.tool.build import execnetconference
-
- conference = execnetconference.conference(gw, port, False)
- channel = conference.remote_exec(initcode % (path, request))
- return channel
-
-if __name__ == '__main__':
- from py.execnet import SshGateway, PopenGateway
- from pypy.config.config import make_dict
-
- optparser, options, args = parse_options(config, tool_config)
-
- sysinfo = make_dict(config.system_config)
- compileinfo = make_dict(config.compile_config)
-
- buildrequest = build.BuildRequest(args[0], sysinfo, compileinfo,
- config.svnpath_to_url(
- tool_config.svnpath),
- tool_config.svnrev,
- tool_config.revrange)
-
- print 'going to start compile job with info:'
- for k, v in sysinfo.items():
- print '%s: %r' % (k, v)
- print
- print config.compile_config
-
- if config.server in ['localhost', '127.0.0.1']:
- gw = PopenGateway()
- else:
- gw = SshGateway(config.server)
-
- channel = init(gw, buildrequest, config.path, port=config.port)
- data = channel.receive()
- if type(data) == str:
- print data
- for line in channel:
- print line
- elif type(data) != dict:
- raise ValueError, 'invalid data returned: %r' % (data,)
- else:
- if data['path']:
- print ('a suitable result is already available, you can find it '
- 'at "%s" on %s' % (data['path'], config.server))
- else:
- print data['message']
- print 'you will be mailed once it\'s ready'
- channel.close()
- gw.exit()
+from pypy.tool.build.compile import main, getrequest
+from py.execnet import SshGateway, PopenGateway
+request, foreground = getrequest(config)
+main(config, request, foreground)
Modified: pypy/dist/pypy/tool/build/bin/metaserver
==============================================================================
--- pypy/dist/pypy/tool/build/bin/metaserver (original)
+++ pypy/dist/pypy/tool/build/bin/metaserver Mon Mar 19 15:11:56 2007
@@ -3,22 +3,8 @@
import autopath
import path
from pypy.tool.build import config
+from pypy.tool.build.metaserver import main
if __name__ == '__main__':
- from py.execnet import SshGateway, PopenGateway
- from pypy.tool.build.metaserver import init
-
- if config.server in ['localhost', '127.0.0.1']:
- gw = PopenGateway()
- else:
- gw = SshGateway(config.server)
- channel = init(gw, config)
-
- try:
- while 1:
- data = channel.receive()
- print data
- finally:
- channel.close()
- gw.exit()
+ main(config)
Added: pypy/dist/pypy/tool/build/compile.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/tool/build/compile.py Mon Mar 19 15:11:56 2007
@@ -0,0 +1,204 @@
+import sys
+import random
+import time
+import py
+from pypy.tool.build.bin import path
+from pypy.tool.build import config
+from pypy.tool.build import build
+from pypy.tool.build import execnetconference
+
+POLLTIME = 5 # for --foreground polling
+
+def get_gateway(config):
+ if config.server in ['localhost', '127.0.0.1']:
+ gw = py.execnet.PopenGateway()
+ else:
+ gw = py.execnet.SshGateway(config.server)
+ return gw
+
+def parse_options(config, args=None):
+ # merge system + compile options into one optionparser
+ from py.compat.optparse import OptionParser, OptionGroup
+ from pypy.config.config import to_optparse
+
+ optparser = to_optparse(config.system_config)
+ to_optparse(config.compile_config, parser=optparser)
+ to_optparse(config.tool_config, parser=optparser)
+ optparser.add_option('', '--foreground', action="store_true",
+ dest='foreground', default=False,
+ help='block until build is available and download it '
+ 'immediately')
+
+ (options, args) = optparser.parse_args()
+
+ if not args or len(args) != 1:
+ optparser.error('please provide an email address')
+
+ return optparser, options, args
+
+initcode = """
+ import sys
+ import time
+ sys.path += %r
+ bufsize = 1024
+
+ try:
+ try:
+ from pypy.tool.build import metaserver_instance
+ from pypy.tool.build import build
+ ret = metaserver_instance.compile(%r)
+ channel.send(ret)
+ except Exception, e:
+ channel.send(str(e))
+ finally:
+ channel.close()
+"""
+def init(gw, request, path, port):
+ conference = execnetconference.conference(gw, port, False)
+ channel = conference.remote_exec(initcode % (path, request))
+ return channel
+
+checkcode = """
+ import sys
+ sys.path += %r
+ bufsize = 1024
+ try:
+ reqid = channel.receive()
+ from pypy.tool.build import metaserver_instance
+ from pypy.tool.build import build
+ for tb in metaserver_instance._done:
+ if tb.request.id() == reqid:
+ channel.send({'error': str(tb.error)})
+ else:
+ channel.send(None)
+ finally:
+ channel.close()
+"""
+def check_server(config, id, path, port):
+ gw = get_gateway(config)
+ try:
+ conference = execnetconference.conference(gw, port, False)
+ channel = conference.remote_exec(checkcode % (path,))
+ try:
+ channel.send(id)
+ ret = channel.receive()
+ finally:
+ channel.close()
+ finally:
+ gw.exit()
+ return ret
+
+zipcode = """
+ import sys
+ sys.path += %r
+ bufsize = 1024
+ try:
+ reqid = channel.receive()
+ from pypy.tool.build import metaserver_instance
+ from pypy.tool.build import build
+ for tb in metaserver_instance._done:
+ if tb.request.id() == reqid:
+ fp = tb.zipfile.open('rb')
+ try:
+ while 1:
+ data = fp.read(bufsize)
+ channel.send(data)
+ if len(data) < bufsize:
+ channel.send(None)
+ break
+ finally:
+ fp.close()
+ finally:
+ channel.close()
+"""
+def savezip(config, id, path, port, savepath):
+ gw = get_gateway(config)
+ savepath = py.path.local(savepath)
+ try:
+ conference = execnetconference.conference(gw, port, False)
+ channel = conference.remote_exec(zipcode % (path,))
+ try:
+ channel.send(id)
+ fp = savepath.open('wb')
+ try:
+ while 1:
+ data = channel.receive()
+ if data is None:
+ break
+ fp.write(data)
+ finally:
+ fp.close()
+ finally:
+ channel.close()
+ finally:
+ gw.exit()
+
+def getrequest(config, args=None):
+ from pypy.config.config import make_dict
+
+ optparser, options, args = parse_options(config, args=args)
+
+ sysinfo = make_dict(config.system_config)
+ compileinfo = make_dict(config.compile_config)
+
+ buildrequest = build.BuildRequest(args[0], sysinfo, compileinfo,
+ config.svnpath_to_url(
+ config.tool_config.svnpath),
+ config.tool_config.svnrev,
+ config.tool_config.revrange)
+ return buildrequest, options.foreground
+
+def main(config, request, foreground=False):
+ gateway = get_gateway(config)
+
+ inprogress = False
+ try:
+ print 'going to start compile job with info:'
+ for k, v in request.sysinfo.items():
+ print '%s: %r' % (k, v)
+ print
+ print config.compile_config
+
+ channel = init(gateway, request, config.path, port=config.port)
+ try:
+ data = channel.receive()
+ if type(data) == str:
+ print data
+ for line in channel:
+ print line
+ elif type(data) != dict:
+ raise ValueError, 'invalid data returned: %r' % (data,)
+ else:
+ if data['path']:
+ print ('a suitable result is already available, you can '
+ 'find it at "%s" on %s' % (data['path'],
+ config.server))
+ else:
+ print data['message']
+ print 'the id of this build request is: %s' % (data['id'],)
+ inprogress = True
+ finally:
+ channel.close()
+ finally:
+ gateway.exit()
+
+ if foreground and inprogress:
+ print 'waiting until it\'s done'
+ error = None
+ while 1:
+ ret = check_server(config, request.id(), config.path,
+ config.port)
+ if ret is not None:
+ error = ret['error']
+ break
+ time.sleep(POLLTIME)
+ if error and error != 'None':
+ print 'error:', error
+ else:
+ zipfile = py.path.local('data.zip')
+ savezip(config, request.id(), config.path,
+ config.port, zipfile)
+ print 'done, the result can be found in "data.zip"'
+ elif inprogress:
+ print 'you will be mailed once it\'s ready'
+
Modified: pypy/dist/pypy/tool/build/config.py
==============================================================================
--- pypy/dist/pypy/tool/build/config.py (original)
+++ pypy/dist/pypy/tool/build/config.py Mon Mar 19 15:11:56 2007
@@ -29,6 +29,10 @@
compile_config.override({'translation.backend': 'c',
'translation.gc': 'boehm'})
+# svn path and revision, etc.
+from pypy.tool.build.tooloption import tool_optiondescription
+tool_config = Config(tool_optiondescription)
+
# settings for the server
projectname = 'pypy'
buildpath = packageparent.ensure('/pypy/tool/build/builds', dir=True)
@@ -64,3 +68,7 @@
return 'http://codespeak.net/pypy/%s/data.zip' % (
p.relto(py.magic.autopath().dirpath()),)
+# this should contain the dotted name of the package where 'config'
+# can be found on the metaserver (used for remote imports)
+configpath = 'pypy.tool.build.config'
+
Modified: pypy/dist/pypy/tool/build/metaserver.py
==============================================================================
--- pypy/dist/pypy/tool/build/metaserver.py (original)
+++ pypy/dist/pypy/tool/build/metaserver.py Mon Mar 19 15:11:56 2007
@@ -85,18 +85,33 @@
for bp in self._done:
if request.has_satisfying_data(bp.request):
path = str(bp)
- self._channel.send('already a build for this info available')
- return {'path': path, 'id': requestid, 'isbuilding': True,
+ self._channel.send(
+ 'already a build for this info available as %s' % (
+ bp.request.id(),))
+ return {'path': path, 'id': bp.request.id(),
+ 'isbuilding': True,
'message': 'build is already available'}
for builder in self._builders:
- if builder.busy_on and request.has_satisfying_data(builder.busy_on):
+ if (builder.busy_on and
+ request.has_satisfying_data(builder.busy_on)):
+ id = builder.busy_on.id()
self._channel.send(
- "build for %s currently in progress on '%s'" % (
- request, builder.hostname))
+ "build for %s currently in progress on '%s' as %s" % (
+ request.id(), builder.hostname, id))
self._waiting.append(request)
- return {'path': None, 'id': requestid, 'isbuilding': True,
+ return {'path': None, 'id': id, 'isbuilding': True,
'message': "this build is already in progress "
"on '%s'" % (builder.hostname,)}
+ for br in self._waiting + self._queued:
+ if br.has_satisfying_data(request):
+ id = br.id()
+ self.channel.send(
+ 'build for %s already queued as %s' % (
+ request.id(), id))
+ return {'path': None, 'id': id, 'isbuilding': False,
+ 'message': ('no suitable server found, and a '
+ 'similar request was already queued '
+ 'as %s' % (id,))}
# we don't have a build for this yet, find a builder to compile it
hostname = self.run(request)
if hostname is not None:
@@ -247,9 +262,11 @@
if self.config.mailhost is not None:
try:
if buildpath.error:
+ excname = str(buildpath.error)
+ if hasattr(buildpath.error, '__class__'):
+ excname = buildpath.error.__class__.__name__
subject = '%s - %s during compilation' % (
- self.config.projectname,
- buildpath.error.__class__.__name__)
+ self.config.projectname, excname)
body = ('There was an error during the compilation you '
'requested. The log can be found below.'
'\n\n%s' % (buildpath.log,))
@@ -284,7 +301,7 @@
try:
try:
from pypy.tool.build.metaserver import MetaServer
- from pypy.tool.build import config
+ import %s as config
server = MetaServer(config, channel)
# make the metaserver available to build servers as
@@ -307,6 +324,25 @@
from pypy.tool.build import execnetconference
conference = execnetconference.conference(gw, config.port, True)
- channel = conference.remote_exec(initcode % (config.path,))
+ channel = conference.remote_exec(initcode % (config.path,
+ config.configpath))
return channel
+def main(config):
+ from py.execnet import SshGateway, PopenGateway
+ from pypy.tool.build.metaserver import init
+
+ if config.server in ['localhost', '127.0.0.1']:
+ gw = PopenGateway()
+ else:
+ gw = SshGateway(config.server)
+ channel = init(gw, config)
+
+ try:
+ while 1:
+ data = channel.receive()
+ print data
+ finally:
+ channel.close()
+ gw.exit()
+
Modified: pypy/dist/pypy/tool/build/systemoption.py
==============================================================================
--- pypy/dist/pypy/tool/build/systemoption.py (original)
+++ pypy/dist/pypy/tool/build/systemoption.py Mon Mar 19 15:11:56 2007
@@ -1,6 +1,6 @@
import py
from pypy.config.config import OptionDescription, BoolOption, IntOption
-from pypy.config.config import ChoiceOption, to_optparse, Config
+from pypy.config.config import ChoiceOption, Config
import sys
system_optiondescription = OptionDescription('system', '', [
Added: pypy/dist/pypy/tool/build/test/test_compile.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/tool/build/test/test_compile.py Mon Mar 19 15:11:56 2007
@@ -0,0 +1,155 @@
+import py
+import threading
+from pypy.tool.build import execnetconference
+from pypy.tool.build import config
+from pypy.tool.build.compile import main
+from pypy.tool.build.test import fake
+from pypy.tool.build import build
+from py.__.path.svn.testing import svntestbase
+from pypy.tool.build.conftest import option
+
+here = py.magic.autopath().dirpath()
+packageparent = here.dirpath().dirpath().dirpath()
+
+class FakeServer(object):
+ remote_code = """
+ import sys
+ sys.path.append(%r)
+
+ from pypy.tool import build
+ from pypy.tool.build.build import BuildPath
+
+ class FakeMetaServer(object):
+ def __init__(self):
+ self._waiting = []
+ self._done = []
+
+ def compile(self, request):
+ self._waiting.append(request)
+ return {'path': None, 'id': request.id(), 'isbuilding': True,
+ 'message': 'found server'}
+
+ def waiting_ids(self):
+ ret = []
+ for r in self._waiting:
+ ret.append(r.id())
+ return ret
+
+ def compilation_done(self, id, path, log):
+ for r in self._waiting:
+ if r.id() == id:
+ self._waiting.remove(r)
+ bp = BuildPath(path)
+ bp.log = log
+ bp.request = r
+ bp.zip = 'foo'
+ self._done.append(bp)
+
+ try:
+ build.metaserver_instance = ms = FakeMetaServer()
+
+ # notify we're done
+ channel.send(None)
+
+ while 1:
+ command, data = channel.receive()
+ if command == 'quit':
+ break
+ elif command == 'compilation_done':
+ id, path, log = data
+ ms.compilation_done(id, path, log)
+ channel.send(None)
+ elif command == 'waiting_ids':
+ channel.send(ms.waiting_ids())
+ finally:
+ channel.close()
+ """ % (str(packageparent),)
+ def __init__(self):
+ self.gw = gw = py.execnet.PopenGateway()
+ conference = execnetconference.conference(gw, config.testport, True)
+ self.channel = channel = conference.remote_exec(self.remote_code)
+ channel.receive()
+
+ def close(self):
+ self.channel.send(('quit', None))
+ self.channel.close()
+ self.gw.exit()
+
+ def compilation_done(self, id, path, log):
+ self.channel.send(('compilation_done', (id, path, log)))
+ self.channel.receive()
+
+ def waiting_ids(self):
+ self.channel.send(('waiting_ids', None))
+ return self.channel.receive()
+
+def get_test_config():
+ from pypy.config.config import OptionDescription, IntOption
+ from pypy.config.config import ChoiceOption, Config
+ sysconfig = Config(OptionDescription('system', '', [
+ ChoiceOption('os', 'operating system', ['win32', 'linux2'],
+ default='linux'),
+ ]))
+ compileconfig = Config(OptionDescription('compileinfo', '', [
+ IntOption('somevalue', 'some value', default=0),
+ ]))
+ return fake.Container(
+ server='localhost',
+ port=config.testport,
+ system_config=sysconfig,
+ compile_config=compileconfig,
+ path=[str(packageparent)],
+ check_svnroot=lambda r: True,
+ svnpath_to_url=lambda p: 'file://%s' % (p,),
+ )
+
+def test_compile():
+ # functional test, sorry :|
+ if not option.functional:
+ py.test.skip('skipping functional test, use --functional to run it')
+
+ repo, wc = svntestbase.getrepowc('test_compile')
+ temp = py.test.ensuretemp('test_compile.buildpath')
+ wc.ensure('foo', dir=True)
+ wc.commit('added foo')
+ path = repo + '/foo'
+ gw = py.execnet.PopenGateway()
+ s = FakeServer()
+ try:
+ ids = s.waiting_ids()
+ assert len(ids) == 0
+
+ # first test a non-blocking compile
+ req = build.BuildRequest('foo at bar.com', {'foo': 'bar'}, {}, path, 1, 1)
+ reqid = req.id()
+ t = threading.Thread(target=main, args=(get_test_config(), req))
+ t.start()
+ t.join(2)
+ assert not t.isAlive()
+ ids = s.waiting_ids()
+ assert ids == [reqid]
+ s.compilation_done(reqid, str(temp), 'no problems')
+ ids = s.waiting_ids()
+ assert len(ids) == 0
+
+ # now for a blocking one
+ req = build.BuildRequest('foo at baz.com', {'foo': 'bar'}, {}, path, 1, 1)
+ reqid = req.id()
+ t = threading.Thread(target=main, args=(get_test_config(), req, True))
+ t.start()
+ t.join(5)
+ assert t.isAlive() # still blocking after 2 secs
+ ids = s.waiting_ids()
+ assert ids == [reqid]
+ s.compilation_done(reqid, str(temp), 'no problems')
+ t.join(15)
+ assert not t.isAlive() # should have stopped blocking now
+ ids = s.waiting_ids()
+ assert ids == []
+ finally:
+ try:
+ s.close()
+ gw.exit()
+ except IOError:
+ pass
+
Modified: pypy/dist/pypy/tool/build/test/test_pypybuilder.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_pypybuilder.py (original)
+++ pypy/dist/pypy/tool/build/test/test_pypybuilder.py Mon Mar 19 15:11:56 2007
@@ -44,7 +44,7 @@
cfg = Container(projectname='pypytest', server='localhost',
port=config.testport,
path=config.testpath, buildpath=temppath,
- mailhost=None)
+ mailhost=None, configpath='pypy.tool.build.config')
mod.sc = sc = metaserver.init(sgw, cfg)
Modified: pypy/dist/pypy/tool/build/testproject/compileoption.py
==============================================================================
--- pypy/dist/pypy/tool/build/testproject/compileoption.py (original)
+++ pypy/dist/pypy/tool/build/testproject/compileoption.py Mon Mar 19 15:11:56 2007
@@ -4,6 +4,6 @@
import sys
compile_optiondescription = OptionDescription('compile', '', [
- BoolOption('moo', 'moo while compiling', default=False),
+ IntOption('moo', 'moo level', default=1),
])
Modified: pypy/dist/pypy/tool/build/web/templates/buildersinfo.html
==============================================================================
--- pypy/dist/pypy/tool/build/web/templates/buildersinfo.html (original)
+++ pypy/dist/pypy/tool/build/web/templates/buildersinfo.html Mon Mar 19 15:11:56 2007
@@ -35,7 +35,7 @@
%(not_busy)[c<div class="value">nothing</div>%(not_busy)]c
%(busy_on)[b
<div>
- <a class="title" href="%(href)s">%(id)s</a>
+ <a class="title" href="%(vhostroot)s%(href)s">%(id)s</a>
<div class="pair">
<span class="key">request time:</span>
<span class="value">%(request_time)s</span>
Modified: pypy/dist/pypy/tool/build/web/test/test_app.py
==============================================================================
--- pypy/dist/pypy/tool/build/web/test/test_app.py (original)
+++ pypy/dist/pypy/tool/build/web/test/test_app.py Mon Mar 19 15:11:56 2007
@@ -137,7 +137,8 @@
binfo.update({'href': 'file:///foo',
'log': 'everything went well',
'error': None,
- 'id': 'somebuild'})
+ 'id': 'somebuild',
+ 'vhostroot': ''})
return [
{'hostname': 'host1',
'sysinfo': [{
More information about the Pypy-commit
mailing list