[pypy-svn] r41202 - in pypy/dist/pypy/tool/build: . test

guido at codespeak.net guido at codespeak.net
Fri Mar 23 17:42:17 CET 2007


Author: guido
Date: Fri Mar 23 17:42:15 2007
New Revision: 41202

Modified:
   pypy/dist/pypy/tool/build/compile.py
   pypy/dist/pypy/tool/build/metaserver.py
   pypy/dist/pypy/tool/build/test/test_compile.py
   pypy/dist/pypy/tool/build/test/test_metaserver.py
Log:
Fixed a threading bug in metaserver which made that build requests could end
up on the same build server at the same time, blocking each other (and the
rest of the system), improved logging of the meta server a bit, fixed
functional test.


Modified: pypy/dist/pypy/tool/build/compile.py
==============================================================================

Modified: pypy/dist/pypy/tool/build/metaserver.py
==============================================================================
--- pypy/dist/pypy/tool/build/metaserver.py	(original)
+++ pypy/dist/pypy/tool/build/metaserver.py	Fri Mar 23 17:42:15 2007
@@ -53,14 +53,14 @@
         self._queued = [] # no builders available
         self._waiting = [] # compilation already in progress for someone else
 
+        self._requestlock = thread.allocate_lock()
         self._queuelock = thread.allocate_lock()
         self._namelock = thread.allocate_lock()
 
     def register(self, builder):
         """ register a builder (instance) """
         self._builders.append(builder)
-        self._channel.send('registered %s with info %r' % (
-                            builder, builder.sysinfo))
+        self._channel.send('registered build server %s' % (builder.hostname,))
 
     def compile(self, request):
         """start a compilation
@@ -79,53 +79,61 @@
             in any case, if the first item of the tuple returned is False,
             an email will be sent once the build is available
         """
-        # store the request, if there's already a build available the
-        # storage will return that path
-        requestid = request.id()
-        for bp in self._done:
-            if request.has_satisfying_data(bp.request):
-                path = str(bp)
-                self._channel.send(
-                    'already a build for this info available as %s' % (
-                        bp.request.id(),))
-                return {'path': path, 'id': bp.request.id(),
-                        'isbuilding': True,
-                        'message': 'build is already available'}
-        for builder in self._builders:
-            if (builder.busy_on and
-                    request.has_satisfying_data(builder.busy_on)):
-                id = builder.busy_on.id()
-                self._channel.send(
-                    "build for %s currently in progress on '%s' as %s" % (
-                        request.id(), builder.hostname, id))
-                self._waiting.append(request)
-                return {'path': None, 'id': id, 'isbuilding': True,
-                        'message': "this build is already in progress "
-                                   "on '%s'" % (builder.hostname,)}
-        for br in self._waiting + self._queued:
-            if br.has_satisfying_data(request):
-                id = br.id()
-                self._channel.send(
-                    'build for %s already queued as %s' % (
-                        request.id(), id))
-                return {'path': None, 'id': id, 'isbuilding': False,
-                        'message': ('no suitable server found, and a '
-                                    'similar request was already queued '
-                                    'as %s' % (id,))}
-        # we don't have a build for this yet, find a builder to compile it
-        hostname = self.run(request)
-        if hostname is not None:
-            return {'path': None, 'id': requestid, 'isbuilding': True,
-                    'message': "found a suitable server, going to build "
-                               "on '%s'" % (hostname, )}
-        self._queuelock.acquire()
+        self._requestlock.acquire()
         try:
-            self._queued.append(request)
+            # store the request, if there's already a build available the
+            # storage will return that path
+            requestid = request.id()
+            for bp in self._done:
+                if request.has_satisfying_data(bp.request):
+                    path = str(bp)
+                    self._channel.send(
+                        'already a build for this info available as %s' % (
+                            bp.request.id(),))
+                    return {'path': path, 'id': bp.request.id(),
+                            'isbuilding': True,
+                            'message': 'build is already available'}
+            for builder in self._builders:
+                if (builder.busy_on and
+                        request.has_satisfying_data(builder.busy_on)):
+                    id = builder.busy_on.id()
+                    self._channel.send(
+                        "build for %s currently in progress on '%s' as %s" % (
+                            request.id(), builder.hostname, id))
+                    self._waiting.append(request)
+                    return {'path': None, 'id': id, 'isbuilding': True,
+                            'message': "this build is already in progress "
+                                       "on '%s'" % (builder.hostname,)}
+            for br in self._waiting + self._queued:
+                if br.has_satisfying_data(request):
+                    id = br.id()
+                    self._channel.send(
+                        'build for %s already queued as %s' % (
+                            request.id(), id))
+                    return {'path': None, 'id': id, 'isbuilding': False,
+                            'message': ('no suitable server found, and a '
+                                        'similar request was already queued '
+                                        'as %s' % (id,))}
+            # we don't have a build for this yet, find a builder to compile it
+            hostname = self.run(request)
+            if hostname is not None:
+                self._channel.send('going to send compile job for request %s'
+                                   'to %s' % (request.id(), builder.hostname))
+                return {'path': None, 'id': requestid, 'isbuilding': True,
+                        'message': "found a suitable server, going to build "
+                                   "on '%s'" % (hostname, )}
+            self._channel.send('no suitable build server available for '
+                               'compilation of %s' % (request.id(),))
+            self._queuelock.acquire()
+            try:
+                self._queued.append(request)
+            finally:
+                self._queuelock.release()
+            return {'path': None, 'id': requestid, 'isbuilding': False,
+                    'message': 'no suitable build server found; your request '
+                               'is queued'}
         finally:
-            self._queuelock.release()
-        return {'path': None, 'id': requestid, 'isbuilding': False,
-                'message': 'no suitable build server found; your request '
-                           'is queued'}
+            self._requestlock.release()
     
     def run(self, request):
         """find a suitable build server and run the job if possible"""
@@ -140,22 +148,9 @@
                     request in builder.refused):
                 continue
             else:
-                self._channel.send(
-                    'going to send compile job for request %s to %s' % (
-                        request, builder.hostname
-                    )
-                )
                 accepted = builder.compile(request)
                 if accepted:
-                    self._channel.send('compile job accepted')
                     return builder.hostname
-                else:
-                    self._channel.send('compile job denied')
-        self._channel.send(
-            'no suitable build server available for compilation of %s' % (
-                request,
-            )
-        )
 
     def serve_forever(self):
         """this keeps the script from dying, and re-tries jobs"""
@@ -177,14 +172,6 @@
                 self._test_waiting()
                 self._try_queued()
 
-    def _cleanup(self):
-        for builder in self._builders:
-            try:
-                builder.channel.close()
-            except EOFError:
-                pass
-        self._channel.close()
-
     def get_new_buildpath(self, request):
         path = BuildPath(str(self._buildroot / self._create_filename()))
         path.request = request
@@ -195,7 +182,8 @@
         self._queuelock.acquire()
         try:
             self._channel.send('compilation done for %s, written to %s' % (
-                                                buildpath.request, buildpath))
+                                                buildpath.request.id(),
+                                                buildpath))
             emails = [buildpath.request.email]
             self._done.append(buildpath)
             waiting = self._waiting[:]
@@ -216,13 +204,21 @@
             for builder in builders:
                 if builder.channel.isclosed():
                     self._channel.send('build server %s disconnected' % (
-                        builder,))
+                        builder.hostname,))
                     if builder.busy_on:
                         self._queued.append(builder.busy_on)
                     self._builders.remove(builder)
         finally:
             self._queuelock.release()
 
+    def _cleanup(self):
+        for builder in self._builders:
+            try:
+                builder.channel.close()
+            except EOFError:
+                pass
+        self._channel.close()
+
     def _test_waiting(self):
         """ for each waiting request, see if the compilation is still alive
 

Modified: pypy/dist/pypy/tool/build/test/test_compile.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_compile.py	(original)
+++ pypy/dist/pypy/tool/build/test/test_compile.py	Fri Mar 23 17:42:15 2007
@@ -105,7 +105,6 @@
 
 def create_test_repo_file(name):
     repo, wc = svntestbase.getrepowc('test_compile')
-    temp = py.test.ensuretemp('test_compile.%s' % (name,))
     wc.ensure('foo', dir=True)
     wc.commit('added foo')
     path = repo + '/foo'
@@ -115,6 +114,7 @@
     # functional test, sorry :|
     if not option.functional:
         py.test.skip('skipping functional test, use --functional to run it')
+    temp = py.test.ensuretemp('test_compile.test_blocking')
     path = create_test_repo_file('test_blocking')
 
     gw = py.execnet.PopenGateway()

Modified: pypy/dist/pypy/tool/build/test/test_metaserver.py
==============================================================================
--- pypy/dist/pypy/tool/build/test/test_metaserver.py	(original)
+++ pypy/dist/pypy/tool/build/test/test_metaserver.py	Fri Mar 23 17:42:15 2007
@@ -60,8 +60,6 @@
     assert "fake" in ret['message'] # hostname
     ret = svr._channel.receive()
     assert ret.find('going to send compile job') > -1
-    acceptedmsg = svr._channel.receive()
-    assert acceptedmsg == 'compile job accepted'
     ret = c1.channel.receive()
     assert ret == br.serialize()
     none = c1.channel.receive()
@@ -79,8 +77,6 @@
     svr.compile(br3)
     ret = svr._channel.receive()
     assert ret.find('going to send') > -1
-    acceptedmsg = svr._channel.receive()
-    assert acceptedmsg == 'compile job accepted'
     assert c2.channel.receive() == br3.serialize()
     assert c2.channel.receive() is None
     py.test.raises(IndexError, "c1.channel.receive()")



More information about the Pypy-commit mailing list