[pypy-commit] buildbot default: use buildbot locks to make sure that benchmarks are run when nothing else is using the CPU; it needs a bit of care because we use two different slaves (tannit32 and tannit64) on the same physical machine, look at the comment at the top of builds.py

antocuni noreply at buildbot.pypy.org
Fri Jul 29 14:20:58 CEST 2011


Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r541:9724de536c4c
Date: 2011-07-29 14:07 +0200
http://bitbucket.org/pypy/buildbot/changeset/9724de536c4c/

Log:	use buildbot locks to make sure that benchmarks are run when nothing
	else is using the CPU; it needs a bit of care because we use two
	different slaves (tannit32 and tannit64) on the same physical
	machine, look at the comment at the top of builds.py

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -2,9 +2,26 @@
 from buildbot.steps import source, shell, transfer, master
 from buildbot.status.builder import SUCCESS
 from buildbot.process.properties import WithProperties
+from buildbot import locks
 from pypybuildbot.util import symlink_force
 import os
 
+# buildbot supports SlaveLocks, which can be used to limit the amout of builds
+# to be run on each slave in parallel.  However, they assume that each
+# buildslave is on a differen physical machine, which is not the case for
+# tannit32 and tannit64.  As a result, we have to use a global lock, and
+# manually tell each builder that uses tannit to acquire it.
+#
+# Look at the various "locks" session in master.py/BuildmasterConfig.  For
+# benchmarks, the locks is aquired for the single steps: this way we can run
+# translations in parallel, but then the actual benchmarks are run in
+# sequence.
+
+# there are 8 logical CPUs, but only 4 physical ones. We use a maxCount of 6
+# to allow a bit more parallelism, but not too much
+TannitCPU = locks.MasterLock('tannit_cpu', maxCount=6)
+
+
 class ShellCmd(shell.ShellCommand):
     # our own version that can distinguish abort cases (rc == -1)
 
@@ -142,7 +159,7 @@
     repourl = 'https://bitbucket.org/pypy/pypy/'
     if getpass.getuser() == 'antocuni':
         # for debugging
-        repourl = '/home/antocuni/pypy/pypy-hg'
+        repourl = '/home/antocuni/pypy/default'
     #
     if platform == 'win32':
         command = "if not exist .hg rmdir /q /s ."
@@ -285,13 +302,23 @@
             command=['svn', 'co', 'https://bitbucket.org/pypy/benchmarks/trunk',
                      'benchmarks'],
             workdir='.'))
-        self.addStep(Translate(['-Ojit'], []))
+        self.addStep(
+            Translate(
+                translationArgs=['-Ojit'],
+                targetArgs=[],
+                haltOnFailure=False,
+                # this step can be executed in parallel with other builds
+                locks=[TannitCPU.access('counting')],
+                )
+            )
         pypy_c_rel = "../build/pypy/translator/goal/pypy-c"
         if postfix:
             addopts = ['--postfix', postfix]
         else:
             addopts = []
         self.addStep(ShellCmd(
+            # this step needs exclusive access to the CPU
+            locks=[TannitCPU.access('exclusive')],
             description="run benchmarks on top of pypy-c",
             command=["python", "runner.py", '--output-filename', 'result.json',
                     '--pypy-c', pypy_c_rel,
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -105,6 +105,7 @@
 
 
 pypybuilds = load('pypybuildbot.builds')
+TannitCPU = pypybuilds.TannitCPU
 
 pypyOwnTestFactory = pypybuilds.Own()
 pypyOwnTestFactoryWin = pypybuilds.Own(platform="win32")
@@ -250,13 +251,25 @@
                    "slavenames": ["cobra", "bigdogvm1", "tannit32"],
                    "builddir": LINUX32,
                    "factory": pypyOwnTestFactory,
-                   "category": 'linux32'
+                   "category": 'linux32',
+                   # this build needs 4 CPUs
+                   "locks": [TannitCPU.access('counting'),
+                             TannitCPU.access('counting'),
+                             TannitCPU.access('counting'),
+                             TannitCPU.access('counting'),
+                             ],
                   },
                   {"name": LINUX64,
                    "slavenames": ["tannit64"],
                    "builddir": LINUX64,
                    "factory": pypyOwnTestFactory,
-                   "category": 'linux64'
+                   "category": 'linux64',
+                   # this build needs 4 CPUs
+                   "locks": [TannitCPU.access('counting'),
+                             TannitCPU.access('counting'),
+                             TannitCPU.access('counting'),
+                             TannitCPU.access('counting'),
+                             ],
                   },
                   {"name": MACOSX32,
                    "slavenames": ["minime"],
@@ -274,25 +287,29 @@
                    "slavenames": ["bigdogvm1", "tannit32"],
                    "builddir": APPLVLLINUX32,
                    "factory": pypyTranslatedAppLevelTestFactory,
-                   'category': 'linux32'
+                   'category': 'linux32',
+                   "locks": [TannitCPU.access('counting')],
                   },
                   {"name": APPLVLLINUX64,
                    "slavenames": ["tannit64"],
                    "builddir": APPLVLLINUX64,
                    "factory": pypyTranslatedAppLevelTestFactory64,
-                   "category": "linux64"
+                   "category": "linux64",
+                   "locks": [TannitCPU.access('counting')],
                   },
                   {"name": STACKLESSAPPLVLLINUX32,
                    "slavenames": ["bigdogvm1", "tannit32"],
                    "builddir": STACKLESSAPPLVLLINUX32,
                    "factory": pypyStacklessTranslatedAppLevelTestFactory,
-                   "category": 'linux32-stackless'
+                   "category": 'linux32-stackless',
+                   "locks": [TannitCPU.access('counting')],
                   },
                   {"name": OJITLINUX32,
                    "slavenames": ["bigdogvm1", "tannit32"],
                    "builddir": OJITLINUX32,
                    "factory": pypy_OjitTranslatedTestFactory,
-                   "category": 'linux32'
+                   "category": 'linux32',
+                   "locks": [TannitCPU.access('counting')],
                   },
                   {"name": APPLVLWIN32,
                    "slavenames": ["bigboard"],
@@ -311,12 +328,14 @@
                    'builddir' : JITLINUX32,
                    'factory' : pypyJITTranslatedTestFactory,
                    'category' : 'linux32',
+                   "locks": [TannitCPU.access('counting')],
                    },
                   {'name': JITLINUX64,
                    'slavenames': ['tannit64'],
                    'builddir': JITLINUX64,
                    'factory': pypyJITTranslatedTestFactory64,
                    'category': 'linux64',
+                   "locks": [TannitCPU.access('counting')],
                   },
                   {"name" : JITMACOSX64,
                    "slavenames": ["macmini-mvt", "xerxes"],
@@ -334,19 +353,22 @@
                    "slavenames": ["tannit32", "bigdogvm1"],
                    "builddir": JITONLYLINUX32,
                    "factory": pypyJitOnlyOwnTestFactory,
-                   "category": 'linux32'
+                   "category": 'linux32',
+                   "locks": [TannitCPU.access('counting')],
                   },
                   {"name": JITBENCH,
                    "slavenames": ["tannit32"],
                    "builddir": JITBENCH,
                    "factory": pypyJITBenchmarkFactory,
                    "category": 'benchmark-run',
+                   # the locks are acquired with fine grain inside the build
                   },
                   {"name": JITBENCH64,
                    "slavenames": ["tannit64"],
                    "builddir": JITBENCH64,
                    "factory": pypyJITBenchmarkFactory64,
                    "category": "benchmark-run",
+                   # the locks are acquired with fine grain inside the build
                    },
                 ],
 


More information about the pypy-commit mailing list