[pypy-commit] buildbot default: parallel -> parallel_runs, update 64-bit Dockerfile with lessons learned

[mattip]

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1117:36c2b9791482
Date: 2019-12-02 15:47 +0200
http://bitbucket.org/pypy/buildbot/changeset/36c2b9791482/

Log:	parallel -> parallel_runs, update 64-bit Dockerfile with lessons
	learned

diff --git a/docker/Dockerfile b/docker/Dockerfile
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,7 +1,7 @@
 # Build with something like this, where
 #  -t is the name of the image
 #  -f is this file
-#  1001 is the UID of the user to run as
+#  1001 is the UID of the user to run as, you might want to use $UID
 #  docker is the directory where install_ffi.sh is
 # docker build -t buildslave_x86_64 --build-arg BUILDSLAVE_UID=1001 -f docker/Dockerfile docker 
 #
@@ -24,27 +24,29 @@
 # docker run -it --rm -v<abspath/to/builder/dir>:/build_dir> \
 #     -ePYPY_MAKE_PORTABLE=1 buildslave_x86_64
 #
-# You might want to keep the PYPY_USESSION_DIR where the testing/building
-# artifacts are. Docker will not do this for you, so do something like this
+# You might want to keep the TMPDIR where the testing/building
+# artifacts are. This will normally be inside the docker, so do something like this
 # to save the files outside the docker
 #
 # mkdir -p build_dir/tmp
 # docker run -it --rm -v<abspath/to/builder/dir>:/build_dir> \
-#     -ePYPY_USESSION_DIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 buildslave_x86_64
+#     -eTMPDIR=/build_dir/tmp -ePYPY_MAKE_PORTABLE=1 buildslave_x86_64
 #
 # To enter the buildslave image, add a shell command to the end
 #
 # docker run -it -v<abspath/to/builder/dir>:/build_dir> \
-#     -ePYPY_USESSION_DIR=/build_dir/tmp buildslave_x86_64 /bin/bash
+#     -eTMPDIR=/build_dir/tmp buildslave_x86_64 /bin/bash
 #
+# This will enter the docker as root. Don't do "su - buildslave",
+# do "su buildslave"
 
 FROM quay.io/pypa/manylinux2010_x86_64:latest
 WORKDIR /root
 
 RUN yum -y update
 RUN yum install -y wget bzip2-devel zlib-devel glibc-devel libX11-devel \
-   libXt-devel patch expat-devel libXft-devel tk-devel gdbm-devel gdb \
-   perl xz-devel ncurses-devel sqlite-devel gc-devel prelink python-virtualenv
+   libXt-devel patch expat-devel libXft-devel tk-devel gdbm-devel gdb vim \
+   perl xz-devel ncurses-devel sqlite-devel prelink python-virtualenv
 
 # Taken from pyca/infra/cryptography-manylinux
 # centos6 libffi is buggy, download and use a newer one
@@ -54,12 +56,24 @@
 ADD install_libffi.sh /root/install_libffi.sh
 RUN sh install_libffi.sh manylinux2010 2>&1 | tee /root/install_libffi.log
 RUN sh install_openssl.sh manylinux2010 2>&1 | tee /root/install_openssl.log
+# Use an up-to-date version of xz for lzma
 ADD install_xz5.sh /root/install_xz5.sh
 ADD lasse_collin_pubkey.txt /root/lasse_collin_pubkey.txt
 ADD xz-5.2.4.tar.gz.sig /root/xz-5.2.4.tar.gz.sig
-RUN sh install_xz5.sh manylinux2010 m32 # 2>&1 | tee /root/install_xz5.log
+RUN sh install_xz5.sh manylinux2010 2>&1 | tee /root/install_xz5.log
+# centos6 provides gc7.1, which does not work in a docker. Use a newer version
+# This is for testing only
+Add install_gc.sh /root/install_gc.sh
+RUN sh install_gc.sh 2>&1 | tee /root/install_gc.sh
+
+# prefer our libraries in /usr/local/lib
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 
 # get a pypy for translation. On x86_64 we can use portable pypy
+# XXX update this to use release versions when we have one
+
+# RUN wget -q http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-98204-a8a24325ea83-linux.tar.bz2 -O - | tar -C /opt -xj
+# RUN ln -s /opt/pypy-c-jit-98204-a8a24325ea83-linux/bin/pypy /usr/local/bin/pypy
 RUN wget -q https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-7.0.0-linux_x86_64-portable.tar.bz2 -O - | tar -C /opt -xj
 RUN ln -s /opt/pypy-7.0.0-linux_x86_64-portable/bin/pypy /usr/local/bin/pypy
 
@@ -75,8 +89,13 @@
 # Define a user
 ARG BUILDSLAVE_UID=1001
 RUN adduser buildslave --uid=$BUILDSLAVE_UID
-RUN echo parallel=4 > /home/buildslave/machine_cfg.py
+RUN echo parallel_runs=4 > /home/buildslave/machine_cfg.py
 
+# NOTE: always use
+# su buildslave
+# not
+# su - buildslave
+# to preserve env variables
 CMD if [ -e /build_dir/buildbot.tac ]; then \
     su buildslave -c "buildslave start --nodaemon /build_dir"; \
   else \
diff --git a/docker/Dockerfile32 b/docker/Dockerfile32
--- a/docker/Dockerfile32
+++ b/docker/Dockerfile32
@@ -89,7 +89,7 @@
 # Define a user
 ARG BUILDSLAVE_UID=1001
 RUN adduser buildslave --uid=$BUILDSLAVE_UID
-RUN echo parallel=4 > /home/buildslave/machine_cfg.py
+RUN echo parallel_runs=4 > /home/buildslave/machine_cfg.py
 
 # NOTE: always use
 # su buildslave