From 0dffc2fb29279482dbdebde42fbac6de0dc9d402 Mon Sep 17 00:00:00 2001 From: Paulo Matos Date: Wed, 8 May 2019 12:26:25 +0200 Subject: [PATCH] Use all CPUS available for CI on the runner and install qemu magic There are two parts to this commit: * Until now we have been hardcoding the number of cores used per job but this is wrong. We should use all cores available but limit new jobs spawning if the load is too high. This enables us to use the 40 cores available in the pinwheel server. * Unless qemu has been already installed, the job will fail because the qemu magic has not been ran. Now we run the qemu magic in any case. If the magic has been previously installed, it gets updated. This was noticed when I started running racket jobs on the brand new pinwheel. --- .gitlab-ci.yml | 44 +++++++++++++++++-------------------- .gitlab/build-test.sh | 50 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a67ea62caa..fd785451d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,6 @@ prepare-cache:llvm: tags: - linux - x86_64 - - shared-cache variables: INSTALL_DIR: $CI_PROJECT_DIR/install script: @@ -32,8 +31,10 @@ prepare-cache:llvm: - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCLANG_ANALYZER_ENABLE_Z3_SOLVER=ON -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PROJECTS=clang -DZ3_INCLUDE_DIR=$INSTALL_DIR/include/ -DCMAKE_BUILD_TYPE=MinSizeRel ../llvm/ - - make -j5 - - make -j5 install + - export cpus=$(grep -c ^processor /proc/cpuinfo) + - echo "Detected $cpus cpus" + - make -j$((cpus + 1)) -l$cpus + - make -j$((cpus + 1)) -l$cpus install cache: key: llvm-8x-HEAD paths: @@ -50,7 +51,6 @@ scan-build:racket: tags: - linux - x86_64 - - shared-cache variables: INSTALL_DIR: $CI_PROJECT_DIR/install before_script: @@ -59,7 +59,9 @@ scan-build:racket: script: - export PATH=$INSTALL_DIR/bin:$PATH - export LD_LIBRARY_PATH=$INSTALL_DIR/bin:$LD_LIBRARY_PATH - - scan-build -o scan-report_cc -analyzer-config 'crosscheck-with-z3=true' make PKGS="" CPUS=5 CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip' + - export cpus=$(grep -c ^processor /proc/cpuinfo) + - echo "Detected $cpus cpus" + - scan-build -o scan-report_cc -analyzer-config 'crosscheck-with-z3=true' make --load-average=$cpus PKGS="" CPUS=$((cpus + 1)) CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip' dependencies: - prepare-cache:llvm artifacts: @@ -72,7 +74,6 @@ scan-build:racketcs: tags: - linux - x86_64 - - shared-cache variables: INSTALL_DIR: $CI_PROJECT_DIR/install before_script: @@ -81,7 +82,9 @@ scan-build:racketcs: script: - export PATH=$INSTALL_DIR/bin:$PATH - export LD_LIBRARY_PATH=$INSTALL_DIR/bin:$LD_LIBRARY_PATH - - scan-build -o scan-report-cs_cc -analyzer-config 'crosscheck-with-z3=true' make PKGS="" CPUS=5 CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip' cs + - export cpus=$(grep -c ^processor /proc/cpuinfo) + - echo "Detected $cpus cpus" + - scan-build -o scan-report-cs_cc -analyzer-config 'crosscheck-with-z3=true' make --load-average=$cpus PKGS="" CPUS=$((cpus + 1)) CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip' cs dependencies: - prepare-cache:llvm artifacts: @@ -193,8 +196,10 @@ prepare-cache:qemu: - mkdir qemu-build - cd qemu-build - ../qemu-3.1.0/configure --static --disable-kvm --disable-xen --disable-spice --target-list='i386-linux-user aarch64-linux-user arm-linux-user mips-linux-user mipsel-linux-user mips64el-linux-user s390x-linux-user ppc64le-linux-user riscv64-linux-user' --prefix=$INSTALL_DIR - - make -j5 - - make -j5 install + - export cpus=$(grep -c ^processor /proc/cpuinfo) + - echo "Detected $cpus cpus" + - make -j$((cpus + 1)) --load-average=$cpus + - make -j$((cpus + 1)) --load-average=$cpus install cache: key: qemu-3.1.0 paths: @@ -218,7 +223,9 @@ prepare-cache:qemu: - export PATH=$INSTALL_DIR/bin:$PATH - apt-get update script: - - .gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --with-qemu-path $INSTALL_DIR + - export cpus=$(grep -c ^processor /proc/cpuinfo) + - echo "Detected $cpus cpus" + - .gitlab/build-test.sh --jobs $((cpus + 1)) --maxload ${cpus} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --with-qemu-path $INSTALL_DIR dependencies: - prepare-cache:qemu @@ -226,7 +233,6 @@ test:native:x86_64: extends: .preparearch variables: ARCH: "x86_64" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:native:armv7l: @@ -234,7 +240,6 @@ test:native:armv7l: allow_failure: true variables: ARCH: "armv7l" - JOBS: 5 INSTALL_DIR: $CI_PROJECT_DIR/install tags: - armv7l @@ -247,7 +252,6 @@ test:emulation:arm64: - schedules variables: ARCH: "arm64" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:armel: @@ -257,7 +261,6 @@ test:emulation:armel: - schedules variables: ARCH: "armel" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:armhf: @@ -267,7 +270,6 @@ test:emulation:armhf: - schedules variables: ARCH: "armhf" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:i386: @@ -277,7 +279,6 @@ test:emulation:i386: - schedules variables: ARCH: "i386" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:mips: @@ -287,7 +288,6 @@ test:emulation:mips: - schedules variables: ARCH: "mips" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:mipsel: @@ -297,7 +297,6 @@ test:emulation:mipsel: - schedules variables: ARCH: "mipsel" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:mips64el: @@ -307,7 +306,6 @@ test:emulation:mips64el: - schedules variables: ARCH: "mips64el" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:s390x: @@ -317,7 +315,6 @@ test:emulation:s390x: - schedules variables: ARCH: "s390x" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:ppc64el: @@ -327,7 +324,6 @@ test:emulation:ppc64el: - schedules variables: ARCH: "ppc64el" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install .preparearch:cs: @@ -341,7 +337,9 @@ test:emulation:ppc64el: - export PATH=$INSTALL_DIR:$PATH - apt-get update script: - - .gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --enable-cs --with-qemu-path $INSTALL_DIR + - export cpus=$(grep -c ^processor /proc/cpuinfo) + - echo "Detected $cpus cpus" + - .gitlab/build-test.sh --jobs $((cpus + 1)) --maxload ${cpus} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --enable-cs --with-qemu-path $INSTALL_DIR dependencies: - prepare-cache:qemu @@ -350,7 +348,6 @@ test:native:x86_64:cs: allow_failure: true variables: ARCH: "x86_64" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install test:emulation:i386:cs: @@ -360,6 +357,5 @@ test:emulation:i386:cs: - schedules variables: ARCH: "i386" - JOBS: 6 INSTALL_DIR: $CI_PROJECT_DIR/install diff --git a/.gitlab/build-test.sh b/.gitlab/build-test.sh index 008b72479c..1801c4278b 100755 --- a/.gitlab/build-test.sh +++ b/.gitlab/build-test.sh @@ -22,6 +22,7 @@ function usage () { echo "${MSG}" echo echo "Usage: ./build-test.sh [--jobs ]" + echo " [--maxload ]" echo " [--single-thread]" echo " [--with-arch ]" echo " [--with-debian ]" @@ -37,6 +38,7 @@ function usage () { DEBIAN= DEBIAN_MIRROR= JOBS= +MAXLOAD= RACKET_CONFIGURE_ARGS= ARCH="$(uname -m)" BUILD_DIR=${CI_PROJECT_DIR} @@ -57,6 +59,10 @@ until --single-thread) JOBS=1 ;; + --maxload) + shift + MAXLOAD=$1 + ;; --with-arch) shift ARCH=$1 @@ -154,6 +160,7 @@ function setup_chroot { mkdir "${CHROOT_DIR}" debootstrap --foreign --no-check-gpg --include=fakeroot,build-essential \ --arch="${ARCH}" "${DEBIAN}" "${CHROOT_DIR}" "${DEBIAN_MIRROR}" + setup_binfmts cp ${QEMU_PATH}/bin/qemu-${QEMU_ARCH} "${CHROOT_DIR}"/usr/bin/qemu-${QEMU_ARCH}-static chroot "${CHROOT_DIR}" ./debootstrap/debootstrap --second-stage sbuild-createchroot --arch="${ARCH}" --foreign --setup-only \ @@ -176,17 +183,54 @@ function setup_chroot { # Call ourselves again which will cause tests to run echo "Recursively calling script" if [ ${MAKE_TARGET} = "cs" ]; then - chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR} --enable-cs" + chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --maxload ${MAXLOAD} --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR} --enable-cs" else - chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR}" + chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --maxload ${MAXLOAD} --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR}" fi } +function setup_binfmts { + case ${QEMU_ARCH} in + "x86_64") + # nothing to be done + ;; + "aarch64") + update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff" + ;; + "arm") + update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff" + ;; + "i386") + # nothing to be done + ;; + "mips") + update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff" + ;; + "mipsel") + update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff" + ;; + "mips64el") + update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff" + ;; + "s390x") + update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16" --mask "\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff" + ;; + "ppc64el") + update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00" + ;; + *) + echo "Unknown QEMU architecture ${QEMU_ARCH}" + exit 1 + ;; + esac +} + # Information about environment echo "Environment information" echo "=======================" echo " Machine : $(uname -m)" echo " Jobs : ${JOBS}" +echo " Max Load : ${MAXLOAD}" echo " Target Arch : ${ARCH}" echo " chroot Path : ${CHROOT_DIR}" echo " Build Path : ${BUILD_DIR}" @@ -211,7 +255,7 @@ fi echo "Compiling" echo "Environment: $(uname -a)" -annotate-output make CPUS=${JOBS} \ +annotate-output make --load-average=${MAXLOAD} CPUS=${JOBS} \ PKGS="racket-test db-test unstable-flonum-lib net-test" \ CONFIGURE_ARGS_qq="${RACKET_CONFIGURE_ARGS}" \ ${MAKE_TARGET}