suzanne.soy/racket
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use all CPUS available for CI on the runner and install qemu magic

Browse Source 
There are two parts to this commit:
* Until now we have been hardcoding the number of cores used per job
  but this is wrong. We should use all cores available but limit new
  jobs spawning if the load is too high. This enables us to use the 40
  cores available in the pinwheel server.
* Unless qemu has been already installed, the job will fail because
  the qemu magic has not been ran. Now we run the qemu magic in any
  case. If the magic has been previously installed, it gets
  updated. This was noticed when I started running racket jobs on the
  brand new pinwheel.
This commit is contained in:
Paulo Matos 2019-05-08 12:26:25 +02:00
parent 75cc43aa56
commit 0dffc2fb29
2 changed files with 67 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
.gitlab-ci.yml
View File

@ -16,7 +16,6 @@ prepare-cache:llvm:
tags: tags:
- linux - linux
- x86_64 - x86_64
- shared-cache
variables: variables:
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
script: script:
@ -32,8 +31,10 @@ prepare-cache:llvm:
- mkdir build - mkdir build
- cd build - cd build
- cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCLANG_ANALYZER_ENABLE_Z3_SOLVER=ON -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PROJECTS=clang -DZ3_INCLUDE_DIR=$INSTALL_DIR/include/ -DCMAKE_BUILD_TYPE=MinSizeRel ../llvm/ - cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCLANG_ANALYZER_ENABLE_Z3_SOLVER=ON -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PROJECTS=clang -DZ3_INCLUDE_DIR=$INSTALL_DIR/include/ -DCMAKE_BUILD_TYPE=MinSizeRel ../llvm/
- make -j5 - export cpus=$(grep -c ^processor /proc/cpuinfo)
- make -j5 install - echo "Detected $cpus cpus"
- make -j$((cpus + 1)) -l$cpus
- make -j$((cpus + 1)) -l$cpus install
cache: cache:
key: llvm-8x-HEAD key: llvm-8x-HEAD
paths: paths:
@ -50,7 +51,6 @@ scan-build:racket:
tags: tags:
- linux - linux
- x86_64 - x86_64
- shared-cache
variables: variables:
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
before_script: before_script:
@ -59,7 +59,9 @@ scan-build:racket:
script: script:
- export PATH=$INSTALL_DIR/bin:$PATH - export PATH=$INSTALL_DIR/bin:$PATH
- export LD_LIBRARY_PATH=$INSTALL_DIR/bin:$LD_LIBRARY_PATH - export LD_LIBRARY_PATH=$INSTALL_DIR/bin:$LD_LIBRARY_PATH
- scan-build -o scan-report_cc -analyzer-config 'crosscheck-with-z3=true' make PKGS="" CPUS=5 CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip' - export cpus=$(grep -c ^processor /proc/cpuinfo)
- echo "Detected $cpus cpus"
- scan-build -o scan-report_cc -analyzer-config 'crosscheck-with-z3=true' make --load-average=$cpus PKGS="" CPUS=$((cpus + 1)) CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip'
dependencies: dependencies:
- prepare-cache:llvm - prepare-cache:llvm
artifacts: artifacts:
@ -72,7 +74,6 @@ scan-build:racketcs:
tags: tags:
- linux - linux
- x86_64 - x86_64
- shared-cache
variables: variables:
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
before_script: before_script:
@ -81,7 +82,9 @@ scan-build:racketcs:
script: script:
- export PATH=$INSTALL_DIR/bin:$PATH - export PATH=$INSTALL_DIR/bin:$PATH
- export LD_LIBRARY_PATH=$INSTALL_DIR/bin:$LD_LIBRARY_PATH - export LD_LIBRARY_PATH=$INSTALL_DIR/bin:$LD_LIBRARY_PATH
- scan-build -o scan-report-cs_cc -analyzer-config 'crosscheck-with-z3=true' make PKGS="" CPUS=5 CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip' cs - export cpus=$(grep -c ^processor /proc/cpuinfo)
- echo "Detected $cpus cpus"
- scan-build -o scan-report-cs_cc -analyzer-config 'crosscheck-with-z3=true' make --load-average=$cpus PKGS="" CPUS=$((cpus + 1)) CONFIGURE_ARGS_qq='CFLAGS="-O0 -g" CPPFLAGS="-DMZ_DECLARE_NORETURN" --disable-strip' cs
dependencies: dependencies:
- prepare-cache:llvm - prepare-cache:llvm
artifacts: artifacts:
@ -193,8 +196,10 @@ prepare-cache:qemu:
- mkdir qemu-build - mkdir qemu-build
- cd qemu-build - cd qemu-build
- ../qemu-3.1.0/configure --static --disable-kvm --disable-xen --disable-spice --target-list='i386-linux-user aarch64-linux-user arm-linux-user mips-linux-user mipsel-linux-user mips64el-linux-user s390x-linux-user ppc64le-linux-user riscv64-linux-user' --prefix=$INSTALL_DIR - ../qemu-3.1.0/configure --static --disable-kvm --disable-xen --disable-spice --target-list='i386-linux-user aarch64-linux-user arm-linux-user mips-linux-user mipsel-linux-user mips64el-linux-user s390x-linux-user ppc64le-linux-user riscv64-linux-user' --prefix=$INSTALL_DIR
- make -j5 - export cpus=$(grep -c ^processor /proc/cpuinfo)
- make -j5 install - echo "Detected $cpus cpus"
- make -j$((cpus + 1)) --load-average=$cpus
- make -j$((cpus + 1)) --load-average=$cpus install
cache: cache:
key: qemu-3.1.0 key: qemu-3.1.0
paths: paths:
@ -218,7 +223,9 @@ prepare-cache:qemu:
- export PATH=$INSTALL_DIR/bin:$PATH - export PATH=$INSTALL_DIR/bin:$PATH
- apt-get update - apt-get update
script: script:
- .gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --with-qemu-path $INSTALL_DIR - export cpus=$(grep -c ^processor /proc/cpuinfo)
- echo "Detected $cpus cpus"
- .gitlab/build-test.sh --jobs $((cpus + 1)) --maxload ${cpus} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --with-qemu-path $INSTALL_DIR
dependencies: dependencies:
- prepare-cache:qemu - prepare-cache:qemu
@ -226,7 +233,6 @@ test:native:x86_64:
extends: .preparearch extends: .preparearch
variables: variables:
ARCH: "x86_64" ARCH: "x86_64"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:native:armv7l: test:native:armv7l:
@ -234,7 +240,6 @@ test:native:armv7l:
allow_failure: true allow_failure: true
variables: variables:
ARCH: "armv7l" ARCH: "armv7l"
JOBS: 5
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
tags: tags:
- armv7l - armv7l
@ -247,7 +252,6 @@ test:emulation:arm64:
- schedules - schedules
variables: variables:
ARCH: "arm64" ARCH: "arm64"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:armel: test:emulation:armel:
@ -257,7 +261,6 @@ test:emulation:armel:
- schedules - schedules
variables: variables:
ARCH: "armel" ARCH: "armel"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:armhf: test:emulation:armhf:
@ -267,7 +270,6 @@ test:emulation:armhf:
- schedules - schedules
variables: variables:
ARCH: "armhf" ARCH: "armhf"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:i386: test:emulation:i386:
@ -277,7 +279,6 @@ test:emulation:i386:
- schedules - schedules
variables: variables:
ARCH: "i386" ARCH: "i386"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:mips: test:emulation:mips:
@ -287,7 +288,6 @@ test:emulation:mips:
- schedules - schedules
variables: variables:
ARCH: "mips" ARCH: "mips"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:mipsel: test:emulation:mipsel:
@ -297,7 +297,6 @@ test:emulation:mipsel:
- schedules - schedules
variables: variables:
ARCH: "mipsel" ARCH: "mipsel"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:mips64el: test:emulation:mips64el:
@ -307,7 +306,6 @@ test:emulation:mips64el:
- schedules - schedules
variables: variables:
ARCH: "mips64el" ARCH: "mips64el"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:s390x: test:emulation:s390x:
@ -317,7 +315,6 @@ test:emulation:s390x:
- schedules - schedules
variables: variables:
ARCH: "s390x" ARCH: "s390x"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:ppc64el: test:emulation:ppc64el:
@ -327,7 +324,6 @@ test:emulation:ppc64el:
- schedules - schedules
variables: variables:
ARCH: "ppc64el" ARCH: "ppc64el"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
.preparearch:cs: .preparearch:cs:
@ -341,7 +337,9 @@ test:emulation:ppc64el:
- export PATH=$INSTALL_DIR:$PATH - export PATH=$INSTALL_DIR:$PATH
- apt-get update - apt-get update
script: script:
- .gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --enable-cs --with-qemu-path $INSTALL_DIR - export cpus=$(grep -c ^processor /proc/cpuinfo)
- echo "Detected $cpus cpus"
- .gitlab/build-test.sh --jobs $((cpus + 1)) --maxload ${cpus} --with-arch ${ARCH} --with-debian stretch --with-debian-mirror http://ftp.de.debian.org/debian/ --with-project-path ${CI_PROJECT_DIR} --with-chroot-path /tmp/racket-${ARCH}-${CI_COMMIT_SHORT_SHA}-chroot --enable-cs --with-qemu-path $INSTALL_DIR
dependencies: dependencies:
- prepare-cache:qemu - prepare-cache:qemu
@ -350,7 +348,6 @@ test:native:x86_64:cs:
allow_failure: true allow_failure: true
variables: variables:
ARCH: "x86_64" ARCH: "x86_64"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install
test:emulation:i386:cs: test:emulation:i386:cs:
@ -360,6 +357,5 @@ test:emulation:i386:cs:
- schedules - schedules
variables: variables:
ARCH: "i386" ARCH: "i386"
JOBS: 6
INSTALL_DIR: $CI_PROJECT_DIR/install INSTALL_DIR: $CI_PROJECT_DIR/install

50
.gitlab/build-test.sh
View File

@ -22,6 +22,7 @@ function usage () {
echo "${MSG}" echo "${MSG}"
echo echo
echo "Usage: ./build-test.sh [--jobs <count>]" echo "Usage: ./build-test.sh [--jobs <count>]"
echo " [--maxload <value>]"
echo " [--single-thread]" echo " [--single-thread]"
echo " [--with-arch <arch>]" echo " [--with-arch <arch>]"
echo " [--with-debian <debian>]" echo " [--with-debian <debian>]"
@ -37,6 +38,7 @@ function usage () {
DEBIAN= DEBIAN=
DEBIAN_MIRROR= DEBIAN_MIRROR=
JOBS= JOBS=
MAXLOAD=
RACKET_CONFIGURE_ARGS= RACKET_CONFIGURE_ARGS=
ARCH="$(uname -m)" ARCH="$(uname -m)"
BUILD_DIR=${CI_PROJECT_DIR} BUILD_DIR=${CI_PROJECT_DIR}
@ -57,6 +59,10 @@ until
--single-thread) --single-thread)
JOBS=1 JOBS=1
;; ;;
--maxload)
shift
MAXLOAD=$1
;;
--with-arch) --with-arch)
shift shift
ARCH=$1 ARCH=$1
@ -154,6 +160,7 @@ function setup_chroot {
mkdir "${CHROOT_DIR}" mkdir "${CHROOT_DIR}"
debootstrap --foreign --no-check-gpg --include=fakeroot,build-essential \ debootstrap --foreign --no-check-gpg --include=fakeroot,build-essential \
--arch="${ARCH}" "${DEBIAN}" "${CHROOT_DIR}" "${DEBIAN_MIRROR}" --arch="${ARCH}" "${DEBIAN}" "${CHROOT_DIR}" "${DEBIAN_MIRROR}"
setup_binfmts
cp ${QEMU_PATH}/bin/qemu-${QEMU_ARCH} "${CHROOT_DIR}"/usr/bin/qemu-${QEMU_ARCH}-static cp ${QEMU_PATH}/bin/qemu-${QEMU_ARCH} "${CHROOT_DIR}"/usr/bin/qemu-${QEMU_ARCH}-static
chroot "${CHROOT_DIR}" ./debootstrap/debootstrap --second-stage chroot "${CHROOT_DIR}" ./debootstrap/debootstrap --second-stage
sbuild-createchroot --arch="${ARCH}" --foreign --setup-only \ sbuild-createchroot --arch="${ARCH}" --foreign --setup-only \
@ -176,17 +183,54 @@ function setup_chroot {
# Call ourselves again which will cause tests to run # Call ourselves again which will cause tests to run
echo "Recursively calling script" echo "Recursively calling script"
if [ ${MAKE_TARGET} = "cs" ]; then if [ ${MAKE_TARGET} = "cs" ]; then
chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR} --enable-cs" chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --maxload ${MAXLOAD} --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR} --enable-cs"
else else
chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR}" chroot "${CHROOT_DIR}" bash -c "cd ${BUILD_DIR} && ./.gitlab/build-test.sh --maxload ${MAXLOAD} --jobs ${JOBS} --with-arch ${ARCH} --with-project-path ${BUILD_DIR}"
fi fi
} }
function setup_binfmts {
case ${QEMU_ARCH} in
"x86_64")
# nothing to be done
;;
"aarch64")
update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff"
;;
"arm")
update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff"
;;
"i386")
# nothing to be done
;;
"mips")
update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff"
;;
"mipsel")
update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff"
;;
"mips64el")
update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08" --mask "\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff"
;;
"s390x")
update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16" --mask "\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff"
;;
"ppc64el")
update-binfmts --install qemu-${QEMU_ARCH} /usr/bin/qemu-${QEMU_ARCH}-static --magic "\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00" --mask "\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00"
;;
*)
echo "Unknown QEMU architecture ${QEMU_ARCH}"
exit 1
;;
esac
}
# Information about environment # Information about environment
echo "Environment information" echo "Environment information"
echo "=======================" echo "======================="
echo " Machine : $(uname -m)" echo " Machine : $(uname -m)"
echo " Jobs : ${JOBS}" echo " Jobs : ${JOBS}"
echo " Max Load : ${MAXLOAD}"
echo " Target Arch : ${ARCH}" echo " Target Arch : ${ARCH}"
echo " chroot Path : ${CHROOT_DIR}" echo " chroot Path : ${CHROOT_DIR}"
echo " Build Path : ${BUILD_DIR}" echo " Build Path : ${BUILD_DIR}"
@ -211,7 +255,7 @@ fi
echo "Compiling" echo "Compiling"
echo "Environment: $(uname -a)" echo "Environment: $(uname -a)"
annotate-output make CPUS=${JOBS} \ annotate-output make --load-average=${MAXLOAD} CPUS=${JOBS} \
PKGS="racket-test db-test unstable-flonum-lib net-test" \ PKGS="racket-test db-test unstable-flonum-lib net-test" \
CONFIGURE_ARGS_qq="${RACKET_CONFIGURE_ARGS}" \ CONFIGURE_ARGS_qq="${RACKET_CONFIGURE_ARGS}" \
${MAKE_TARGET} ${MAKE_TARGET}