From 029073148bd9d03fa848fbbfc5a7ede5f82510c3 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 19 Nov 2024 09:44:50 +0100 Subject: [PATCH 01/33] write system information as well --- jenkins/helper/tools/killall.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/jenkins/helper/tools/killall.py b/jenkins/helper/tools/killall.py index acfc43bf..24950e8c 100644 --- a/jenkins/helper/tools/killall.py +++ b/jenkins/helper/tools/killall.py @@ -86,6 +86,7 @@ def get_all_processes_stats_json(load): 'diskio': psutil.disk_io_counters(perdisk=True, nowrap=True), 'netio': psutil.net_io_counters(pernic=True, nowrap=True), } +<<<<<<< HEAD processes = psutil.process_iter() for process in processes: name = "" @@ -98,4 +99,24 @@ def get_all_processes_stats_json(load): pass except Exception as ex: print(f"while inspecting {name}: {ex} ") +======= + for n in [True, False]: + processes = psutil.process_iter() + for process in processes: + name = "" + try: + name = process.name() + if process.pid not in [1, 2] and process.ppid() != 2: + procstat = gather_process_thread_statistics(process) + if n: + process_full_list[f"p{process.pid}"] = procstat + else: + add_delta(process_full_list[f"p{process.pid}"], procstat) + except psutil.AccessDenied: + pass + except Exception as ex: + print(f"while inspecting {name}: {ex} ") + if n: + time.sleep(1) +>>>>>>> 9bac7477 (write system information as well) return json.dumps(process_full_list) From 91da7c4fb5071be566f50cd1088106efb5154cc9 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 25 Apr 2025 14:22:50 +0200 Subject: [PATCH 02/33] attempt adding coverage builds --- jenkins/runRTAtest.fish | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jenkins/runRTAtest.fish b/jenkins/runRTAtest.fish index 28543d48..2d7b282a 100755 --- a/jenkins/runRTAtest.fish +++ b/jenkins/runRTAtest.fish @@ -5,6 +5,7 @@ source jenkins/helper/jenkins.fish cleanPrepareLockUpdateClear2 and TT_init and set -xg RTA_EDITION "C,Cr2" +and maintainerOn and eval $EDITION and setAllLogsToWorkspace and switchBranches $ARANGODB_BRANCH $ENTERPRISE_BRANCH true @@ -14,6 +15,11 @@ if test "$ASAN" = "true" sanOn and buildSanFlags "$WORKDIR/work/ArangoDB" end +if test "$COVERAGE" = "true" + echo "Coverage build" + coverageOn + and buildSanFlags "$WORKDIR/work/ArangoDB" +end if test "$BUILD_MODE" = "debug" echo "switching to debug build" debugMode @@ -45,6 +51,10 @@ for f in $matches echo $f | grep -qv testreport ; and echo "mv $f $WORKSPACE" ; and mv $f $WORKSPACE; or echo "skipping $f" end +if test "$COVERAGE" = "true" + collectCoverage +end + unlockDirectory exit $s From 6176fba1818d56ec361ba43f7fea85dac6d3f169 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 5 Feb 2025 10:35:58 +0100 Subject: [PATCH 03/33] revert --- jenkins/helper/tools/killall.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/jenkins/helper/tools/killall.py b/jenkins/helper/tools/killall.py index 24950e8c..acfc43bf 100644 --- a/jenkins/helper/tools/killall.py +++ b/jenkins/helper/tools/killall.py @@ -86,7 +86,6 @@ def get_all_processes_stats_json(load): 'diskio': psutil.disk_io_counters(perdisk=True, nowrap=True), 'netio': psutil.net_io_counters(pernic=True, nowrap=True), } -<<<<<<< HEAD processes = psutil.process_iter() for process in processes: name = "" @@ -99,24 +98,4 @@ def get_all_processes_stats_json(load): pass except Exception as ex: print(f"while inspecting {name}: {ex} ") -======= - for n in [True, False]: - processes = psutil.process_iter() - for process in processes: - name = "" - try: - name = process.name() - if process.pid not in [1, 2] and process.ppid() != 2: - procstat = gather_process_thread_statistics(process) - if n: - process_full_list[f"p{process.pid}"] = procstat - else: - add_delta(process_full_list[f"p{process.pid}"], procstat) - except psutil.AccessDenied: - pass - except Exception as ex: - print(f"while inspecting {name}: {ex} ") - if n: - time.sleep(1) ->>>>>>> 9bac7477 (write system information as well) return json.dumps(process_full_list) From 7df62b38712dc6e63c6289aa96aeeb80fee0664b Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 11 Mar 2025 15:39:46 +0100 Subject: [PATCH 04/33] work on using podman instead of docker --- helper.fish | 10 + helper.linux.fish | 231 ++++++++++--------- jenkins/docker/starter.sh | 4 +- jenkins/forTestDocker.fish | 12 +- jenkins/forTestDockerCommunity.fish | 6 +- jenkins/forTestDockerEnterprise.fish | 6 +- jenkins/helper/runAnyPerformanceResults.fish | 2 +- jenkins/helper/server.inc | 14 +- jenkins/runGobenchPerformance.fish | 4 +- jenkins/runGobenchPerformanceResults.fish | 2 +- jenkins/runSimplePerformance.fish | 6 +- 11 files changed, 155 insertions(+), 142 deletions(-) diff --git a/helper.fish b/helper.fish index 41552c6c..75e1c68c 100755 --- a/helper.fish +++ b/helper.fish @@ -1,6 +1,15 @@ set -gx KEYNAME 86FEC04D set -gx KEYNAME_OLD 115E1684 +if test -f /usr/bin/podman + echo podman + set -xg DOCKER "podman" + set -xg DOCKER_URL_PREFIX "docker.io/" +else + set -xg DOCKER "docker" +end + + function lockDirectory # Now grab the lock ourselves: set -l pid (echo %self) @@ -2278,6 +2287,7 @@ function findArangoDBVersion echo "ARANGODB_SNIPPETS: $ARANGODB_SNIPPETS" echo "ARANGODB_TGZ_UPSTREAM: $ARANGODB_TGZ_UPSTREAM" echo "DOCKER_TAG: $DOCKER_TAG" + echo "DOCKER: $DOCKER" echo '------------------------------------------------------------------------------' echo end diff --git a/helper.linux.fish b/helper.linux.fish index 6a7f0784..7fdebdd6 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -21,6 +21,7 @@ else set -xg UBUNTUBUILDIMAGE_TAG_ARCH "x86_64" end + set -gx UBUNTUBUILDIMAGE_312_NAME arangodb/ubuntubuildarangodb-devel set -gx UBUNTUBUILDIMAGE_312_TAG 15 set -gx UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:$UBUNTUBUILDIMAGE_312_TAG-$UBUNTUBUILDIMAGE_TAG_ARCH @@ -226,29 +227,29 @@ function checkoutMirror exit 1 end - runInContainer -v $argv[1]:/mirror $ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutMirror.fish + runInContainer -v $argv[1]:/mirror $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutMirror.fish or return $status end function checkoutArangoDB - runInContainer $ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutArangoDB.fish + runInContainer $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutArangoDB.fish or return $status community end function checkoutEnterprise - runInContainer $ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutEnterprise.fish + runInContainer $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutEnterprise.fish or return $status enterprise end function checkoutMiniChaos - runInContainer $ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutMiniChaos.fish + runInContainer $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutMiniChaos.fish or return $status end function checkoutRTA - runInContainer -e RTA_BRANCH="$RTA_BRANCH" $ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutRTA.fish + runInContainer -e RTA_BRANCH="$RTA_BRANCH" $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/checkoutRTA.fish or return $status end @@ -267,7 +268,7 @@ function switchBranches end checkoutIfNeeded - and runInContainer $ALPINEUTILSIMAGE $SCRIPTSDIR/switchBranches.fish $argv + and runInContainer $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/switchBranches.fish $argv and convertSItoJSON and findArangoDBVersion and findRequiredCompiler @@ -290,20 +291,20 @@ set -gx LDAPHOST "$LDAPDOCKERCONTAINERNAME$LDAPEXT" set -gx LDAPHOST2 "$LDAP2DOCKERCONTAINERNAME$LDAPEXT" function stopLdapServer - docker stop "$LDAPDOCKERCONTAINERNAME$LDAPEXT" - and docker rm "$LDAPDOCKERCONTAINERNAME$LDAPEXT" - docker stop "$LDAP2DOCKERCONTAINERNAME$LDAPEXT" - and docker rm "$LDAP2DOCKERCONTAINERNAME$LDAPEXT" - docker network rm "$LDAPNETWORK$LDAPEXT" + "$DOCKER" stop "$LDAPDOCKERCONTAINERNAME$LDAPEXT" + and "$DOCKER" rm "$LDAPDOCKERCONTAINERNAME$LDAPEXT" + "$DOCKER" stop "$LDAP2DOCKERCONTAINERNAME$LDAPEXT" + and "$DOCKER" rm "$LDAP2DOCKERCONTAINERNAME$LDAPEXT" + "$DOCKER" network rm "$LDAPNETWORK$LDAPEXT" echo "LDAP servers stopped" true end function launchLdapServer stopLdapServer - and docker network create "$LDAPNETWORK$LDAPEXT" - and docker run -d --name "$LDAPHOST" --net="$LDAPNETWORK$LDAPEXT" $LDAPIMAGE - and docker run -d --name "$LDAPHOST2" --net="$LDAPNETWORK$LDAPEXT" $LDAPIMAGE + and "$DOCKER" network create "$LDAPNETWORK$LDAPEXT" + and "$DOCKER" run -d --name "$LDAPHOST" --net="$LDAPNETWORK$LDAPEXT" $LDAPIMAGE + and "$DOCKER" run -d --name "$LDAPHOST2" --net="$LDAPNETWORK$LDAPEXT" $LDAPIMAGE and echo "LDAP servers launched" end @@ -321,7 +322,7 @@ function buildArangoDB and findDefaultArchitecture and findUseARM and set -xg STATIC_EXECUTABLES Off - and runInContainer (findBuildImage) $SCRIPTSDIR/(findBuildScript) $argv + and runInContainer $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/(findBuildScript) $argv and packBuildFiles set -l s $status if test $s -ne 0 @@ -337,7 +338,7 @@ function makeArangoDB and findDefaultArchitecture and findUseARM end - and runInContainer (findBuildImage) $SCRIPTSDIR/makeArangoDB.fish $argv + and runInContainer $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/makeArangoDB.fish $argv and packBuildFiles set -l s $status if test $s -ne 0 @@ -358,7 +359,7 @@ function buildStaticArangoDB unpackBuildFiles "$BUILD_FILES_ARCHIVE" else echo "UNPACK_BUILD_FILES: $UNPACK_BUILD_FILES" - runInContainer (findStaticBuildImage) $SCRIPTSDIR/(findStaticBuildScript) $argv + runInContainer $DOCKER_URL_PREFIX(findStaticBuildImage) $SCRIPTSDIR/(findStaticBuildScript) $argv and packBuildFiles and if test "$ENTERPRISEEDITION" = "On"; and test "$ARANGODB_VERSION_MAJOR" -eq 3 if test "$ARANGODB_VERSION_MINOR" -ge 12; or begin; test "$ARANGODB_VERSION_MINOR" -eq 11; and test "$ARANGODB_VERSION_PATCH" -ge 10; end @@ -380,7 +381,7 @@ function makeStaticArangoDB and findDefaultArchitecture and findUseARM end - and runInContainer (findStaticBuildImage) $SCRIPTSDIR/makeAlpine.fish $argv + and runInContainer $DOCKER_URL_PREFIX(findStaticBuildImage) $SCRIPTSDIR/makeAlpine.fish $argv and packBuildFiles set -l s $status if test $s -ne 0 @@ -399,7 +400,7 @@ function buildExamples and if test "$NO_RM_BUILD" != 1 buildStaticArangoDB end - and runInContainer (findStaticBuildImage) $SCRIPTSDIR/buildExamples.fish $argv + and runInContainer $DOCKER_URL_PREFIX(findStaticBuildImage) $SCRIPTSDIR/buildExamples.fish $argv set -l s $status if test $s -ne 0 echo Build error! @@ -420,11 +421,11 @@ function oskar and if test "$SAN" = "On" parallelism 2 clearSanStatus - runInContainer --security-opt seccomp=unconfined --cap-add SYS_NICE --cap-add SYS_PTRACE (findBuildImage) $SCRIPTSDIR/runTests.fish $argv + runInContainer --security-opt seccomp=unconfined --cap-add SYS_NICE --cap-add SYS_PTRACE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runTests.fish $argv set s $status set s (math $s + (getSanStatus)) else - runInContainer --security-opt seccomp=unconfined --cap-add SYS_NICE (findBuildImage) $SCRIPTSDIR/runTests.fish $argv + runInContainer --security-opt seccomp=unconfined --cap-add SYS_NICE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runTests.fish $argv set s $status end @@ -441,11 +442,11 @@ function rlogTests and if test "$SAN" = "On" parallelism 2 clearSanStatus - runInContainer --cap-add SYS_NICE --cap-add SYS_PTRACE (findBuildImage) $SCRIPTSDIR/rlog/pr.fish $argv + runInContainer --cap-add SYS_NICE --cap-add SYS_PTRACE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/rlog/pr.fish $argv set s $status set s (math $s + (getSanStatus)) else - runInContainer --cap-add SYS_NICE (findBuildImage) $SCRIPTSDIR/rlog/pr.fish $argv + runInContainer --cap-add SYS_NICE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/rlog/pr.fish $argv set s $status end @@ -463,17 +464,17 @@ function oskarFull launchLdapServer and if test "$SAN" = "On" parallelism 2 - runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE --cap-add SYS_PTRACE (findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv + runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE --cap-add SYS_PTRACE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv else - runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE (findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv + runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv end set s $status else if test "$SAN" = "On" parallelism 2 - runInContainer --cap-add SYS_NICE --cap-add SYS_PTRACE (findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv + runInContainer --cap-add SYS_NICE --cap-add SYS_PTRACE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv else - runInContainer --cap-add SYS_NICE (findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv + runInContainer --cap-add SYS_NICE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runFullTests.fish $argv end end set s $status @@ -496,17 +497,17 @@ function oskarOneTest launchLdapServer and if test "$SAN" = "On" parallelism 2 - runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE --cap-add SYS_PTRACE (findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv + runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE --cap-add SYS_PTRACE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv else - runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE (findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv + runInContainer --net="$LDAPNETWORK$LDAPEXT" --cap-add SYS_NICE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv end set s $status else if test "$SAN" = "On" parallelism 2 - runInContainer --cap-add SYS_NICE --cap-add SYS_PTRACE (findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv + runInContainer --cap-add SYS_NICE --cap-add SYS_PTRACE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv else - runInContainer --cap-add SYS_NICE (findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv + runInContainer --cap-add SYS_NICE $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/runOneTest.fish $argv end end set s $status @@ -543,7 +544,7 @@ function jslint set -l s 0 findArangoDBVersion - and runInContainer arangodb/arangodb:$ARANGODB_VERSION_MAJOR.$ARANGODB_VERSION_MINOR /scripts/jslint.sh + and runInContainer "$DOCKER_URL_PREFIX"arangodb/arangodb:$ARANGODB_VERSION_MAJOR.$ARANGODB_VERSION_MINOR /scripts/jslint.sh set s $status popd @@ -557,7 +558,7 @@ end function cppcheckArangoDB checkoutIfNeeded - runInContainer $CPPCHECKIMAGE /scripts/cppcheck.sh $argv + runInContainer $DOCKER_URL_PREFIX$CPPCHECKIMAGE /scripts/cppcheck.sh $argv return $status end @@ -598,7 +599,7 @@ function collectCoverage findRequiredCompiler and findRequiredOpenSSL and echo "collecting llvm coverage" - and runInContainer --env LLVM_PROFILE_FILE=/work/gcov/ (findStaticBuildImage) python3 -u "$WORKSPACE/jenkins/helper/aggregate_coverage.py" "$INNERWORKDIR/" gcov coverage + and runInContainer --env LLVM_PROFILE_FILE=/work/gcov/ $DOCKER_URL_PREFIX(findStaticBuildImage) python3 -u "$WORKSPACE/jenkins/helper/aggregate_coverage.py" "$INNERWORKDIR/" gcov coverage return $status end @@ -613,7 +614,7 @@ function signSourcePackage and runInContainer \ -e ARANGO_SIGN_PASSWD="$ARANGO_SIGN_PASSWD" \ -v $WORKSPACE/signing-keys/.gnupg4:/root/.gnupg \ - (findBuildImage) $SCRIPTSDIR/signFile.fish \ + $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/signFile.fish \ /work/ArangoDB-$SOURCE_TAG.tar.gz \ /work/ArangoDB-$SOURCE_TAG.tar.bz2 \ /work/ArangoDB-$SOURCE_TAG.zip @@ -625,7 +626,7 @@ function createCompleteTar set -l RELEASE_TAG $argv[1] pushd $WORKDIR/work and runInContainer \ - $ALPINEUTILSIMAGE $SCRIPTSDIR/createCompleteTar.fish \ + $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/createCompleteTar.fish \ $RELEASE_TAG and popd or begin ; popd ; return 1 ; end @@ -775,7 +776,7 @@ function buildDebianPackage and echo -n " -- ArangoDB " >> $ch and date -R >> $ch and sed -i "s/@ARCHITECTURE@/$ARCH/g" $TARGET/control - and runInContainer $UBUNTUPACKAGINGIMAGE $SCRIPTSDIR/buildDebianPackage.fish + and runInContainer $DOCKER_URL_PREFIX$UBUNTUPACKAGINGIMAGE $SCRIPTSDIR/buildDebianPackage.fish set -l s $status if test $s -ne 0 echo Error when building a debian package @@ -810,7 +811,7 @@ function buildRPMPackage and cp $WORKDIR/rpm/$pd/arangodb3.initd $WORKDIR/work and cp $WORKDIR/rpm/$pd/arangodb3.service $WORKDIR/work and cp $WORKDIR/rpm/$pd/arangodb3.logrotate $WORKDIR/work - and runInContainer $CENTOSPACKAGINGIMAGE $SCRIPTSDIR/buildRPMPackage.fish + and runInContainer $DOCKER_URL_PREFIX$CENTOSPACKAGINGIMAGE $SCRIPTSDIR/buildRPMPackage.fish end ## ############################################################################# @@ -840,7 +841,7 @@ function runMiniChaos -v $WORKDIR/work/ArangoDB/mini-chaos:/mini-chaos \ -v $WORKDIR/work/mini-chaos/$package:/$package \ -e ARANGODB_OVERRIDE_CRASH_HANDLER=0 \ - (findBuildImage) $SCRIPTSDIR/startMiniChaos.fish $package $duration + $DOCKER_URL_PREFIX(findBuildImage) $SCRIPTSDIR/startMiniChaos.fish $package $duration end @@ -1245,12 +1246,12 @@ function buildDockerAny end and buildDockerImage $IMAGE_NAME1 and if test "$IMAGE_NAME1" != "$IMAGE_NAME2" - docker tag $IMAGE_NAME1 $IMAGE_NAME2 + "$DOCKER" tag $IMAGE_NAME1 $IMAGE_NAME2 end and validateDockerImageIfNeeded $IMAGE_NAME2 and pushDockerImage $IMAGE_NAME2 and if test "$GCR_REG" = "On" - docker tag $IMAGE_NAME1 $GCR_REG_PREFIX$IMAGE_NAME2 + "$DOCKER" tag $IMAGE_NAME1 $GCR_REG_PREFIX$IMAGE_NAME2 and pushDockerImage $GCR_REG_PREFIX$IMAGE_NAME2 end and if test "$ENTERPRISEEDITION" = "On" @@ -1259,10 +1260,10 @@ function buildDockerAny echo $IMAGE_NAME1 > $WORKDIR/work/arangodb3.docker end and if test "$IMAGE_NAME3" != "" - docker tag $IMAGE_NAME1 $IMAGE_NAME3 + "$DOCKER" tag $IMAGE_NAME1 $IMAGE_NAME3 and pushDockerImage $IMAGE_NAME3 and if test "$GCR_REG" = "On" - docker tag $IMAGE_NAME3 $GCR_REG_PREFIX$IMAGE_NAME3 + "$DOCKER" tag $IMAGE_NAME3 $GCR_REG_PREFIX$IMAGE_NAME3 and pushDockerImage $GCR_REG_PREFIX$IMAGE_NAME3 end end @@ -1390,7 +1391,7 @@ function buildDockerImage popd pushd $containerpath - and eval "docker build $BUILD_ARGS --pull --no-cache -t $imagename ." + and eval "$DOCKER build $BUILD_ARGS --pull --no-cache -t $DOCKER_URL_PREFIX$imagename ." or begin ; popd ; return 1 ; end popd end @@ -1403,12 +1404,12 @@ function pushDockerImage set -l imagename $argv[1] - if test (docker images -q $imagename 2> /dev/null) = "" + if test ("$DOCKER" images -q $DOCKER_URL_PREFIX$imagename 2> /dev/null) = "" echo Given image is not present locally return 1 end - docker push $imagename + "$DOCKER" push $DOCKER_URL_PREFIX$imagename end function pushDockerManifest @@ -1419,14 +1420,14 @@ function pushDockerManifest set manifestname $argv[1] - docker manifest inspect $manifestname - and docker manifest rm $manifestname + "$DOCKER" manifest inspect $manifestname + and "$DOCKER" manifest rm $manifestname - docker manifest create \ + "$DOCKER" manifest create \ $manifestname \ --amend $manifestname-amd64 \ --amend $manifestname-arm64v8 - and docker manifest push --purge $manifestname + and "$DOCKER" manifest push --purge $DOCKER_URL_PREFIX$manifestname and return 0 or return 1 end @@ -1462,7 +1463,7 @@ function buildDockerLocal pushd $containerpath set -l tag (date +%Y%m%d%H%M%S) - and eval "docker build -t $imagename --pull . 2>&1" + and eval "$DOCKER build -t $imagename --pull . 2>&1" or begin ; popd ; return 1 ; end popd end @@ -1506,9 +1507,9 @@ function buildUbuntuBuildImage311 and cd $WORKDIR/containers/buildUbuntu311.docker and switch "$ARCH" case "x86_64" - eval "docker build $IMAGE_ARGS --pull -t $UBUNTUBUILDIMAGE_311 -f ./Dockerfile.x86-64 ." + eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_311 -f ./Dockerfile.x86-64 ." case "aarch64" - eval "docker build $IMAGE_ARGS --pull -t $UBUNTUBUILDIMAGE_311 -f ./Dockerfile.arm64 ." + eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_311 -f ./Dockerfile.arm64 ." case '*' echo "fatal, unknown architecture $ARCH to build $UBUNTUBUILDIMAGE_311" exit 1 @@ -1518,21 +1519,21 @@ function buildUbuntuBuildImage311 end function pushUbuntuBuildImage311 - docker tag $UBUNTUBUILDIMAGE_311 $UBUNTUBUILDIMAGE_311_NAME:latest-$ARCH - and docker push $UBUNTUBUILDIMAGE_311 - and docker push $UBUNTUBUILDIMAGE_311_NAME:latest-$ARCH + "$DOCKER" tag $UBUNTUBUILDIMAGE_311 $UBUNTUBUILDIMAGE_311_NAME:latest-$ARCH + and "$DOCKER" push $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_311 + and "$DOCKER" push $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_311_NAME:latest-$ARCH end -function pullUbuntuBuildImage311 ; docker pull $UBUNTUBUILDIMAGE_311 ; end +function pullUbuntuBuildImage311 ; "$DOCKER" pull $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_311 ; end function buildUbuntuBuildImageDevel pushd $WORKDIR and cd $WORKDIR/containers/buildUbuntuDevel.docker and switch "$ARCH" case "x86_64" - eval "docker build $IMAGE_ARGS --pull -t $UBUNTUBUILDIMAGE_312 -f ./Dockerfile.x86-64 ." + eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_312 -f ./Dockerfile.x86-64 ." case "aarch64" - eval "docker build $IMAGE_ARGS --pull -t $UBUNTUBUILDIMAGE_312 -f ./Dockerfile.arm64 ." + eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_312 -f ./Dockerfile.arm64 ." case '*' echo "fatal, unknown architecture $ARCH to build $UBUNTUBUILDIMAGE_312" exit 1 @@ -1542,101 +1543,101 @@ function buildUbuntuBuildImageDevel end function pushUbuntuBuildImageDevel - docker tag $UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:latest-$ARCH - and docker push $UBUNTUBUILDIMAGE_312 - and docker push $UBUNTUBUILDIMAGE_312_NAME:latest-$ARCH + "$DOCKER" tag $UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:latest-$ARCH + and "$DOCKER" push $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_312 + and "$DOCKER" push $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_312_NAME:latest-$ARCH end -function pullUbuntuBuildImageDevel ; docker pull $UBUNTUBUILDIMAGE_312 ; end +function pullUbuntuBuildImageDevel ; "$DOCKER" pull $DOCKER_URL_PREFIX$UBUNTUBUILDIMAGE_312 ; end function buildUbuntuPackagingImage pushd $WORKDIR and cp -a scripts/buildDebianPackage.fish containers/buildUbuntuPackaging.docker/scripts and cd $WORKDIR/containers/buildUbuntuPackaging.docker - and eval "docker build $IMAGE_ARGS --pull -t $UBUNTUPACKAGINGIMAGE ." + and eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$UBUNTUPACKAGINGIMAGE ." and rm -f $WORKDIR/containers/buildUbuntuPackaging.docker/scripts/*.fish or begin ; popd ; return 1 ; end popd end -function pushUbuntuPackagingImage ; docker push $UBUNTUPACKAGINGIMAGE ; end +function pushUbuntuPackagingImage ; "$DOCKER" push $DOCKER_URL_PREFIX$UBUNTUPACKAGINGIMAGE ; end -function pullUbuntuPackagingImage ; docker pull $UBUNTUPACKAGINGIMAGE ; end +function pullUbuntuPackagingImage ; "$DOCKER" pull $DOCKER_URL_PREFIX$UBUNTUPACKAGINGIMAGE ; end function buildUbuntuPackagingImage2 pushd $WORKDIR and cp -a scripts/buildDebianPackage.fish containers/buildUbuntuPackaging2.docker/scripts and cd $WORKDIR/containers/buildUbuntuPackaging2.docker - and eval "docker build $IMAGE_ARGS --pull -t $UBUNTUPACKAGINGIMAGE2 ." + and eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$UBUNTUPACKAGINGIMAGE2 ." and rm -f $WORKDIR/containers/buildUbuntuPackaging2.docker/scripts/*.fish or begin ; popd ; return 1 ; end popd end -function pushUbuntuPackagingImage2 ; docker push $UBUNTUPACKAGINGIMAGE2 ; end +function pushUbuntuPackagingImage2 ; "$DOCKER" push $DOCKER_URL_PREFIX$UBUNTUPACKAGINGIMAGE2 ; end -function pullUbuntuPackagingImage2 ; docker pull $UBUNTUPACKAGINGIMAGE2 ; end +function pullUbuntuPackagingImage2 ; "$DOCKER" pull $DOCKER_URL_PREFIX$UBUNTUPACKAGINGIMAGE2 ; end function buildAlpineUtilsImage pushd $WORKDIR and cp -a scripts/{checkoutArangoDB,checkoutEnterprise,clearWorkDir,downloadStarter,downloadSyncer,downloadRclone,runTests,runFullTests,switchBranches,recursiveChown}.fish containers/buildUtils.docker/scripts and cd $WORKDIR/containers/buildUtils.docker - and eval "docker build $IMAGE_ARGS --pull -t $ALPINEUTILSIMAGE ." + and eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE ." or begin ; popd ; return 1 ; end popd end function pushAlpineUtilsImage - docker tag $ALPINEUTILSIMAGE $ALPINEUTILSIMAGE_NAME:latest - and docker push $ALPINEUTILSIMAGE - and docker push $ALPINEUTILSIMAGE_NAME:latest + "$DOCKER" tag $ALPINEUTILSIMAGE $ALPINEUTILSIMAGE_NAME:latest + and "$DOCKER" push $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE + and "$DOCKER" push $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE_NAME:latest end -function pullAlpineUtilsImage ; docker pull $ALPINEUTILSIMAGE ; end +function pullAlpineUtilsImage ; "$DOCKER" pull $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE ; end function buildCentosPackagingImage pushd $WORKDIR and cp -a scripts/buildRPMPackage.fish containers/buildCentos7Packaging.docker/scripts and cd $WORKDIR/containers/buildCentos7Packaging.docker - and eval "docker build $IMAGE_ARGS --pull -t $CENTOSPACKAGINGIMAGE ." + and eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$CENTOSPACKAGINGIMAGE ." and rm -f $WORKDIR/containers/buildCentos7Packaging.docker/scripts/*.fish or begin ; popd ; return 1 ; end popd end function pushCentosPackagingImage - docker tag $CENTOSPACKAGINGIMAGE $CENTOSPACKAGINGIMAGE_NAME:latest - and docker push $CENTOSPACKAGINGIMAGE - and docker push $CENTOSPACKAGINGIMAGE_NAME:latest + "$DOCKER" tag $CENTOSPACKAGINGIMAGE $CENTOSPACKAGINGIMAGE_NAME:latest + and "$DOCKER" push $DOCKER_URL_PREFIX$CENTOSPACKAGINGIMAGE + and "$DOCKER" push $DOCKER_URL_PREFIX$CENTOSPACKAGINGIMAGE_NAME:latest end -function pullCentosPackagingImage ; docker pull $CENTOSPACKAGINGIMAGE ; end +function pullCentosPackagingImage ; "$DOCKER" pull $DOCKER_URL_PREFIX$CENTOSPACKAGINGIMAGE ; end function buildCppcheckImage pushd $WORKDIR/containers/cppcheck.docker - and eval "docker build $IMAGE_ARGS --pull -t $CPPCHECKIMAGE ." + and eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$CPPCHECKIMAGE ." or begin ; popd ; return 1 ; end popd end function pushCppcheckImage - docker tag $CPPCHECKIMAGE $CPPCHECKIMAGE_NAME:latest - and docker push $CPPCHECKIMAGE - and docker push $CPPCHECKIMAGE_NAME:latest + "$DOCKER" tag $CPPCHECKIMAGE $CPPCHECKIMAGE_NAME:latest + and "$DOCKER" push $DOCKER_URL_PREFIX$CPPCHECKIMAGE + and "$DOCKER" push $DOCKER_URL_PREFIX$CPPCHECKIMAGE_NAME:latest end -function pullCppcheckImage ; docker pull $CPPCHECKIMAGE ; end +function pullCppcheckImage ; "$DOCKER" pull $DOCKER_URL_PREFIX$CPPCHECKIMAGE ; end function buildLdapImage pushd $WORKDIR/containers/ldap.docker - and eval "docker build $IMAGE_ARGS --pull -t $LDAPIMAGE ." + and eval "$DOCKER build $IMAGE_ARGS --pull -t $DOCKER_URL_PREFIX$LDAPIMAGE ." or begin ; popd ; return 1 ; end popd end function pushLdapImage - docker tag $LDAPIMAGE $LDAPIMAGE_NAME:latest - and docker push $LDAPIMAGE - and docker push $LDAPIMAGE_NAME:latest + "$DOCKER" tag $LDAPIMAGE $LDAPIMAGE_NAME:latest + and "$DOCKER" push $DOCKER_URL_PREFIX$LDAPIMAGE + and "$DOCKER" push $DOCKER_URL_PREFIX$LDAPIMAGE_NAME:latest end -function pullLdapImage ; docker pull $LDAPIMAGE ; end +function pullLdapImage ; "$DOCKER" pull $DOCKER_URL_PREFIX$LDAPIMAGE ; end function remakeImages set -l s 0 @@ -1698,7 +1699,7 @@ function runInContainer # from a regular user. Therefore we have to do some Eiertanz to stop it # if we receive a TERM outside the container. Note that this does not # cover SIGINT, since this will directly abort the whole function. - set c (docker run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ + set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ @@ -1793,24 +1794,26 @@ function runInContainer $argv) function termhandler --on-signal TERM --inherit-variable c if test -n "$c" - docker stop $c >/dev/null - docker rm $c >/dev/null + "$DOCKER" stop $c >/dev/null + "$DOCKER" rm $c >/dev/null end end - docker logs -f $c # print output to stdout - docker stop $c >/dev/null # happens when the previous command gets a SIGTERM - set s (docker inspect $c --format "{{.State.ExitCode}}") - docker rm $c >/dev/null + "$DOCKER" logs -f $c # print output to stdout + "$DOCKER" stop $c >/dev/null # happens when the previous command gets a SIGTERM + set s ("$DOCKER" inspect $c --format "{{.State.ExitCode}}") + "$DOCKER" rm $c >/dev/null functions -e termhandler # Cleanup ownership: - docker run \ - -v $WORKDIR/work:$INNERWORKDIR \ - -e UID=(id -u) \ - -e GID=(id -g) \ - -e INNERWORKDIR=$INNERWORKDIR \ - --rm \ - $ALPINEUTILSIMAGE $SCRIPTSDIR/recursiveChown.fish - + if test "$DOCKER" = "docker" + "$DOCKER" run \ + -v $WORKDIR/work:$INNERWORKDIR \ + -v "$WORKDIR/scripts/":"/scripts" \ + -e UID=(id -u) \ + -e GID=(id -g) \ + -e INNERWORKDIR=$INNERWORKDIR \ + --rm \ + $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/recursiveChown.fish + end if test -n "$agentstarted" ssh-agent -k > /dev/null set -e SSH_AUTH_SOCK @@ -1833,7 +1836,7 @@ function interactiveContainer set -l agentstarted "" end - docker run -it --rm --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ + "$DOCKER" run -it --rm --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ @@ -1944,7 +1947,7 @@ function buildRclone rm -rf rclone/$argv[1] mkdir rclone/$argv[1] - docker run \ + "$DOCKER" run \ -v (pwd)/scripts:/scripts \ -v (pwd)/rclone/$argv[1]:/data \ -e RCLONE_VERSION=$argv[1] \ @@ -1967,7 +1970,7 @@ function findOpenSSLVersion end function clearWorkDir - runInContainer $ALPINEUTILSIMAGE $SCRIPTSDIR/clearWorkDir.fish + runInContainer $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/clearWorkDir.fish end function transformSpec @@ -2059,13 +2062,13 @@ function updateDockerBuildImage checkoutIfNeeded and findRequiredCompiler and findRequiredOpenSSL - and docker pull (findBuildImage) - and docker pull (findStaticBuildImage) + and "$DOCKER" pull $DOCKER_URL_PREFIX(findBuildImage) + and "$DOCKER" pull $DOCKER_URL_PREFIX(findStaticBuildImage) end function downloadStarter mkdir -p $WORKDIR/work/$THIRDPARTY_BIN - and runInContainer $ALPINEUTILSIMAGE $SCRIPTSDIR/downloadStarter.fish $INNERWORKDIR/$THIRDPARTY_BIN $argv + and runInContainer $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/downloadStarter.fish $INNERWORKDIR/$THIRDPARTY_BIN $argv and convertSItoJSON end @@ -2077,7 +2080,7 @@ function downloadSyncer end mkdir -p $WORKDIR/work/$THIRDPARTY_SBIN and rm -f $WORKDIR/work/ArangoDB/build/install/usr/sbin/arangosync $WORKDIR/work/ArangoDB/build/install/usr/bin/arangosync - and runInContainer -e DOWNLOAD_SYNC_USER=$DOWNLOAD_SYNC_USER $ALPINEUTILSIMAGE $SCRIPTSDIR/downloadSyncer.fish $INNERWORKDIR/$THIRDPARTY_SBIN $argv + and runInContainer -e DOWNLOAD_SYNC_USER=$DOWNLOAD_SYNC_USER $DOCKER_URL_PREFIX$ALPINEUTILSIMAGE $SCRIPTSDIR/downloadSyncer.fish $INNERWORKDIR/$THIRDPARTY_SBIN $argv and ln -s ../sbin/arangosync $WORKDIR/work/ArangoDB/build/install/usr/bin/arangosync and convertSItoJSON end @@ -2104,17 +2107,17 @@ function downloadAuxBinariesToBuildBin end function packObjectFiles - runInContainer (eval echo \$UBUNTUBUILDIMAGE_$ARANGODB_VERSION_MAJOR$ARANGODB_VERSION_MINOR) $SCRIPTSDIR/packObjectFiles.fish + runInContainer $DOCKER_URL_PREFIX(eval echo \$UBUNTUBUILDIMAGE_$ARANGODB_VERSION_MAJOR$ARANGODB_VERSION_MINOR) $SCRIPTSDIR/packObjectFiles.fish end function packBuildFiles if test "$PACK_BUILD_FILES" = "On" - runInContainer (eval echo \$UBUNTUBUILDIMAGE_$ARANGODB_VERSION_MAJOR$ARANGODB_VERSION_MINOR) $SCRIPTSDIR/packBuildFiles.fish + runInContainer $DOCKER_URL_PREFIX(eval echo \$UBUNTUBUILDIMAGE_$ARANGODB_VERSION_MAJOR$ARANGODB_VERSION_MINOR) $SCRIPTSDIR/packBuildFiles.fish end end function unpackBuildFiles - runInContainer (eval echo \$UBUNTUBUILDIMAGE_$ARANGODB_VERSION_MAJOR$ARANGODB_VERSION_MINOR) $SCRIPTSDIR/unpackBuildFiles.fish "$argv[1]" + runInContainer $DOCKER_URL_PREFIX(eval echo \$UBUNTUBUILDIMAGE_$ARANGODB_VERSION_MAJOR$ARANGODB_VERSION_MINOR) $SCRIPTSDIR/unpackBuildFiles.fish "$argv[1]" end function installGrype diff --git a/jenkins/docker/starter.sh b/jenkins/docker/starter.sh index 7134c2d9..8a882002 100755 --- a/jenkins/docker/starter.sh +++ b/jenkins/docker/starter.sh @@ -15,9 +15,9 @@ if test -z "$SSH_AUTH_SOCK"; then agentstarted=1 fi -docker run -v $HOME/.ssh:/root/.ssh arangodb/ssh-client ssh -o StrictHostKeyChecking=no -T git@github.com +"$DOCKER" run -v $HOME/.ssh:/root/.ssh arangodb/ssh-client ssh -o StrictHostKeyChecking=no -T git@github.com -docker run \ +"$DOCKER" run \ -e "PDOCKER=`getent group docker | cut -d: -f3`" \ -e "PGID=`id -g`" \ -e "PHOME=$HOME" \ diff --git a/jenkins/forTestDocker.fish b/jenkins/forTestDocker.fish index 5de5f1d8..497c1d47 100755 --- a/jenkins/forTestDocker.fish +++ b/jenkins/forTestDocker.fish @@ -39,9 +39,9 @@ and setArchSuffix and set -xg HUB_COMMUNITY "arangodb/arangodb-test:$DOCKER_TAG_JENKINS$archSuffix" and buildDockerImage $HUB_COMMUNITY and validateDockerImageIfNeeded $HUB_COMMUNITY -and docker push $HUB_COMMUNITY -and docker tag $HUB_COMMUNITY $GCR_REG_PREFIX$HUB_COMMUNITY -and docker push $GCR_REG_PREFIX$HUB_COMMUNITY +and "$DOCKER" push $HUB_COMMUNITY +and "$DOCKER" tag $HUB_COMMUNITY $GCR_REG_PREFIX$HUB_COMMUNITY +and "$DOCKER" push $GCR_REG_PREFIX$HUB_COMMUNITY and echo $HUB_COMMUNITY >> $WORKSPACE/imagenames.log and echo $GCR_REG_PREFIX$HUB_COMMUNITY >> $WORKSPACE/imagenames.log @@ -61,9 +61,9 @@ and downloadRclone and setArchSuffix and set -xg HUB_ENTERPRISE "arangodb/enterprise-test:$DOCKER_TAG_JENKINS$archSuffix" and buildDockerImage $HUB_ENTERPRISE -and docker push $HUB_ENTERPRISE -and docker tag $HUB_ENTERPRISE $GCR_REG_PREFIX$HUB_ENTERPRISE -and docker push $GCR_REG_PREFIX$HUB_ENTERPRISE +and "$DOCKER" push $HUB_ENTERPRISE +and "$DOCKER" tag $HUB_ENTERPRISE $GCR_REG_PREFIX$HUB_ENTERPRISE +and "$DOCKER" push $GCR_REG_PREFIX$HUB_ENTERPRISE and echo $HUB_ENTERPRISE >> $WORKSPACE/imagenames.log and echo $GCR_REG_PREFIX$HUB_ENTERPRISE >> $WORKSPACE/imagenames.log diff --git a/jenkins/forTestDockerCommunity.fish b/jenkins/forTestDockerCommunity.fish index a3719c81..4cc4fd83 100755 --- a/jenkins/forTestDockerCommunity.fish +++ b/jenkins/forTestDockerCommunity.fish @@ -39,9 +39,9 @@ and setArchSuffix and set -xg HUB_COMMUNITY "arangodb/arangodb-test:$DOCKER_TAG_JENKINS$archSuffix" and buildDockerImage $HUB_COMMUNITY and validateDockerImageIfNeeded $HUB_COMMUNITY -and docker push $HUB_COMMUNITY -and docker tag $HUB_COMMUNITY $GCR_REG_PREFIX$HUB_COMMUNITY -and docker push $GCR_REG_PREFIX$HUB_COMMUNITY +and "$DOCKER" push $HUB_COMMUNITY +and "$DOCKER" tag $HUB_COMMUNITY $GCR_REG_PREFIX$HUB_COMMUNITY +and "$DOCKER" push $GCR_REG_PREFIX$HUB_COMMUNITY and echo $HUB_COMMUNITY >> $WORKSPACE/imagenames.log and echo $GCR_REG_PREFIX$HUB_COMMUNITY >> $WORKSPACE/imagenames.log diff --git a/jenkins/forTestDockerEnterprise.fish b/jenkins/forTestDockerEnterprise.fish index 24927a8d..4542aba9 100755 --- a/jenkins/forTestDockerEnterprise.fish +++ b/jenkins/forTestDockerEnterprise.fish @@ -41,9 +41,9 @@ and setArchSuffix and set -xg HUB_ENTERPRISE "arangodb/enterprise-test:$DOCKER_TAG_JENKINS$archSuffix" and buildDockerImage $HUB_ENTERPRISE and validateDockerImageIfNeeded $HUB_COMMUNITY -and docker push $HUB_ENTERPRISE -and docker tag $HUB_ENTERPRISE $GCR_REG_PREFIX$HUB_ENTERPRISE -and docker push $GCR_REG_PREFIX$HUB_ENTERPRISE +and "$DOCKER" push $HUB_ENTERPRISE +and "$DOCKER" tag $HUB_ENTERPRISE $GCR_REG_PREFIX$HUB_ENTERPRISE +and "$DOCKER" push $GCR_REG_PREFIX$HUB_ENTERPRISE and echo $HUB_ENTERPRISE >> $WORKSPACE/imagenames.log and echo $GCR_REG_PREFIX$HUB_ENTERPRISE >> $WORKSPACE/imagenames.log diff --git a/jenkins/helper/runAnyPerformanceResults.fish b/jenkins/helper/runAnyPerformanceResults.fish index 12de6bd6..95ffaed0 100755 --- a/jenkins/helper/runAnyPerformanceResults.fish +++ b/jenkins/helper/runAnyPerformanceResults.fish @@ -141,7 +141,7 @@ if count $src/results-*.csv > /dev/null end echo "Generating images" - docker run -v (pwd)/work:/work pavlov99/gnuplot gnuplot $gp + "$DOCKER" run -v (pwd)/work:/work pavlov99/gnuplot gnuplot $gp or begin echo "=== $gp ===" cat $gp diff --git a/jenkins/helper/server.inc b/jenkins/helper/server.inc index 440231f4..c5173ba8 100644 --- a/jenkins/helper/server.inc +++ b/jenkins/helper/server.inc @@ -41,10 +41,10 @@ echo "AUTH: $ARANGO_AUTH" echo "EDITION: $ARANGO_EDITION" echo -docker kill $ARANGO_DOCKER_NAME > /dev/null 2>&1 || true -docker rm -fv $ARANGO_DOCKER_NAME > /dev/null 2>&1 || true -docker pull $DOCKER_IMAGE -docker run --rm --cap-add SYS_NICE $DOCKER_IMAGE arangosh --version | tee ./${ARANGO_DOCKER_NAME}.txt +"$DOCKER" kill $ARANGO_DOCKER_NAME > /dev/null 2>&1 || true +"$DOCKER" rm -fv $ARANGO_DOCKER_NAME > /dev/null 2>&1 || true +"$DOCKER" pull $DOCKER_IMAGE +"$DOCKER" run --rm --cap-add SYS_NICE $DOCKER_IMAGE arangosh --version | tee ./${ARANGO_DOCKER_NAME}.txt export SERVER_FULL_VERSION=$(cat ./${ARANGO_DOCKER_NAME}.txt | grep -oP '^server-version: \K.*$') OUTDIR="`pwd`/output" @@ -52,7 +52,7 @@ rm -rf $OUTDIR mkdir $OUTDIR DOCKER_AUTH="" STARTER_AUTH="" -DOCKER_CMD="docker run --rm --hostname localhost --cap-add SYS_NICE --name $ARANGO_DOCKER_NAME -d $EXPORT_PORTS -v $OUTDIR:/testrun" +DOCKER_CMD="$DOCKER run --rm --hostname localhost --cap-add SYS_NICE --name $ARANGO_DOCKER_NAME -d $EXPORT_PORTS -v $OUTDIR:/testrun" #DOCKER_IMAGE="registry.arangodb.biz:5000/arangodb/linux-${ARANGO_EDITION}-maintainer:$ARANGO_BRANCH" STARTER_CMD="arangodb --starter.address localhost --starter.local --server.storage-engine $ARANGO_STORAGE_ENGINE --starter.data-dir /testrun ${MMFILES_DEPRECATED_OPTION}" STARTER_MODE="" @@ -101,7 +101,7 @@ command="$DOCKER_CMD $DOCKER_AUTH $DOCKER_IMAGE /testrun/starter.sh $STARTER_CMD echo $command $command -trap "$(trap -p | grep EXIT | sed "s/.*-- '\(.*\)'.EXIT/\1;/g")docker rm -fv $ARANGO_DOCKER_NAME ; sudo chown jenkins:jenkins -R output ; tar czf arango_data.tar.gz output" EXIT +trap "$(trap -p | grep EXIT | sed "s/.*-- '\(.*\)'.EXIT/\1;/g") "$DOCKER" rm -fv $ARANGO_DOCKER_NAME ; sudo chown jenkins:jenkins -R output ; tar czf arango_data.tar.gz output" EXIT echo "Waiting until ArangoDB is ready on port ${ARANGO_PORTS[0]}" @@ -131,7 +131,7 @@ done if [ $count -ge 240 ]; then echo "docker logs:" - docker logs $ARANGO_DOCKER_NAME + "$DOCKER" logs $ARANGO_DOCKER_NAME echo echo "curl:" diff --git a/jenkins/runGobenchPerformance.fish b/jenkins/runGobenchPerformance.fish index 6893721d..751433fb 100755 --- a/jenkins/runGobenchPerformance.fish +++ b/jenkins/runGobenchPerformance.fish @@ -29,11 +29,11 @@ and popd and sudo rm -rf work/database $simple/results.csv and echo "==== starting performance run ====" -and echo "docker run -e ARANGO_LICENSE_KEY=$ARANGODB_LICENSE_KEY -v (pwd)/work/ArangoDB:/ArangoDB -v (pwd)/work:/data -v $simple:/performance -v $gobenchdir:/gobench" +and echo "$DOCKER run -e ARANGO_LICENSE_KEY=$ARANGODB_LICENSE_KEY -v (pwd)/work/ArangoDB:/ArangoDB -v (pwd)/work:/data -v $simple:/performance -v $gobenchdir:/gobench" and for protocol in VST HTTP echo "Protocol: " $protocol - docker run \ + "$DOCKER" run \ --cap-add SYS_NICE \ -e ARANGO_LICENSE_KEY=$ARANGODB_LICENSE_KEY \ -e ARANGO_BRANCH=$ARANGODB_BRANCH \ diff --git a/jenkins/runGobenchPerformanceResults.fish b/jenkins/runGobenchPerformanceResults.fish index 3f7127e8..ad106151 100755 --- a/jenkins/runGobenchPerformanceResults.fish +++ b/jenkins/runGobenchPerformanceResults.fish @@ -81,4 +81,4 @@ if test (count work/images/*.png) -gt 0 end echo "Generating images" -docker run -v (pwd)/work:/work pavlov99/gnuplot gnuplot $gp +"$DOCKER" run -v (pwd)/work:/work pavlov99/gnuplot gnuplot $gp diff --git a/jenkins/runSimplePerformance.fish b/jenkins/runSimplePerformance.fish index c664a37a..24214a3e 100755 --- a/jenkins/runSimplePerformance.fish +++ b/jenkins/runSimplePerformance.fish @@ -26,7 +26,7 @@ and sudo rm -rf work/database $simple/results.csv and cat /proc/sys/kernel/core_pattern and echo "==== starting performance run ====" and if test -z "$DOCKER_IMAGE" - docker run \ + "$DOCKER" run \ --ulimit core=-1 \ -e ARANGO_LICENSE_KEY=$ARANGODB_LICENSE_KEY \ -v (pwd)/work/ArangoDB:/ArangoDB \ @@ -44,7 +44,7 @@ and if test -z "$DOCKER_IMAGE" /data/database \ --javascript.script simple/$ARANGODB_TEST_CONFIG" else - docker run \ + "$DOCKER" run \ --ulimit core=-1 \ -e ARANGO_LICENSE_KEY=$ARANGODB_LICENSE_KEY \ -v (pwd)/work:/data \ @@ -63,7 +63,7 @@ end set -l s $status if count $simple/core* >/dev/null - docker run \ + "$DOCKER" run \ -v $simple:/performance \ --rm \ $DOCKER_IMAGE \ From 4b96d23eda3dae4f5d7b7b5dcbeaff1dc996e536 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 12 Mar 2025 10:37:37 +0100 Subject: [PATCH 05/33] trace --- jenkins/runFullNightlyTest.fish | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/runFullNightlyTest.fish b/jenkins/runFullNightlyTest.fish index 9365bfe0..212cbad0 100755 --- a/jenkins/runFullNightlyTest.fish +++ b/jenkins/runFullNightlyTest.fish @@ -1,4 +1,5 @@ #!/usr/bin/env fish +set -l fish_trace on source jenkins/helper/jenkins.fish cleanPrepareLockUpdateClear From 96f4c5f98e43bcaa3189b0fa5c64643c4442f65e Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 12 Mar 2025 10:43:54 +0100 Subject: [PATCH 06/33] trace --- scripts/switchBranches.fish | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/switchBranches.fish b/scripts/switchBranches.fish index 46ca2e99..839dc35f 100755 --- a/scripts/switchBranches.fish +++ b/scripts/switchBranches.fish @@ -1,4 +1,5 @@ #!/usr/bin/env fish +set -l fish_trace on function setupSourceInfo set -l field $argv[1] From 34cac104eb6b868dc0d62d9f0ff2dcf2dff969b2 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 13 Mar 2025 15:00:04 +0100 Subject: [PATCH 07/33] debug --- scripts/makeAlpine.fish | 2 +- scripts/makeArangoDB.fish | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/makeAlpine.fish b/scripts/makeAlpine.fish index 22822f4f..b6a2381b 100755 --- a/scripts/makeAlpine.fish +++ b/scripts/makeAlpine.fish @@ -19,4 +19,4 @@ or exit $status mkdir -p install set -x DESTDIR (pwd)/install -nice make -j$PARALLELISM $argv +nice make -j$PARALLELISM $argv VERBOSE=1 diff --git a/scripts/makeArangoDB.fish b/scripts/makeArangoDB.fish index 33290ad2..96b6f49b 100755 --- a/scripts/makeArangoDB.fish +++ b/scripts/makeArangoDB.fish @@ -21,4 +21,4 @@ if test "$PLATFORM" = "linux" set GOLD = -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=gold -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=gold end -nice make -j$PARALLELISM $MAKE_TARGETS $argv +nice make -j$PARALLELISM $MAKE_TARGETS $argv VERBOSE=1 From fda28616412ca5dae7e8ae649e0897b6626b3be7 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 13 Mar 2025 15:14:19 +0100 Subject: [PATCH 08/33] debug --- scripts/makeAlpine.fish | 2 +- scripts/makeArangoDB.fish | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/makeAlpine.fish b/scripts/makeAlpine.fish index b6a2381b..85da2f9b 100755 --- a/scripts/makeAlpine.fish +++ b/scripts/makeAlpine.fish @@ -19,4 +19,4 @@ or exit $status mkdir -p install set -x DESTDIR (pwd)/install -nice make -j$PARALLELISM $argv VERBOSE=1 +nice make -j$PARALLELISM $argv VERBOSE=1 V=1 diff --git a/scripts/makeArangoDB.fish b/scripts/makeArangoDB.fish index 96b6f49b..2294e4e8 100755 --- a/scripts/makeArangoDB.fish +++ b/scripts/makeArangoDB.fish @@ -21,4 +21,4 @@ if test "$PLATFORM" = "linux" set GOLD = -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=gold -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=gold end -nice make -j$PARALLELISM $MAKE_TARGETS $argv VERBOSE=1 +nice make -j$PARALLELISM $MAKE_TARGETS $argv VERBOSE=1 V=1 From a4ef4aff225ea0cfc53b7d0618d5810146b15228 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 13 Mar 2025 15:37:32 +0100 Subject: [PATCH 09/33] debug --- scripts/lib/build.fish | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/lib/build.fish b/scripts/lib/build.fish index 2f5e8849..2d27fe2e 100755 --- a/scripts/lib/build.fish +++ b/scripts/lib/build.fish @@ -220,10 +220,10 @@ function runMake end if test "$SHOW_DETAILS" = "On" - make $MAKEFLAGS $argv[1] 2>&1 + make $MAKEFLAGS $argv[1] VERBOSE=1 2>&1 or exit $status else - echo make output in work/buildArangoDB.log + echo make output in work/buildArangoDB.log set -l ep "" if test "$SHOW_DETAILS" = "Ping" @@ -232,9 +232,9 @@ function runMake end if test "$argv[1]" = "install" - nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log 2>&1 + nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 end - and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log 2>&1 + and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 or begin if test -n "$ep" kill $ep From 9fff47da8ab404b5ccb70684d5ee291bd5521481 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 13 Mar 2025 18:05:55 +0100 Subject: [PATCH 10/33] specify docker default args --- helper.fish | 1 + helper.linux.fish | 1 + 2 files changed, 2 insertions(+) diff --git a/helper.fish b/helper.fish index 75e1c68c..9ca9913e 100755 --- a/helper.fish +++ b/helper.fish @@ -5,6 +5,7 @@ if test -f /usr/bin/podman echo podman set -xg DOCKER "podman" set -xg DOCKER_URL_PREFIX "docker.io/" + set -xg DEFAULT_DOCKER_ARGS "--pids-limit 256704" else set -xg DOCKER "docker" end diff --git a/helper.linux.fish b/helper.linux.fish index 7fdebdd6..1e4101c0 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1700,6 +1700,7 @@ function runInContainer # if we receive a TERM outside the container. Note that this does not # cover SIGINT, since this will directly abort the whole function. set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ + $DEFAULT_DOCKER_ARGS \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ From 1598c03b2d3ea10972a5c607efb835542e0415d7 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 13 Mar 2025 18:09:03 +0100 Subject: [PATCH 11/33] specify docker default args --- helper.linux.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index 1e4101c0..4c5f4a62 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1700,7 +1700,7 @@ function runInContainer # if we receive a TERM outside the container. Note that this does not # cover SIGINT, since this will directly abort the whole function. set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ - $DEFAULT_DOCKER_ARGS \ + --pids-limit 256704 \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ From 615e0ef0c8a2ed5c0de0d7999a6f4bd67432ba00 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 13 Mar 2025 19:12:35 +0100 Subject: [PATCH 12/33] revert --- scripts/lib/build.fish | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/lib/build.fish b/scripts/lib/build.fish index 2d27fe2e..2f5e8849 100755 --- a/scripts/lib/build.fish +++ b/scripts/lib/build.fish @@ -220,10 +220,10 @@ function runMake end if test "$SHOW_DETAILS" = "On" - make $MAKEFLAGS $argv[1] VERBOSE=1 2>&1 + make $MAKEFLAGS $argv[1] 2>&1 or exit $status else - echo make output in work/buildArangoDB.log + echo make output in work/buildArangoDB.log set -l ep "" if test "$SHOW_DETAILS" = "Ping" @@ -232,9 +232,9 @@ function runMake end if test "$argv[1]" = "install" - nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 + nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log 2>&1 end - and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 + and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log 2>&1 or begin if test -n "$ep" kill $ep From efc00aa9bd9d83dc9c03cfe55f14f7ad87cfa2c2 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 14 Mar 2025 11:11:43 +0100 Subject: [PATCH 13/33] debug --- scripts/lib/build.fish | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/lib/build.fish b/scripts/lib/build.fish index 2f5e8849..2d27fe2e 100755 --- a/scripts/lib/build.fish +++ b/scripts/lib/build.fish @@ -220,10 +220,10 @@ function runMake end if test "$SHOW_DETAILS" = "On" - make $MAKEFLAGS $argv[1] 2>&1 + make $MAKEFLAGS $argv[1] VERBOSE=1 2>&1 or exit $status else - echo make output in work/buildArangoDB.log + echo make output in work/buildArangoDB.log set -l ep "" if test "$SHOW_DETAILS" = "Ping" @@ -232,9 +232,9 @@ function runMake end if test "$argv[1]" = "install" - nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log 2>&1 + nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 end - and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log 2>&1 + and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 or begin if test -n "$ep" kill $ep From 9d9cf1ff53f38a174b2c8b79ec9f73e2fdd5a5bf Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 14 Mar 2025 13:33:11 +0100 Subject: [PATCH 14/33] debug --- scripts/lib/build.fish | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/lib/build.fish b/scripts/lib/build.fish index 2d27fe2e..38c65423 100755 --- a/scripts/lib/build.fish +++ b/scripts/lib/build.fish @@ -286,6 +286,7 @@ function TT_cmake end function TT_make + set -l fish_trace on set -g TT_t3 (date -u +%s) and echo $TT_t0,make,(expr $TT_t3 - $TT_t2) >> $INNERWORKDIR/buildTimes.csv end From f75d85d900e3ef7095146169e3d6ebb8a5758ff0 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 14 Mar 2025 13:50:28 +0100 Subject: [PATCH 15/33] specify memory --- helper.linux.fish | 1 + 1 file changed, 1 insertion(+) diff --git a/helper.linux.fish b/helper.linux.fish index 4c5f4a62..068d8236 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1701,6 +1701,7 @@ function runInContainer # cover SIGINT, since this will directly abort the whole function. set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ --pids-limit 256704 \ + -m 64G \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ From 855e7276faa3ae82ddb558ebd86b8c435837c241 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 14 Mar 2025 14:02:14 +0100 Subject: [PATCH 16/33] specify memory --- helper.linux.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index 068d8236..bf644d14 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1701,7 +1701,7 @@ function runInContainer # cover SIGINT, since this will directly abort the whole function. set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ --pids-limit 256704 \ - -m 64G \ + -m 128G \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ From 7c1c5d69e7733716ef338c1384606242aac6cb5c Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 14 Mar 2025 14:08:39 +0100 Subject: [PATCH 17/33] specify memory --- helper.linux.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index bf644d14..fa15472a 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1701,7 +1701,7 @@ function runInContainer # cover SIGINT, since this will directly abort the whole function. set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ --pids-limit 256704 \ - -m 128G \ + -m 256G \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ From a2ed2824bf18ee2aca10ddd5bf72067674ce4895 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 14 Mar 2025 14:26:33 +0100 Subject: [PATCH 18/33] specify memory --- helper.linux.fish | 2 +- scripts/lib/build.fish | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helper.linux.fish b/helper.linux.fish index fa15472a..068d8236 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1701,7 +1701,7 @@ function runInContainer # cover SIGINT, since this will directly abort the whole function. set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ --pids-limit 256704 \ - -m 256G \ + -m 64G \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \ diff --git a/scripts/lib/build.fish b/scripts/lib/build.fish index 38c65423..a03781b1 100755 --- a/scripts/lib/build.fish +++ b/scripts/lib/build.fish @@ -220,7 +220,7 @@ function runMake end if test "$SHOW_DETAILS" = "On" - make $MAKEFLAGS $argv[1] VERBOSE=1 2>&1 + make $MAKEFLAGS $argv[1] VERBOSE=1 -j 32 2>&1 or exit $status else echo make output in work/buildArangoDB.log From 5ffb1b1c90bf18f6d46e72ae02898b2af0746141 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 4 Apr 2025 15:42:54 +0200 Subject: [PATCH 19/33] silence --- jenkins/runFullNightlyTest.fish | 1 - scripts/lib/build.fish | 7 +++---- scripts/makeAlpine.fish | 2 +- scripts/makeArangoDB.fish | 2 +- scripts/switchBranches.fish | 1 - 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/jenkins/runFullNightlyTest.fish b/jenkins/runFullNightlyTest.fish index 212cbad0..9365bfe0 100755 --- a/jenkins/runFullNightlyTest.fish +++ b/jenkins/runFullNightlyTest.fish @@ -1,5 +1,4 @@ #!/usr/bin/env fish -set -l fish_trace on source jenkins/helper/jenkins.fish cleanPrepareLockUpdateClear diff --git a/scripts/lib/build.fish b/scripts/lib/build.fish index a03781b1..42d13fd5 100755 --- a/scripts/lib/build.fish +++ b/scripts/lib/build.fish @@ -220,7 +220,7 @@ function runMake end if test "$SHOW_DETAILS" = "On" - make $MAKEFLAGS $argv[1] VERBOSE=1 -j 32 2>&1 + make $MAKEFLAGS $argv[1] 2>&1 or exit $status else echo make output in work/buildArangoDB.log @@ -232,9 +232,9 @@ function runMake end if test "$argv[1]" = "install" - nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 + nice make $MAKEFLAGS > $INNERWORKDIR/buildArangoDB.log 2>&1 end - and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log VERBOSE=1 2>&1 + and nice make $MAKEFLAGS $argv[1] >> $INNERWORKDIR/buildArangoDB.log 2>&1 or begin if test -n "$ep" kill $ep @@ -286,7 +286,6 @@ function TT_cmake end function TT_make - set -l fish_trace on set -g TT_t3 (date -u +%s) and echo $TT_t0,make,(expr $TT_t3 - $TT_t2) >> $INNERWORKDIR/buildTimes.csv end diff --git a/scripts/makeAlpine.fish b/scripts/makeAlpine.fish index 85da2f9b..22822f4f 100755 --- a/scripts/makeAlpine.fish +++ b/scripts/makeAlpine.fish @@ -19,4 +19,4 @@ or exit $status mkdir -p install set -x DESTDIR (pwd)/install -nice make -j$PARALLELISM $argv VERBOSE=1 V=1 +nice make -j$PARALLELISM $argv diff --git a/scripts/makeArangoDB.fish b/scripts/makeArangoDB.fish index 2294e4e8..33290ad2 100755 --- a/scripts/makeArangoDB.fish +++ b/scripts/makeArangoDB.fish @@ -21,4 +21,4 @@ if test "$PLATFORM" = "linux" set GOLD = -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=gold -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=gold end -nice make -j$PARALLELISM $MAKE_TARGETS $argv VERBOSE=1 V=1 +nice make -j$PARALLELISM $MAKE_TARGETS $argv diff --git a/scripts/switchBranches.fish b/scripts/switchBranches.fish index 839dc35f..46ca2e99 100755 --- a/scripts/switchBranches.fish +++ b/scripts/switchBranches.fish @@ -1,5 +1,4 @@ #!/usr/bin/env fish -set -l fish_trace on function setupSourceInfo set -l field $argv[1] From 1e068f264f9fc0e809966f30218a95d6a1a6e04b Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 4 Apr 2025 15:52:29 +0200 Subject: [PATCH 20/33] use default container args --- helper.linux.fish | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index 068d8236..2b080da7 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1699,8 +1699,9 @@ function runInContainer # from a regular user. Therefore we have to do some Eiertanz to stop it # if we receive a TERM outside the container. Note that this does not # cover SIGINT, since this will directly abort the whole function. + set -l fish_trace on set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ - --pids-limit 256704 \ + "${DEFAULT_DOCKER_ARGS}" \ -m 64G \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ From b0f4a3184e797e745219a5716ed3d92b2298f7a6 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 4 Apr 2025 15:53:51 +0200 Subject: [PATCH 21/33] use default container args --- helper.linux.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index 2b080da7..cb60a052 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1701,7 +1701,7 @@ function runInContainer # cover SIGINT, since this will directly abort the whole function. set -l fish_trace on set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ - "${DEFAULT_DOCKER_ARGS}" \ + "$DEFAULT_DOCKER_ARGS" \ -m 64G \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ From 6339ce81d622a8ed1d190e05a2bf7da61966cc2d Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 4 Apr 2025 15:58:40 +0200 Subject: [PATCH 22/33] use default container args --- helper.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.fish b/helper.fish index 9ca9913e..e34cd238 100755 --- a/helper.fish +++ b/helper.fish @@ -5,7 +5,7 @@ if test -f /usr/bin/podman echo podman set -xg DOCKER "podman" set -xg DOCKER_URL_PREFIX "docker.io/" - set -xg DEFAULT_DOCKER_ARGS "--pids-limit 256704" + set -xg DEFAULT_DOCKER_ARGS --pids-limit 256704 else set -xg DOCKER "docker" end From 7f23aa4df7ea098883d2732ed501bce073894b88 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 4 Apr 2025 15:59:23 +0200 Subject: [PATCH 23/33] use default container args --- helper.linux.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index cb60a052..5f098b21 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1701,7 +1701,7 @@ function runInContainer # cover SIGINT, since this will directly abort the whole function. set -l fish_trace on set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ - "$DEFAULT_DOCKER_ARGS" \ + $DEFAULT_DOCKER_ARGS \ -m 64G \ -v $WORKDIR/work/:$INNERWORKDIR \ -v $SSH_AUTH_SOCK:/ssh-agent \ From 59d01585ba9ffc031318f7b0962469079c224d35 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 4 Apr 2025 16:01:32 +0200 Subject: [PATCH 24/33] silence --- helper.linux.fish | 1 - 1 file changed, 1 deletion(-) diff --git a/helper.linux.fish b/helper.linux.fish index 5f098b21..efd4c171 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1699,7 +1699,6 @@ function runInContainer # from a regular user. Therefore we have to do some Eiertanz to stop it # if we receive a TERM outside the container. Note that this does not # cover SIGINT, since this will directly abort the whole function. - set -l fish_trace on set c ("$DOCKER" run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \ $DEFAULT_DOCKER_ARGS \ -m 64G \ From bc64088a708847b1e573913eb391e619b1334074 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 4 Apr 2025 16:10:09 +0200 Subject: [PATCH 25/33] add pppid --- jenkins/helper/async_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/helper/async_client.py b/jenkins/helper/async_client.py index 0e972ffe..4599687e 100644 --- a/jenkins/helper/async_client.py +++ b/jenkins/helper/async_client.py @@ -211,7 +211,7 @@ def kill_children(identifier, params, children): killed.append(one_child.pid) err += add_message_to_report( params, - f"{identifier}: killing {one_child.name()} - {str(one_child.pid)}", + f"{identifier}: killing {one_child.name()} - {str(one_child.pid)} Parent: {str(one_child.ppid)}", True, True) one_child.resume() except FileNotFoundError: From 40a2577fc822e3617a29c97fdcb8d9fc64761b23 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 7 Apr 2025 16:54:30 +0200 Subject: [PATCH 26/33] fix script? --- jenkins/helper/aggregate_coverage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins/helper/aggregate_coverage.py b/jenkins/helper/aggregate_coverage.py index e97943c4..94f93ddc 100644 --- a/jenkins/helper/aggregate_coverage.py +++ b/jenkins/helper/aggregate_coverage.py @@ -283,6 +283,10 @@ def combine_coverage_dirs_multi(cfg, if combined_dir.exists(): shutil.rmtree(str(combined_dir)) combined_dir.mkdir() + coverage_dir = base_dir / 'coverage' + # if coverage_dir.exists(): + # shutil.rmtree(str(coverage_dir)) + coverage_dir.mkdir() count = 0 jobcount = 0 if len(sub_jobs) == 0: From 0de1cc53329ea895f83435fdea1af22ec0039054 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 7 Apr 2025 17:26:35 +0200 Subject: [PATCH 27/33] set var in both cases --- helper.fish | 1 + 1 file changed, 1 insertion(+) diff --git a/helper.fish b/helper.fish index e34cd238..c2855d1f 100755 --- a/helper.fish +++ b/helper.fish @@ -8,6 +8,7 @@ if test -f /usr/bin/podman set -xg DEFAULT_DOCKER_ARGS --pids-limit 256704 else set -xg DOCKER "docker" + set -xg DOCKER_URL_PREFIX "" end From 27960fc2ffafe174b8ab41d8c09a62f93d193120 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 8 Apr 2025 11:36:55 +0200 Subject: [PATCH 28/33] abort on error --- jenkins/runRTAtest.fish | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/runRTAtest.fish b/jenkins/runRTAtest.fish index 2d7b282a..ae014a0a 100755 --- a/jenkins/runRTAtest.fish +++ b/jenkins/runRTAtest.fish @@ -10,6 +10,7 @@ and eval $EDITION and setAllLogsToWorkspace and switchBranches $ARANGODB_BRANCH $ENTERPRISE_BRANCH true and updateDockerBuildImage +or exit 1 if test "$ASAN" = "true" echo "San build" sanOn From 87d5e555a01a174bfb86363dc98304e0ddf45c8f Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 25 Apr 2025 14:24:15 +0200 Subject: [PATCH 29/33] move if to the top --- jenkins/runRTAtest.fish | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/jenkins/runRTAtest.fish b/jenkins/runRTAtest.fish index ae014a0a..877cdfd5 100755 --- a/jenkins/runRTAtest.fish +++ b/jenkins/runRTAtest.fish @@ -7,6 +7,9 @@ and TT_init and set -xg RTA_EDITION "C,Cr2" and maintainerOn and eval $EDITION +if test "$ENTERPRISEEDITION" = "On" + set -xg RTA_EDITION "EP,EPr2" +end and setAllLogsToWorkspace and switchBranches $ARANGODB_BRANCH $ENTERPRISE_BRANCH true and updateDockerBuildImage @@ -29,6 +32,16 @@ if test "$BUILD_MODE" = "release" echo "switching to release build" releaseMode end + + +and maintainerOn +and eval $EDITION +and eval $TEST_SUITE +and setAllLogsToWorkspace +and switchBranches $ARANGODB_BRANCH $ENTERPRISE_BRANCH true +and updateDockerBuildImage +and begin +end and pingDetails and TT_setup and oskarCompile @@ -36,11 +49,7 @@ and TT_compile and downloadAuxBinariesToBuildBin and checkoutRTA and cd work/release-test-automation/ - -if test "$ENTERPRISEEDITION" = "On" - set -xg RTA_EDITION "EP,EPr2" -end -bash -x ./jenkins/oskar_tar.sh $argv +and bash -x ./jenkins/oskar_tar.sh $argv set -l s $status From 8ed05d31e7aec0bd82678a9f894ec547a33644a1 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 8 Apr 2025 11:53:54 +0200 Subject: [PATCH 30/33] properly configure git --- scripts/checkoutRTA.fish | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/checkoutRTA.fish b/scripts/checkoutRTA.fish index ab772061..682d4502 100755 --- a/scripts/checkoutRTA.fish +++ b/scripts/checkoutRTA.fish @@ -4,6 +4,10 @@ ssh -o StrictHostKeyChecking=no -T git@$ARANGODB_GIT_HOST cd $INNERWORKDIR and git config --global http.postBuffer 524288000 and git config --global https.postBuffer 524288000 + +git config --global user.email "release-bot@arangodb.com" +git config --global user.name "Release test automation" + # and git config --global pull.rebase true and if test ! -d release-test-automation/.git rm -rf release-test-automation From 9157f8e9155c3a10174fd401cc0f17b5c8752f6c Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 8 Apr 2025 11:55:08 +0200 Subject: [PATCH 31/33] properly configure git --- scripts/checkoutRTA.fish | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/checkoutRTA.fish b/scripts/checkoutRTA.fish index 682d4502..7b3f60b5 100755 --- a/scripts/checkoutRTA.fish +++ b/scripts/checkoutRTA.fish @@ -4,9 +4,8 @@ ssh -o StrictHostKeyChecking=no -T git@$ARANGODB_GIT_HOST cd $INNERWORKDIR and git config --global http.postBuffer 524288000 and git config --global https.postBuffer 524288000 - -git config --global user.email "release-bot@arangodb.com" -git config --global user.name "Release test automation" +and git config --global user.email "release-bot@arangodb.com" +and git config --global user.name "Release test automation" # and git config --global pull.rebase true and if test ! -d release-test-automation/.git From 1074f4f3e2a3dbdfb998829ade34e1507026d758 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 11 Apr 2025 11:51:28 +0200 Subject: [PATCH 32/33] try not to join arguments --- jenkins/runRTAtest.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/runRTAtest.fish b/jenkins/runRTAtest.fish index 877cdfd5..9ed6e296 100755 --- a/jenkins/runRTAtest.fish +++ b/jenkins/runRTAtest.fish @@ -49,7 +49,7 @@ and TT_compile and downloadAuxBinariesToBuildBin and checkoutRTA and cd work/release-test-automation/ -and bash -x ./jenkins/oskar_tar.sh $argv +and eval "bash -x ./jenkins/oskar_tar.sh $argv" set -l s $status From 38db423d67fb72131a20d6c4a4577babb7460064 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 25 Apr 2025 13:10:26 +0200 Subject: [PATCH 33/33] delete line that does no good --- jenkins/runRTAtest.fish | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/runRTAtest.fish b/jenkins/runRTAtest.fish index 9ed6e296..89550e4a 100755 --- a/jenkins/runRTAtest.fish +++ b/jenkins/runRTAtest.fish @@ -36,7 +36,6 @@ end and maintainerOn and eval $EDITION -and eval $TEST_SUITE and setAllLogsToWorkspace and switchBranches $ARANGODB_BRANCH $ENTERPRISE_BRANCH true and updateDockerBuildImage